snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/mount/dir_sink.go

@@ -0,0 +1,43 @@
+package mount
+
+import (
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+)
+
+// DirEntrySink receives the entries a readdir produces. The FUSE server packs
+// them straight into the kernel's reply buffer; a front end that is not the
+// kernel reads them out instead of re-parsing that wire format.
+type DirEntrySink interface {
+	// AddEntry reports one entry, returning false once the sink is full. A
+	// full sink ends the batch; the client resumes from the entry's Off.
+	AddEntry(entry fuse.DirEntry) bool
+
+	// AddEntryPlus is AddEntry for readdirplus, returning the attribute block
+	// to fill in, or nil once the sink is full.
+	AddEntryPlus(entry fuse.DirEntry) *fuse.EntryOut
+
+	// TakesLookupRef reports whether AddEntryPlus hands the sink a reference the
+	// mount must hold until a FORGET returns it.
+	TakesLookupRef() bool
+}
+
+// fuseDirEntryList adapts the kernel reply buffer to DirEntrySink.
+type fuseDirEntryList struct {
+	*fuse.DirEntryList
+}
+
+func (l fuseDirEntryList) AddEntry(entry fuse.DirEntry) bool {
+	return l.AddDirEntry(entry)
+}
+
+func (l fuseDirEntryList) AddEntryPlus(entry fuse.DirEntry) *fuse.EntryOut {
+	return l.AddDirLookupEntry(entry)
+}
+
+func (l fuseDirEntryList) TakesLookupRef() bool { return true }
+
+// ReadDirectoryInto runs a readdir against sink. ReadDir and ReadDirPlus are
+// this with the kernel reply buffer as the sink.
+func (wfs *WFS) ReadDirectoryInto(input *fuse.ReadIn, sink DirEntrySink, isPlusMode bool) fuse.Status {
+	return wfs.doReadDirectory(input, sink, isPlusMode)
+}

weed/mount/dirty_pages_chunked.go

@@ -0,0 +1,131 @@
+package mount
+
+import (
+	"fmt"
+	"io"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mount/page_writer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+type ChunkedDirtyPages struct {
+	fh             *FileHandle
+	writeWaitGroup sync.WaitGroup
+	lastErrLock    sync.Mutex
+	lastErr        error
+	collection     string
+	replication    string
+	uploadPipeline *page_writer.UploadPipeline
+	hasWrites      bool
+}
+
+var (
+	_ = page_writer.DirtyPages(&ChunkedDirtyPages{})
+)
+
+func newMemoryChunkPages(fh *FileHandle, chunkSize int64) *ChunkedDirtyPages {
+
+	dirtyPages := &ChunkedDirtyPages{
+		fh: fh,
+	}
+
+	swapFileDir := fh.wfs.option.getUniqueCacheDirForWrite()
+
+	dirtyPages.uploadPipeline = page_writer.NewUploadPipeline(fh.wfs.concurrentWriters, chunkSize,
+		dirtyPages.saveChunkedFileIntervalToStorage, fh.wfs.option.ConcurrentWriters, swapFileDir,
+		fh.wfs.writeBufferAccountant)
+
+	return dirtyPages
+}
+
+func (pages *ChunkedDirtyPages) AddPage(offset int64, data []byte, isSequential bool, tsNs int64) error {
+	// A failed chunk upload already dropped its data, so this handle can never
+	// flush cleanly. Reject further writes so the writer fails fast instead of
+	// pushing the rest of the file through a pipeline that cannot persist it.
+	if err := pages.LastError(); err != nil {
+		return err
+	}
+	pages.hasWrites = true
+
+	glog.V(4).Infof("%v memory AddPage [%d, %d)", pages.fh.fh, offset, offset+int64(len(data)))
+	_, err := pages.uploadPipeline.SaveDataAt(data, offset, isSequential, tsNs)
+
+	return err
+}
+
+func (pages *ChunkedDirtyPages) HasWrites() bool {
+	return pages.hasWrites
+}
+
+func (pages *ChunkedDirtyPages) FlushData() error {
+	if !pages.hasWrites {
+		return nil
+	}
+	pages.uploadPipeline.FlushAll()
+	if err := pages.LastError(); err != nil {
+		return fmt.Errorf("flush data: %w", err)
+	}
+	return nil
+}
+
+// LastError returns the first chunk upload failure on this handle. It is
+// sticky: the failed chunk's data is gone, so the handle stays poisoned.
+func (pages *ChunkedDirtyPages) LastError() error {
+	pages.lastErrLock.Lock()
+	defer pages.lastErrLock.Unlock()
+	return pages.lastErr
+}
+
+func (pages *ChunkedDirtyPages) setLastError(err error) {
+	pages.lastErrLock.Lock()
+	defer pages.lastErrLock.Unlock()
+	if pages.lastErr == nil {
+		pages.lastErr = err
+	}
+}
+
+func (pages *ChunkedDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64, tsNs int64) (maxStop int64) {
+	if !pages.hasWrites {
+		return
+	}
+	return pages.uploadPipeline.MaybeReadDataAt(data, startOffset, tsNs)
+}
+
+func (pages *ChunkedDirtyPages) saveChunkedFileIntervalToStorage(reader io.Reader, offset int64, size int64, modifiedTsNs int64, cleanupFn func()) {
+
+	defer cleanupFn()
+
+	fileFullPath := pages.fh.FullPath()
+	fileName := fileFullPath.Name()
+	chunk, err := pages.fh.wfs.saveDataAsChunk(fileFullPath)(reader, fileName, offset, modifiedTsNs, uint64(size))
+	if err != nil {
+		glog.V(0).Infof("%v saveToStorage [%d,%d): %v", fileFullPath, offset, offset+size, err)
+		pages.setLastError(err)
+		return
+	}
+	pages.fh.AddChunks([]*filer_pb.FileChunk{chunk})
+	pages.fh.entryChunkGroup.AddChunk(chunk)
+	glog.V(3).Infof("%v saveToStorage %s [%d,%d)", fileFullPath, chunk.FileId, offset, offset+size)
+
+}
+
+func (pages *ChunkedDirtyPages) Destroy() {
+	pages.uploadPipeline.Shutdown()
+}
+
+func (pages *ChunkedDirtyPages) EvictOneWritableChunk() bool {
+	return pages.uploadPipeline.EvictOneWritableChunk()
+}
+
+func (pages *ChunkedDirtyPages) ProactiveFlush(nowNs, idleThresholdNs, maxHoldNs, fillRatio int64, frontierLag int, isSequential bool) bool {
+	return pages.uploadPipeline.ProactiveFlush(nowNs, idleThresholdNs, maxHoldNs, fillRatio, frontierLag, isSequential)
+}
+
+func (pages *ChunkedDirtyPages) LockForRead(startOffset, stopOffset int64) {
+	pages.uploadPipeline.LockForRead(startOffset, stopOffset)
+}
+func (pages *ChunkedDirtyPages) UnlockForRead(startOffset, stopOffset int64) {
+	pages.uploadPipeline.UnlockForRead(startOffset, stopOffset)
+}

weed/mount/dirty_pages_chunked_test.go

@@ -0,0 +1,31 @@
+package mount
+
+import (
+	"errors"
+	"testing"
+)
+
+// Once a chunk upload fails its data is gone, so the handle must reject
+// further writes instead of buffering the rest of the file — against a full
+// cluster that turned cp into an hours-long crawl that only errored at close.
+func TestChunkedDirtyPagesFailWritesAfterUploadError(t *testing.T) {
+	fh := &FileHandle{wfs: &WFS{option: &Option{}}}
+	pages := newMemoryChunkPages(fh, 1024)
+	defer pages.Destroy()
+	pages.hasWrites = true
+
+	uploadErr := errors.New("assign volume failure: no writable volumes")
+	pages.setLastError(uploadErr)
+
+	if err := pages.AddPage(0, []byte("x"), true, 1); !errors.Is(err, uploadErr) {
+		t.Fatalf("AddPage after upload failure = %v, want sticky %v", err, uploadErr)
+	}
+	if err := pages.FlushData(); !errors.Is(err, uploadErr) {
+		t.Fatalf("FlushData after upload failure = %v, want wrapped %v", err, uploadErr)
+	}
+	// First failure wins; later errors must not mask the root cause.
+	pages.setLastError(errors.New("later error"))
+	if err := pages.LastError(); !errors.Is(err, uploadErr) {
+		t.Fatalf("LastError = %v, want first error %v", err, uploadErr)
+	}
+}

weed/mount/error_classifier.go

@@ -0,0 +1,96 @@
+package mount
+
+import (
+	"strings"
+	"syscall"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func grpcErrorToFuseStatus(err error) fuse.Status {
+	if err == nil {
+		return fuse.OK
+	}
+
+	// Unpack error for inspection
+	if s, ok := status.FromError(err); ok {
+		switch s.Code() {
+		case codes.OK:
+			return fuse.OK
+		case codes.Canceled, codes.DeadlineExceeded:
+			return fuse.Status(syscall.ETIMEDOUT)
+		case codes.Unavailable:
+			return fuse.Status(syscall.EAGAIN)
+		case codes.ResourceExhausted:
+			return fuse.Status(syscall.EAGAIN) // Or syscall.ENOSPC
+		case codes.PermissionDenied:
+			return fuse.Status(syscall.EACCES)
+		case codes.Unauthenticated:
+			return fuse.Status(syscall.EPERM)
+		case codes.NotFound:
+			return fuse.ENOENT
+		case codes.AlreadyExists:
+			return fuse.Status(syscall.EEXIST)
+		case codes.InvalidArgument:
+			return fuse.EINVAL
+		}
+	}
+
+	// String matching for errors that don't have proper gRPC codes but are known
+	errStr := err.Error()
+	if strings.Contains(errStr, "transport") {
+		return fuse.Status(syscall.EAGAIN)
+	}
+	// Add other string matches if necessary
+
+	return fuse.EIO
+}
+
+// isStorageFullError reports whether a chunk upload failed because the cluster
+// has no writable volume left (disks full and volume growth failed). The
+// master's message reaches the mount as plain text inside
+// AssignVolumeResponse.Error, so match on text rather than a status code.
+func isStorageFullError(err error) bool {
+	if err == nil {
+		return false
+	}
+	errStr := strings.ToLower(err.Error())
+	return strings.Contains(errStr, "no writable volumes") ||
+		strings.Contains(errStr, "no free volumes")
+}
+
+// writeErrorToFuseStatus maps a dirty-page write/flush failure to the status
+// returned to the kernel: ENOSPC when the cluster is out of space, EIO otherwise.
+func writeErrorToFuseStatus(err error) fuse.Status {
+	if isStorageFullError(err) {
+		return fuse.Status(syscall.ENOSPC)
+	}
+	return fuse.EIO
+}
+
+// isRetryableFilerError reports whether a filer RPC error looks transient
+// enough to retry. It takes a conservative whitelist approach: only errors
+// that clearly describe a permanent application-level failure
+// (NotFound/AlreadyExists/InvalidArgument/PermissionDenied/Unauthenticated/
+// FailedPrecondition) short-circuit the retry loop. Everything else —
+// transport errors, Canceled/Unavailable/ResourceExhausted, or errors with no
+// gRPC status — is treated as potentially transient and retried.
+func isRetryableFilerError(err error) bool {
+	if err == nil {
+		return false
+	}
+	if s, ok := status.FromError(err); ok {
+		switch s.Code() {
+		case codes.NotFound,
+			codes.AlreadyExists,
+			codes.InvalidArgument,
+			codes.PermissionDenied,
+			codes.Unauthenticated,
+			codes.FailedPrecondition:
+			return false
+		}
+	}
+	return true
+}

weed/mount/error_classifier_test.go

@@ -0,0 +1,162 @@
+package mount
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// Regression: wfs_save.go used to wrap the gRPC error with fmt.Errorf(... %v ...)
+// which stringified the status and made status.FromError fall through to the
+// default EIO mapping. Wrapping with %w must preserve the code so that
+// codes.Canceled from a closing filer connection surfaces as ETIMEDOUT (a
+// retryable hint for FUSE callers) rather than EIO.
+func TestGrpcErrorToFuseStatusUnwrapsCanceledThroughFmtErrorf(t *testing.T) {
+	grpcErr := status.Error(codes.Canceled, "grpc: the client connection is closing")
+
+	wrapped := fmt.Errorf("UpdateEntry dir /some/path: %w", grpcErr)
+
+	got := grpcErrorToFuseStatus(wrapped)
+	want := fuse.Status(syscall.ETIMEDOUT)
+	if got != want {
+		t.Fatalf("grpcErrorToFuseStatus(canceled wrapped with %%w) = %v, want %v", got, want)
+	}
+}
+
+// Guard against regressing the wrap verb: %v loses the gRPC status and the
+// classifier must fall through to EIO. This test documents that behavior so
+// anyone reverting the %w change sees the intent.
+func TestGrpcErrorToFuseStatusDropsCanceledThroughPercentV(t *testing.T) {
+	grpcErr := status.Error(codes.Canceled, "grpc: the client connection is closing")
+
+	wrapped := fmt.Errorf("UpdateEntry dir /some/path: %v", grpcErr)
+
+	got := grpcErrorToFuseStatus(wrapped)
+	if got != fuse.EIO {
+		t.Fatalf("grpcErrorToFuseStatus(canceled wrapped with %%v) = %v, want EIO (regression guard)", got)
+	}
+}
+
+// A full cluster must surface as ENOSPC so cp/rsync abort with "No space left
+// on device" instead of a generic I/O error. The master's message reaches the
+// mount stringified inside AssignVolumeResponse.Error, in one of two shapes:
+// the load-shedding gRPC status text or the topology's plain error text.
+func TestWriteErrorToFuseStatus(t *testing.T) {
+	cases := []struct {
+		name string
+		err  error
+		want fuse.Status
+	}{
+		{
+			"assign shed during growth",
+			errors.New("upload data: filerGrpcAddress assign volume: assign volume failure count:1: assign volume: rpc error: code = ResourceExhausted desc = no writable volumes for replication:000 collection:, volume growth in progress"),
+			fuse.Status(syscall.ENOSPC),
+		},
+		{
+			"no free volumes left",
+			errors.New("flush data: upload data: assign volume: No writable volumes available for collection: replication:000 ttl: and no free volumes left for {replication:000}"),
+			fuse.Status(syscall.ENOSPC),
+		},
+		{
+			"unrelated upload failure",
+			errors.New("upload data: put http://volume:8080/3,0144: connection reset by peer"),
+			fuse.EIO,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := writeErrorToFuseStatus(tc.err); got != tc.want {
+				t.Fatalf("writeErrorToFuseStatus(%v) = %v, want %v", tc.err, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestIsRetryableFilerError(t *testing.T) {
+	cases := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{"nil", nil, false},
+		{"canceled", status.Error(codes.Canceled, "grpc: the client connection is closing"), true},
+		{"unavailable", status.Error(codes.Unavailable, "connection refused"), true},
+		{"deadline_exceeded", status.Error(codes.DeadlineExceeded, "deadline exceeded"), true},
+		{"resource_exhausted", status.Error(codes.ResourceExhausted, "too many concurrent requests"), true},
+		{"internal", status.Error(codes.Internal, "server crashed"), true},
+		{"not_found", status.Error(codes.NotFound, "entry missing"), false},
+		{"already_exists", status.Error(codes.AlreadyExists, "duplicate"), false},
+		{"invalid_argument", status.Error(codes.InvalidArgument, "bad request"), false},
+		{"permission_denied", status.Error(codes.PermissionDenied, "no access"), false},
+		{"unauthenticated", status.Error(codes.Unauthenticated, "missing creds"), false},
+		{"failed_precondition", status.Error(codes.FailedPrecondition, "not empty"), false},
+		{"plain_error_retries", errors.New("random network glitch"), true},
+		{"wrapped_canceled_still_retries", fmt.Errorf("ctx: %w", status.Error(codes.Canceled, "closing")), true},
+		{"wrapped_not_found_still_skipped", fmt.Errorf("ctx: %w", status.Error(codes.NotFound, "gone")), false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := isRetryableFilerError(tc.err); got != tc.want {
+				t.Fatalf("isRetryableFilerError(%v) = %v, want %v", tc.err, got, tc.want)
+			}
+		})
+	}
+}
+
+// retryMetadataFlushIf must short-circuit on non-retryable errors so that
+// synchronous FUSE ops (chmod/utimes/xattr) don't hang for ~7s on ENOENT/
+// EACCES/EINVAL.
+func TestRetryMetadataFlushIfShortCircuitsOnPermanentError(t *testing.T) {
+	originalSleep := metadataFlushSleep
+	t.Cleanup(func() {
+		metadataFlushSleep = originalSleep
+	})
+	metadataFlushSleep = func(_ context.Context, _ time.Duration) {
+		t.Fatal("sleep should not be called when shouldRetry returns false")
+	}
+
+	attempts := 0
+	permanent := status.Error(codes.NotFound, "entry missing")
+	err := retryMetadataFlushIf(context.Background(), func() error {
+		attempts++
+		return permanent
+	}, isRetryableFilerError, nil)
+
+	if attempts != 1 {
+		t.Fatalf("attempts = %d, want 1 (permanent error should short-circuit)", attempts)
+	}
+	if !errors.Is(err, permanent) {
+		t.Fatalf("err = %v, want permanent sentinel", err)
+	}
+}
+
+// Transient errors must keep retrying up to the attempt cap even when a
+// predicate is supplied.
+func TestRetryMetadataFlushIfRetriesTransientErrors(t *testing.T) {
+	originalSleep := metadataFlushSleep
+	t.Cleanup(func() {
+		metadataFlushSleep = originalSleep
+	})
+	metadataFlushSleep = func(_ context.Context, _ time.Duration) {}
+
+	attempts := 0
+	transient := status.Error(codes.Canceled, "grpc: the client connection is closing")
+	err := retryMetadataFlushIf(context.Background(), func() error {
+		attempts++
+		return transient
+	}, isRetryableFilerError, nil)
+
+	if attempts != metadataFlushRetries+1 {
+		t.Fatalf("attempts = %d, want %d", attempts, metadataFlushRetries+1)
+	}
+	if !errors.Is(err, transient) {
+		t.Fatalf("err = %v, want transient sentinel", err)
+	}
+}

weed/mount/filehandle.go

@@ -0,0 +1,232 @@
+package mount
+
+import (
+	"os"
+	"sync"
+	"sync/atomic"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type FileHandleId uint64
+
+var IsDebugFileReadWrite = false
+
+type FileHandle struct {
+	fh              FileHandleId
+	counter         int64
+	entry           *LockedEntry
+	entryLock       sync.RWMutex
+	entryChunkGroup *filer.ChunkGroup
+	inode           uint64
+	wfs             *WFS
+
+	// cache file has been written to
+	dirtyMetadata     bool
+	dirtyPages        *PageWriter
+	reader            *filer.ChunkReadAt
+	contentType       string
+	asyncFlushPending bool   // set in writebackCache mode to defer flush to Release
+	asyncFlushUid     uint32 // saved uid for deferred metadata flush
+	asyncFlushGid     uint32 // saved gid for deferred metadata flush
+	savedDir          string // last known parent path if inode-to-path state is forgotten
+	savedName         string // last known file name if inode-to-path state is forgotten
+
+	isDeleted bool
+	// deleteEpoch counts the times isDeleted was raised, all of them under the
+	// handle's flush lock. A caller that raised it and then found it had
+	// nothing to delete after all can tell its own mark from a later one.
+	deleteEpoch uint64
+	isRenamed   bool // set by Rename before waiting for async flush; skips old-path metadata flush
+
+	// entryVersionTsNs is the filer log position the handle's entry reflects.
+	// State at or below it must not replace the entry — that rolls it back.
+	entryVersionTsNs atomic.Int64
+
+	// entryVersionSignature identifies the filer whose clock stamped
+	// entryVersionTsNs, when it came from an RPC fence. Positions from a
+	// different filer are not comparable, so an event that filer did not log
+	// is applied rather than fenced out. Zero when the version came from an
+	// event, whose ordering the subscription already provides.
+	entryVersionSignature atomic.Int32
+
+	// baseEntry snapshots the filer state last installed or acknowledged.
+	// Local writes move the live entry away from it, so "is this event new"
+	// must be judged here, not against the live entry. Always store a clone.
+	baseEntry atomic.Pointer[filer_pb.Entry]
+
+	// dlmLock holds the distributed lock for cross-mount write coordination.
+	// Non-nil only when -dlm is enabled and the file was opened for writing.
+	// Acquired in AcquireHandle, released in ReleaseHandle.
+	dlmLock *cluster.LiveLock
+
+	// remoteInstallMu serializes downloadRemoteEntry's install, which holds
+	// only the handle's shared lock and so races a second concurrent read.
+	remoteInstallMu sync.Mutex
+
+	// for debugging
+	mirrorFile *os.File
+}
+
+func newFileHandle(wfs *WFS, handleId FileHandleId, inode uint64, entry *filer_pb.Entry) *FileHandle {
+	fh := &FileHandle{
+		fh:      handleId,
+		counter: 1,
+		inode:   inode,
+		wfs:     wfs,
+	}
+	// dirtyPages: newContinuousDirtyPages(file, writeOnly),
+	fh.dirtyPages = newPageWriter(fh, wfs.option.ChunkSizeLimit)
+	fh.entry = &LockedEntry{
+		Entry: entry,
+	}
+	if entry != nil {
+		fh.SetEntry(entry)
+		fh.baseEntry.Store(proto.Clone(entry).(*filer_pb.Entry))
+	}
+
+	if IsDebugFileReadWrite {
+		var err error
+		fh.mirrorFile, err = os.OpenFile("/tmp/sw/"+entry.Name, os.O_RDWR|os.O_CREATE, 0600)
+		if err != nil {
+			println("failed to create mirror:", err.Error())
+		}
+	}
+
+	return fh
+}
+
+func (fh *FileHandle) FullPath() util.FullPath {
+	if fp, status := fh.wfs.inodeToPath.GetPath(fh.inode); status == fuse.OK {
+		return fp
+	}
+	if fh.savedName != "" {
+		return util.FullPath(fh.savedDir).Child(fh.savedName)
+	}
+	return ""
+}
+
+func (fh *FileHandle) RememberPath(fullPath util.FullPath) {
+	if fullPath == "" {
+		return
+	}
+	fh.savedDir, fh.savedName = fullPath.DirAndName()
+}
+
+func (fh *FileHandle) GetEntry() *LockedEntry {
+	return fh.entry
+}
+
+func (fh *FileHandle) SetEntry(entry *filer_pb.Entry) {
+	if entry != nil {
+		fileSize := filer.FileSize(entry)
+		entry.Attributes.FileSize = fileSize
+		if fh.entryChunkGroup != nil {
+			_ = fh.entryChunkGroup.Close()
+		}
+		var resolveManifestErr error
+		fh.entryChunkGroup, resolveManifestErr = filer.NewChunkGroup(fh.wfs.LookupFn(), fh.wfs.chunkCache, entry.Chunks, fh.wfs.option.ConcurrentReaders, fh.wfs.CacheInvalidator(), fh.wfs.manifestCache, fh.wfs.readerCacheBudget)
+		if resolveManifestErr != nil {
+			glog.Warningf("failed to resolve manifest chunks in %+v", entry)
+		}
+	} else {
+		glog.Fatalf("setting file handle entry to nil")
+	}
+	fh.entry.SetEntry(entry)
+}
+
+// installAckedEntry installs filer-acknowledged state under the handle lock
+// when it outranks the handle. A version never advances without its value:
+// stamping alone would fence out the events carrying what the handle lacks.
+// Dirty handles are skipped — local writes supersede the ack.
+func (fh *FileHandle) installAckedEntry(entry *filer_pb.Entry, versionTsNs int64, signature int32) {
+	fhActiveLock := fh.wfs.fhLockTable.AcquireLock("installAckedEntry", fh.fh, util.ExclusiveLock)
+	defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
+	if versionTsNs == 0 || fh.dirtyMetadata || entry == fh.GetEntry().GetEntry() {
+		return
+	}
+	// Refuse only what is provably older. Two known, differing filer
+	// signatures mean the positions come from unrelated clocks and say
+	// nothing about each other; dropping the acknowledgment there would leave
+	// the handle holding the very state this mutation replaced. Unknown
+	// signatures still compare, as they did before.
+	handleSignature := fh.entryVersionSignature.Load()
+	provablyOtherClock := signature != 0 && handleSignature != 0 && signature != handleSignature
+	if !provablyOtherClock && versionTsNs <= fh.entryVersionTsNs.Load() {
+		return
+	}
+	fh.SetEntry(entry)
+	fh.setAuthoritativeBase(proto.Clone(entry).(*filer_pb.Entry))
+	fh.advanceEntryVersion(versionTsNs, signature)
+}
+
+// setAuthoritativeBase installs the base snapshot a local ack acknowledged.
+func (fh *FileHandle) setAuthoritativeBase(base *filer_pb.Entry) {
+	fh.baseEntry.Store(base)
+}
+
+// advanceEntryVersion raises the entry version, never regresses it, and
+// records the clock domain the new position belongs to: a filer signature for
+// an RPC fence, zero for an event. The signature travels with the timestamp so
+// the two never disagree. A zero position (an unversioned old filer) is a
+// no-op, leaving the handle open to refreshes.
+func (fh *FileHandle) advanceEntryVersion(tsNs int64, signature int32) {
+	if tsNs == 0 {
+		return
+	}
+	for {
+		current := fh.entryVersionTsNs.Load()
+		if tsNs <= current {
+			return
+		}
+		if fh.entryVersionTsNs.CompareAndSwap(current, tsNs) {
+			fh.entryVersionSignature.Store(signature)
+			return
+		}
+	}
+}
+
+func (fh *FileHandle) ResetDirtyPages() {
+	fh.dirtyPages.Destroy()
+	fh.dirtyPages = newPageWriter(fh, fh.wfs.option.ChunkSizeLimit)
+	fh.dirtyMetadata = false
+	fh.contentType = ""
+}
+
+func (fh *FileHandle) UpdateEntry(fn func(entry *filer_pb.Entry)) *filer_pb.Entry {
+	result := fh.entry.UpdateEntry(fn)
+
+	return result
+}
+
+func (fh *FileHandle) AddChunks(chunks []*filer_pb.FileChunk) {
+	fh.entry.AppendChunks(chunks)
+}
+
+func (fh *FileHandle) ReleaseHandle() {
+	// Release distributed lock before cleaning up, so other mounts can
+	// proceed as soon as this handle is done flushing.
+	if fh.dlmLock != nil {
+		fh.dlmLock.Stop()
+		fh.dlmLock = nil
+		glog.V(1).Infof("DLM lock released for inode %d", fh.inode)
+	}
+
+	fhActiveLock := fh.wfs.fhLockTable.AcquireLock("ReleaseHandle", fh.fh, util.ExclusiveLock)
+	defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
+
+	if fh.entryChunkGroup != nil {
+		_ = fh.entryChunkGroup.Close()
+	}
+	fh.dirtyPages.Destroy()
+	if IsDebugFileReadWrite {
+		fh.mirrorFile.Close()
+	}
+}

weed/mount/filehandle_map.go

@@ -0,0 +1,116 @@
+package mount
+
+import (
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+type FileHandleToInode struct {
+	sync.RWMutex
+	inode2fh map[uint64]*FileHandle
+	fh2inode map[FileHandleId]uint64
+}
+
+func NewFileHandleToInode() *FileHandleToInode {
+	return &FileHandleToInode{
+		inode2fh: make(map[uint64]*FileHandle),
+		fh2inode: make(map[FileHandleId]uint64),
+	}
+}
+
+func (i *FileHandleToInode) GetFileHandle(fh FileHandleId) *FileHandle {
+	i.RLock()
+	defer i.RUnlock()
+	inode, found := i.fh2inode[fh]
+	if found {
+		return i.inode2fh[inode]
+	}
+	return nil
+}
+
+func (i *FileHandleToInode) FindFileHandle(inode uint64) (fh *FileHandle, found bool) {
+	i.RLock()
+	defer i.RUnlock()
+	fh, found = i.inode2fh[inode]
+	return
+}
+
+// MarkInodeRenamed sets isRenamed on any file handle associated with the
+// given inode.  This prevents the async flush from recreating a renamed
+// file's metadata under its old path.
+func (i *FileHandleToInode) MarkInodeRenamed(inode uint64) {
+	i.RLock()
+	defer i.RUnlock()
+	if fh, ok := i.inode2fh[inode]; ok {
+		fh.isRenamed = true
+	}
+}
+
+// AcquireFileHandle fully initializes a handle — entry and the log position it
+// reflects — before exposing it in the map, and reports whether one already
+// existed. An existing handle only gets its counter bumped: its entry belongs
+// to whoever holds the handle lock, so callers install there (AcquireHandle)
+// rather than under this one.
+func (i *FileHandleToInode) AcquireFileHandle(wfs *WFS, inode uint64, entry *filer_pb.Entry, versionTsNs int64, signature int32) (*FileHandle, bool) {
+	i.Lock()
+	defer i.Unlock()
+	fh, found := i.inode2fh[inode]
+	if !found {
+		fh = newFileHandle(wfs, FileHandleId(util.RandomUint64()), inode, entry)
+		fh.entryVersionTsNs.Store(versionTsNs)
+		fh.entryVersionSignature.Store(signature)
+		i.inode2fh[inode] = fh
+		i.fh2inode[fh.fh] = inode
+		return fh, false
+	}
+	fh.counter++
+	return fh, true
+}
+
+func (i *FileHandleToInode) ReleaseByHandle(fh FileHandleId) *FileHandle {
+	i.Lock()
+	defer i.Unlock()
+
+	inode, found := i.fh2inode[fh]
+	if !found {
+		return nil
+	}
+
+	fhHandle, fhFound := i.inode2fh[inode]
+	if !fhFound {
+		delete(i.fh2inode, fh)
+		return nil
+	}
+
+	// If the counter is already <= 0, a prior Release already started the
+	// drain.  Return nil to prevent double-processing.
+	if fhHandle.counter <= 0 {
+		return nil
+	}
+
+	fhHandle.counter--
+	if fhHandle.counter <= 0 {
+		if fhHandle.asyncFlushPending {
+			// Handle stays in fhMap so rename/unlink can still find it
+			// via FindFileHandle during the background drain.
+			return fhHandle
+		}
+		delete(i.inode2fh, inode)
+		delete(i.fh2inode, fhHandle.fh)
+		return fhHandle
+	}
+	return nil
+}
+
+// RemoveFileHandle removes a handle from both maps.  Called after an async
+// drain completes to clean up the handle that was intentionally kept in the
+// maps during the flush.
+func (i *FileHandleToInode) RemoveFileHandle(fh FileHandleId, inode uint64) {
+	i.Lock()
+	defer i.Unlock()
+	delete(i.inode2fh, inode)
+	delete(i.fh2inode, fh)
+}

weed/mount/filehandle_read.go

@@ -0,0 +1,183 @@
+package mount
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func (fh *FileHandle) lockForRead(startOffset int64, size int) {
+	fh.dirtyPages.LockForRead(startOffset, startOffset+int64(size))
+}
+func (fh *FileHandle) unlockForRead(startOffset int64, size int) {
+	fh.dirtyPages.UnlockForRead(startOffset, startOffset+int64(size))
+}
+
+func (fh *FileHandle) readFromDirtyPages(buff []byte, startOffset int64, tsNs int64) (maxStop int64) {
+	maxStop = fh.dirtyPages.ReadDirtyDataAt(buff, startOffset, tsNs)
+	return
+}
+
+func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, int64, error) {
+	return fh.readFromChunksWithContext(context.Background(), buff, offset)
+}
+
+func (fh *FileHandle) readFromChunksWithContext(ctx context.Context, buff []byte, offset int64) (int64, int64, error) {
+	fh.entryLock.RLock()
+	defer fh.entryLock.RUnlock()
+
+	fileFullPath := fh.FullPath()
+
+	entry := fh.GetEntry()
+
+	// IsInRemoteOnly inspects entry.Chunks, so take the LockedEntry lock the
+	// async uploader appends under.
+	entry.RLock()
+	remoteOnly := entry.Entry.IsInRemoteOnly()
+	entry.RUnlock()
+	if remoteOnly {
+		glog.V(4).Infof("download remote entry %s", fileFullPath)
+		err := fh.downloadRemoteEntry(entry)
+		if err != nil {
+			glog.V(1).Infof("download remote entry %s: %v", fileFullPath, err)
+			return 0, 0, err
+		}
+	}
+
+	// Snapshot size, inline content, and the chunk list under the LockedEntry
+	// lock. Async upload workers append chunks under this lock (AddChunks), so
+	// reading entry.Chunks / FileSize without it races with the slice
+	// reallocation and can crash in filer.TotalSize. The captured slice headers
+	// stay valid afterwards: append never mutates the old backing array, and
+	// truncate is excluded by the fh.entryLock held for this whole read.
+	entry.RLock()
+	pbEntry := entry.Entry
+	fileSize := int64(pbEntry.Attributes.FileSize)
+	if fileSize == 0 {
+		fileSize = int64(filer.FileSize(pbEntry))
+	}
+	content := pbEntry.Content
+	chunks := pbEntry.Chunks
+	entry.RUnlock()
+
+	if fileSize == 0 {
+		glog.V(1).Infof("empty fh %v", fileFullPath)
+		return 0, 0, io.EOF
+	} else if offset == fileSize {
+		return 0, 0, io.EOF
+	} else if offset >= fileSize {
+		glog.V(1).Infof("invalid read, fileSize %d, offset %d for %s", fileSize, offset, fileFullPath)
+		return 0, 0, io.EOF
+	}
+
+	if offset < int64(len(content)) {
+		totalRead := copy(buff, content[offset:])
+		glog.V(4).Infof("file handle read cached %s [%d,%d] %d", fileFullPath, offset, offset+int64(totalRead), totalRead)
+		return int64(totalRead), 0, nil
+	}
+
+	// Peer chunk sharing: try a peer mount's cache before the volume tier.
+	// Any failure falls through transparently. See design-weed-mount-
+	// peer-chunk-sharing.md §4.3.
+	if fh.wfs.option.PeerEnabled && fh.wfs.peerGrpcServer != nil {
+		totalRead, ts, err := fh.tryPeerRead(ctx, fileSize, buff, offset, chunks)
+		if err == nil {
+			glog.V(4).Infof("peer read successful for %s [%d,%d] %d", fileFullPath, offset, offset+int64(totalRead), totalRead)
+			return int64(totalRead), ts, nil
+		}
+		// Skip the "failed" log for benign skip reasons (local cache
+		// hit, no peer owner yet, etc.) — the cache/volume fallback is
+		// the expected outcome, not a failure.
+		if err != errPeerReadSkipped {
+			glog.V(4).Infof("peer read failed for %s, falling back to volume: %v", fileFullPath, err)
+		}
+	}
+
+	// Fall back to normal chunk reading
+	totalRead, ts, err := fh.entryChunkGroup.ReadDataAt(ctx, fileSize, buff, offset)
+
+	if err != nil && err != io.EOF {
+		glog.Errorf("file handle read %s: %v", fileFullPath, err)
+	}
+
+	// glog.V(4).Infof("file handle read %s [%d,%d] %d : %v", fileFullPath, offset, offset+int64(totalRead), totalRead, err)
+
+	return int64(totalRead), ts, err
+}
+
+func (fh *FileHandle) downloadRemoteEntry(entry *LockedEntry) error {
+
+	fileFullPath := fh.FullPath()
+	dir, _ := fileFullPath.DirAndName()
+
+	err := fh.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.CacheRemoteObjectToLocalClusterRequest{
+			Directory: string(dir),
+			Name:      entry.Name,
+		}
+
+		glog.V(4).Infof("download entry: %v", request)
+		resp, err := client.CacheRemoteObjectToLocalCluster(context.Background(), request)
+		if err != nil {
+			return fmt.Errorf("CacheRemoteObjectToLocalCluster file %s: %v", fileFullPath, err)
+		}
+
+		// Entry and base are kept in local uid/gid form like every other
+		// install path; the response carries filer ids. Without mapping, an
+		// unchanged re-delivery compares unequal and destroys dirty pages.
+		localEntry := proto.Clone(resp.Entry).(*filer_pb.Entry)
+		if localEntry.Attributes != nil && fh.wfs.option.UidGidMapper != nil {
+			localEntry.Attributes.Uid, localEntry.Attributes.Gid = fh.wfs.option.UidGidMapper.FilerToLocal(localEntry.Attributes.Uid, localEntry.Attributes.Gid)
+		}
+
+		versionTsNs := ackVersionTsNs(resp)
+
+		// Two concurrent reads can both be here (the handle lock is shared;
+		// invalidation is excluded by the exclusive one). Install as one unit,
+		// and only if at least as new — an older response landing last would
+		// keep the newer version over an older entry, fencing out corrections.
+		installed := false
+		fh.remoteInstallMu.Lock()
+		switch {
+		case versionTsNs >= fh.entryVersionTsNs.Load():
+			fh.SetEntry(localEntry)
+			fh.setAuthoritativeBase(proto.Clone(localEntry).(*filer_pb.Entry))
+			fh.advanceEntryVersion(versionTsNs, resp.GetLogSignature())
+			installed = true
+		case versionTsNs == 0 && fh.GetEntry().GetEntry().IsInRemoteOnly():
+			// Unversioned response (pre-upgrade filer) and the handle still has
+			// no local chunks to read: take the content, but claim no position.
+			// A response that is merely *older* is refused instead — its content
+			// predates what the handle already reflects.
+			fh.SetEntry(localEntry)
+			fh.setAuthoritativeBase(proto.Clone(localEntry).(*filer_pb.Entry))
+		}
+		fh.remoteInstallMu.Unlock()
+
+		// Only publish state we accepted, and only when it carries a position
+		// to order it by: an unversioned event would clear the cache entry's
+		// version and let an older subscriber event roll the cache back.
+		// Async: a sync apply deadlocks against the apply loop's invalidate, which needs this read's file-handle lock.
+		if installed && versionTsNs != 0 {
+			event := resp.GetMetadataEvent()
+			if event == nil {
+				event = metadataUpdateEvent(request.Directory, resp.Entry)
+				if event != nil {
+					event.TsNs = versionTsNs
+				}
+			}
+			fh.wfs.applyLocalMetadataEventAsync(event)
+		}
+
+		return nil
+	})
+
+	return err
+}

weed/mount/filehandle_read_remote_test.go

@@ -0,0 +1,150 @@
+package mount
+
+import (
+	"context"
+	"net"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// cacheRemoteTestServer broadcasts the matching invalidate event before
+// returning the cached entry, so the apply loop is busy invalidating the same
+// file handle the read still holds when downloadRemoteEntry applies its event.
+type cacheRemoteTestServer struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+	wfs               *WFS
+	dir, name         string
+	content           []byte
+	invalidateStarted chan struct{}
+}
+
+func (s *cacheRemoteTestServer) CacheRemoteObjectToLocalCluster(ctx context.Context, req *filer_pb.CacheRemoteObjectToLocalClusterRequest) (*filer_pb.CacheRemoteObjectToLocalClusterResponse, error) {
+	cached := &filer_pb.Entry{
+		Name:       s.name,
+		Attributes: &filer_pb.FuseAttributes{FileSize: uint64(len(s.content))},
+		Content:    s.content,
+	}
+	go s.wfs.metaCache.ApplyMetadataResponse(context.Background(), metadataUpdateEvent(s.dir, cached), meta_cache.SubscriberMetadataResponseApplyOptions)
+	<-s.invalidateStarted
+	return &filer_pb.CacheRemoteObjectToLocalClusterResponse{
+		Entry:         cached,
+		MetadataEvent: metadataUpdateEvent(s.dir, cached),
+	}, nil
+}
+
+// TestReadUncachedRemoteEntryDoesNotDeadlock guards the read of an uncached
+// remote file against the apply-loop invalidate that needs its file-handle lock.
+func TestReadUncachedRemoteEntryDoesNotDeadlock(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	t.Cleanup(func() { _ = listener.Close() })
+
+	server := pb.NewGrpcServer()
+	testServer := &cacheRemoteTestServer{
+		dir:               "/dir",
+		name:              "file",
+		content:           []byte("hello remote world"),
+		invalidateStarted: make(chan struct{}, 1),
+	}
+	filer_pb.RegisterSeaweedFilerServer(server, testServer)
+	go server.Serve(listener)
+	t.Cleanup(server.Stop)
+
+	uidGidMapper, err := meta_cache.NewUidGidMapper("", "")
+	if err != nil {
+		t.Fatalf("create uid/gid mapper: %v", err)
+	}
+
+	root := util.FullPath("/")
+	wfs := &WFS{
+		signature:         1,
+		inodeToPath:       NewInodeToPath(root, 0),
+		fhMap:             NewFileHandleToInode(),
+		fhLockTable:       util.NewLockTable[FileHandleId](),
+		hardLinkLockTable: util.NewLockTable[string](),
+		option: &Option{
+			ChunkSizeLimit:     1024,
+			ConcurrentReaders:  1,
+			VolumeServerAccess: "filerProxy",
+			FilerAddresses: []pb.ServerAddress{
+				pb.NewServerAddressWithGrpcPort("127.0.0.1:1", listener.Addr().(*net.TCPAddr).Port),
+			},
+			GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+		},
+	}
+	testServer.wfs = wfs
+
+	wfs.metaCache = meta_cache.NewMetaCache(
+		filepath.Join(t.TempDir(), "meta"),
+		uidGidMapper,
+		root,
+		false,
+		func(path util.FullPath) { wfs.inodeToPath.MarkChildrenCached(path) },
+		func(path util.FullPath) bool { return wfs.inodeToPath.IsChildrenCached(path) },
+		// Mirror weedfs.go's invalidateFunc: take the file handle exclusive lock.
+		func(inv meta_cache.EntryInvalidation) {
+			path := inv.Path
+			inode, ok := wfs.inodeToPath.GetInode(path)
+			if !ok {
+				return
+			}
+			fh, ok := wfs.fhMap.FindFileHandle(inode)
+			if !ok {
+				return
+			}
+			select {
+			case testServer.invalidateStarted <- struct{}{}:
+			default:
+			}
+			lock := wfs.fhLockTable.AcquireLock("invalidateFunc", fh.fh, util.ExclusiveLock)
+			wfs.fhLockTable.ReleaseLock(fh.fh, lock)
+		},
+		nil,
+	)
+	wfs.inodeToPath.MarkChildrenCached(root)
+	wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false)
+	wfs.inodeToPath.MarkChildrenCached(util.FullPath("/dir"))
+	t.Cleanup(func() { wfs.metaCache.Shutdown() })
+
+	inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false)
+	fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{
+		Name:        "file",
+		Attributes:  &filer_pb.FuseAttributes{FileSize: uint64(len(testServer.content))},
+		RemoteEntry: &filer_pb.RemoteEntry{RemoteSize: int64(len(testServer.content))},
+	}, 0, 0)
+
+	buff := make([]byte, len(testServer.content))
+	done := make(chan fuse.Status, 1)
+	go func() {
+		_, status := wfs.Read(make(chan struct{}), &fuse.ReadIn{
+			InHeader: fuse.InHeader{NodeId: inode},
+			Fh:       uint64(fh.fh),
+			Offset:   0,
+			Size:     uint32(len(buff)),
+		}, buff)
+		done <- status
+	}()
+
+	select {
+	case status := <-done:
+		if status != fuse.OK {
+			t.Fatalf("Read status = %v, want OK", status)
+		}
+		if string(buff) != string(testServer.content) {
+			t.Fatalf("Read content = %q, want %q", buff, testServer.content)
+		}
+	case <-time.After(15 * time.Second):
+		t.Fatal("Read of an uncached remote entry deadlocked")
+	}
+}

weed/mount/filehandle_test.go

@@ -0,0 +1,52 @@
+package mount
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestFileHandleFullPathFallsBackAfterForget(t *testing.T) {
+	wfs := &WFS{
+		inodeToPath: NewInodeToPath(util.FullPath("/"), 0),
+	}
+
+	fullPath := util.FullPath("/worker_0/subdir_0/test.txt")
+	inode := wfs.inodeToPath.Lookup(fullPath, 1, false, false, 0, true)
+
+	fh := &FileHandle{
+		inode: inode,
+		wfs:   wfs,
+	}
+	fh.RememberPath(fullPath)
+
+	wfs.inodeToPath.Forget(inode, 1, nil, nil)
+
+	if got := fh.FullPath(); got != fullPath {
+		t.Fatalf("FullPath() after forget = %q, want %q", got, fullPath)
+	}
+}
+
+func TestFileHandleFullPathUsesSavedRenamePathAfterForget(t *testing.T) {
+	wfs := &WFS{
+		inodeToPath: NewInodeToPath(util.FullPath("/"), 0),
+	}
+
+	oldPath := util.FullPath("/worker_0/subdir_0/test.txt")
+	newPath := util.FullPath("/worker_0/subdir_1/test.txt")
+	inode := wfs.inodeToPath.Lookup(oldPath, 1, false, false, 0, true)
+
+	fh := &FileHandle{
+		inode: inode,
+		wfs:   wfs,
+	}
+	fh.RememberPath(oldPath)
+
+	wfs.inodeToPath.MovePath(oldPath, newPath)
+	fh.RememberPath(newPath)
+	wfs.inodeToPath.Forget(inode, 1, nil, nil)
+
+	if got := fh.FullPath(); got != newPath {
+		t.Fatalf("FullPath() after rename+forget = %q, want %q", got, newPath)
+	}
+}

weed/mount/filer_conf.go

@@ -0,0 +1,107 @@
+package mount
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func (wfs *WFS) subscribeFilerConfEvents() (*meta_cache.MetadataFollower, error) {
+	confDir := filer.DirectoryEtcSeaweedFS
+	confName := filer.FilerConfName
+	confFullName := filepath.Join(filer.DirectoryEtcSeaweedFS, filer.FilerConfName)
+
+	// read current conf
+	err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		content, err := filer.ReadInsideFiler(context.Background(), client, confDir, confName)
+		if err != nil {
+			return err
+		}
+
+		fc := filer.NewFilerConf()
+		if len(content) > 0 {
+			if err := fc.LoadFromBytes(content); err != nil {
+				return fmt.Errorf("parse %s: %v", confFullName, err)
+			}
+		}
+
+		wfs.FilerConf = fc
+
+		return nil
+	})
+	if err != nil {
+		if errors.Is(err, filer_pb.ErrNotFound) {
+			glog.V(0).Infof("fuse filer conf %s not found", confFullName)
+		} else {
+			return nil, err
+		}
+	}
+
+	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
+		if !isFilerConfUpdateEvent(resp, confDir, confName) {
+			return nil
+		}
+
+		content := resp.EventNotification.NewEntry.Content
+		fc := filer.NewFilerConf()
+		if len(content) > 0 {
+			if err = fc.LoadFromBytes(content); err != nil {
+				return fmt.Errorf("parse %s: %v", confFullName, err)
+			}
+		}
+
+		wfs.FilerConf = fc
+
+		return nil
+	}
+	return &meta_cache.MetadataFollower{
+		PathPrefixToWatch: confFullName,
+		ProcessEventFn:    processEventFn,
+	}, nil
+}
+
+func isFilerConfUpdateEvent(resp *filer_pb.SubscribeMetadataResponse, confDir, confName string) bool {
+	if resp == nil || resp.EventNotification == nil || resp.EventNotification.NewEntry == nil {
+		return false
+	}
+	return filer_pb.MetadataEventTargetDirectory(resp) == confDir &&
+		resp.EventNotification.NewEntry.Name == confName
+}
+
+func (wfs *WFS) wormEnforcedForEntry(path util.FullPath, entry *filer_pb.Entry) (wormEnforced, wormEnabled bool) {
+	if entry == nil || wfs.FilerConf == nil {
+		return false, false
+	}
+
+	rule := wfs.FilerConf.MatchStorageRule(string(path))
+	if !rule.GetWorm() {
+		return false, false
+	}
+
+	// worm is not enforced
+	if entry.WormEnforcedAtTsNs == 0 {
+		return false, true
+	}
+
+	// worm will never expire
+	if rule.WormRetentionTimeSeconds == 0 {
+		return true, true
+	}
+
+	enforcedAt := time.Unix(0, entry.WormEnforcedAtTsNs)
+
+	// worm is expired
+	if time.Now().Sub(enforcedAt).Seconds() >= float64(rule.WormRetentionTimeSeconds) {
+		return false, true
+	}
+
+	return true, true
+}

weed/mount/filer_conf_test.go

@@ -0,0 +1,23 @@
+package mount
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func TestIsFilerConfUpdateEventMatchesRenameTarget(t *testing.T) {
+	event := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/tmp",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: filer.FilerConfName},
+			NewEntry:      &filer_pb.Entry{Name: filer.FilerConfName},
+			NewParentPath: filer.DirectoryEtcSeaweedFS,
+		},
+	}
+
+	if !isFilerConfUpdateEvent(event, filer.DirectoryEtcSeaweedFS, filer.FilerConfName) {
+		t.Fatalf("expected rename target to match filer.conf watcher")
+	}
+}

weed/mount/inode_to_path.go

@@ -0,0 +1,627 @@
+package mount
+
+import (
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type InodeToPath struct {
+	sync.RWMutex
+	nextInodeId     uint64
+	cacheMetaTtlSec time.Duration
+	inode2path      map[uint64]*InodeEntry
+	path2inode      map[util.FullPath]uint64
+	// dirStates holds directory-only readdir-cache state, keyed by inode. An
+	// inode is a directory iff it has an entry here, registered at creation.
+	dirStates map[uint64]*dirState
+	// dirPaths indexes the same states by path, so a directory lookup never
+	// goes through the map that holds one full path per inode.
+	dirPaths map[util.FullPath]*dirState
+}
+
+// InodeEntry exists per inode the kernel references. Directory cache state is
+// kept out in dirStates so a file entry stays in the 32-byte size class — the
+// dominant cost on a mount with millions of files. A hard link's extra paths
+// hang off a pointer for the same reason.
+type InodeEntry struct {
+	path       util.FullPath
+	nlookup    uint64
+	extraPaths *[]util.FullPath
+}
+
+type dirState struct {
+	path               util.FullPath
+	isChildrenCached   bool
+	readDirDirect      bool
+	cachedExpiresTime  time.Time
+	lastAccess         time.Time
+	lastRefresh        time.Time
+	lastRebuildAttempt time.Time
+	subdirCount        int32 // tracked in-memory for POSIX directory nlink
+}
+
+func (d *dirState) resetCacheState() {
+	d.isChildrenCached = false
+	d.readDirDirect = false
+	d.cachedExpiresTime = time.Time{}
+	d.lastRebuildAttempt = time.Time{}
+}
+
+// appendPaths appends every path the inode is reachable by, primary first.
+func (ie *InodeEntry) appendPaths(dst []util.FullPath) []util.FullPath {
+	if ie.path == "" {
+		return dst
+	}
+	dst = append(dst, ie.path)
+	if ie.extraPaths != nil {
+		dst = append(dst, *ie.extraPaths...)
+	}
+	return dst
+}
+
+func (ie *InodeEntry) addPath(p util.FullPath) {
+	if ie.path == "" {
+		ie.path = p
+		return
+	}
+	if ie.extraPaths == nil {
+		ie.extraPaths = &[]util.FullPath{p}
+		return
+	}
+	*ie.extraPaths = append(*ie.extraPaths, p)
+}
+
+func (ie *InodeEntry) replacePath(from, to util.FullPath) {
+	if ie.path == from {
+		ie.path = to
+	}
+	if ie.extraPaths == nil {
+		return
+	}
+	for i, p := range *ie.extraPaths {
+		if p == from {
+			(*ie.extraPaths)[i] = to
+		}
+	}
+}
+
+func (ie *InodeEntry) setExtraPaths(extra []util.FullPath) {
+	if len(extra) == 0 {
+		ie.extraPaths = nil
+		return
+	}
+	ie.extraPaths = &extra
+}
+
+// removeOnePath promotes an extra path when the primary is the one going away,
+// so an entry that still has a path always has a primary one.
+func (ie *InodeEntry) removeOnePath(p util.FullPath) bool {
+	if ie.path == "" {
+		return false
+	}
+	if ie.path == p {
+		if ie.extraPaths == nil {
+			ie.path = ""
+			return true
+		}
+		extra := *ie.extraPaths
+		ie.path = extra[0]
+		ie.setExtraPaths(extra[1:])
+		return true
+	}
+	if ie.extraPaths == nil {
+		return false
+	}
+	extra := *ie.extraPaths
+	for i, x := range extra {
+		if x != p {
+			continue
+		}
+		ie.setExtraPaths(append(extra[:i], extra[i+1:]...))
+		return true
+	}
+	return false
+}
+
+func NewInodeToPath(root util.FullPath, ttlSec int) *InodeToPath {
+	t := &InodeToPath{
+		inode2path:      make(map[uint64]*InodeEntry),
+		path2inode:      make(map[util.FullPath]uint64),
+		dirStates:       make(map[uint64]*dirState),
+		dirPaths:        make(map[util.FullPath]*dirState),
+		cacheMetaTtlSec: time.Second * time.Duration(ttlSec),
+	}
+	t.inode2path[1] = &InodeEntry{
+		path:    root,
+		nlookup: 1,
+	}
+	t.setDirState(1, &dirState{path: root, lastAccess: time.Now()})
+	t.path2inode[root] = 1
+
+	return t
+}
+
+// EnsurePath make sure the full path is tracked, used by symlink.
+func (i *InodeToPath) EnsurePath(path util.FullPath, isDirectory bool) bool {
+	dir, _ := path.DirAndName()
+	if dir == "/" {
+		return true
+	}
+	if i.EnsurePath(util.FullPath(dir), true) {
+		i.Lookup(path, time.Now().Unix(), isDirectory, false, 0, false)
+		return true
+	}
+	return false
+}
+
+func (i *InodeToPath) Lookup(path util.FullPath, unixTime int64, isDirectory bool, isHardlink bool, possibleInode uint64, isLookup bool) uint64 {
+	i.Lock()
+	defer i.Unlock()
+	inode, found := i.path2inode[path]
+	if !found {
+		if possibleInode == 0 {
+			inode = path.AsInode(unixTime)
+		} else {
+			inode = possibleInode
+		}
+		if !isHardlink {
+			for _, found := i.inode2path[inode]; found; inode++ {
+				_, found = i.inode2path[inode+1]
+			}
+		}
+	}
+	i.path2inode[path] = inode
+
+	if _, found := i.inode2path[inode]; found {
+		if isLookup {
+			i.inode2path[inode].nlookup++
+		}
+	} else {
+		nlookup := uint64(0)
+		if isLookup {
+			nlookup = 1
+		}
+		i.inode2path[inode] = &InodeEntry{
+			path:    path,
+			nlookup: nlookup,
+		}
+		if isDirectory {
+			i.setDirState(inode, &dirState{path: path})
+		}
+	}
+
+	return inode
+}
+
+// IncrementNlookup takes one more reference on an inode already in the table,
+// reporting false if it is not there.
+func (i *InodeToPath) IncrementNlookup(inode uint64) bool {
+	i.Lock()
+	defer i.Unlock()
+	entry, found := i.inode2path[inode]
+	if !found {
+		return false
+	}
+	entry.nlookup++
+	return true
+}
+
+// InodeForListing returns the inode number a readdir should report for path
+// without entering it in the table. Nothing is reserved, so the collision probe
+// Lookup does is skipped: the worst case is a repeated st_ino in one listing.
+func (i *InodeToPath) InodeForListing(path util.FullPath, unixTime int64, possibleInode uint64) uint64 {
+	i.RLock()
+	inode, found := i.path2inode[path]
+	i.RUnlock()
+	if found {
+		return inode
+	}
+	if possibleInode != 0 {
+		return possibleInode
+	}
+	return path.AsInode(unixTime)
+}
+
+func (i *InodeToPath) AllocateInode(path util.FullPath, unixTime int64) uint64 {
+	if path == "/" {
+		return 1
+	}
+	i.Lock()
+	defer i.Unlock()
+	inode := path.AsInode(unixTime)
+	for _, found := i.inode2path[inode]; found; inode++ {
+		_, found = i.inode2path[inode]
+	}
+	return inode
+}
+
+func (i *InodeToPath) GetInode(path util.FullPath) (uint64, bool) {
+	if path == "/" {
+		return 1, true
+	}
+	i.Lock()
+	defer i.Unlock()
+	inode, found := i.path2inode[path]
+	if !found {
+		// glog.Fatalf("GetInode unknown inode for %s", path)
+		// this could be the parent for mount point
+	}
+	return inode, found
+}
+
+func (i *InodeToPath) GetPath(inode uint64) (util.FullPath, fuse.Status) {
+	i.RLock()
+	defer i.RUnlock()
+	path, found := i.inode2path[inode]
+	if !found || path.path == "" {
+		return "", fuse.ENOENT
+	}
+	return path.path, fuse.OK
+}
+
+// GetAllPaths returns a copy of all paths associated with an inode. For a
+// hard-linked file, this includes every link that the mount currently knows
+// about. Returns nil if the inode is unknown.
+func (i *InodeToPath) GetAllPaths(inode uint64) []util.FullPath {
+	i.RLock()
+	defer i.RUnlock()
+	ie, found := i.inode2path[inode]
+	if !found {
+		return nil
+	}
+	return ie.appendPaths(nil)
+}
+
+func (i *InodeToPath) setDirState(inode uint64, d *dirState) {
+	i.dirStates[inode] = d
+	i.dirPaths[d.path] = d
+}
+
+// dropDirPath drops the path index only; Forget releases the state itself. A
+// released directory's state keeps the path it had, so drop the index only
+// while it is still the one that path resolves to: a new directory may have
+// taken the name in the meantime.
+func (i *InodeToPath) dropDirPath(inode uint64) {
+	if d := i.dirStates[inode]; d != nil && i.dirPaths[d.path] == d {
+		delete(i.dirPaths, d.path)
+	}
+}
+
+func (i *InodeToPath) dirStateOf(fullpath util.FullPath) *dirState {
+	return i.dirPaths[fullpath]
+}
+
+func (i *InodeToPath) HasPath(path util.FullPath) bool {
+	i.RLock()
+	defer i.RUnlock()
+	_, found := i.path2inode[path]
+	return found
+}
+
+func (i *InodeToPath) MarkChildrenCached(fullpath util.FullPath) {
+	i.Lock()
+	defer i.Unlock()
+	d := i.dirStateOf(fullpath)
+	if d == nil {
+		// https://github.com/seaweedfs/seaweedfs/issues/4968
+		// glog.Fatalf("MarkChildrenCached not found inode %v", fullpath)
+		glog.Warningf("MarkChildrenCached not a tracked directory: %v", fullpath)
+		return
+	}
+	d.isChildrenCached = true
+	d.readDirDirect = false
+	now := time.Now()
+	d.lastAccess = now
+	d.lastRefresh = now
+	d.lastRebuildAttempt = time.Time{}
+	if i.cacheMetaTtlSec > 0 {
+		d.cachedExpiresTime = now.Add(i.cacheMetaTtlSec)
+	}
+}
+
+func (i *InodeToPath) IsChildrenCached(fullpath util.FullPath) bool {
+	i.RLock()
+	defer i.RUnlock()
+	d := i.dirStateOf(fullpath)
+	if d == nil {
+		return false
+	}
+	if d.isChildrenCached {
+		return d.cachedExpiresTime.IsZero() || time.Now().Before(d.cachedExpiresTime)
+	}
+	return false
+}
+
+// ShouldRebuildExpiredDir reports whether a directory was fully cached, its
+// TTL has elapsed, and enough time has passed since the last failed rebuild
+// attempt to retry. Distinct from IsChildrenCached returning false for a
+// never-cached, invalidated, evicted, or read-through directory: those leave
+// isChildrenCached clear, while a plain TTL expiry keeps it set.
+func (i *InodeToPath) ShouldRebuildExpiredDir(fullpath util.FullPath, cooldown time.Duration) bool {
+	i.RLock()
+	defer i.RUnlock()
+	d := i.dirStateOf(fullpath)
+	if d == nil || !d.isChildrenCached {
+		return false
+	}
+	if d.cachedExpiresTime.IsZero() || time.Now().Before(d.cachedExpiresTime) {
+		return false
+	}
+	return d.lastRebuildAttempt.IsZero() || time.Since(d.lastRebuildAttempt) >= cooldown
+}
+
+// MarkRebuildAttempt records that a rebuild was attempted at now, so
+// ShouldRebuildExpiredDir backs off for the cooldown before retrying.
+func (i *InodeToPath) MarkRebuildAttempt(fullpath util.FullPath, now time.Time) {
+	i.Lock()
+	defer i.Unlock()
+	if d := i.dirStateOf(fullpath); d != nil {
+		d.lastRebuildAttempt = now
+	}
+}
+
+func (i *InodeToPath) HasInode(inode uint64) bool {
+	if inode == 1 {
+		return true
+	}
+	i.RLock()
+	defer i.RUnlock()
+	_, found := i.inode2path[inode]
+	return found
+}
+
+func (i *InodeToPath) InvalidateAllChildrenCache() {
+	i.Lock()
+	defer i.Unlock()
+	for _, d := range i.dirStates {
+		if d.isChildrenCached {
+			d.resetCacheState()
+		}
+	}
+}
+
+func (i *InodeToPath) InvalidateChildrenCache(fullpath util.FullPath) {
+	i.Lock()
+	defer i.Unlock()
+	if d := i.dirStateOf(fullpath); d != nil {
+		d.resetCacheState()
+	}
+}
+
+// AdjustSubdirCount adjusts the subdirectory count for a directory inode.
+// delta is typically +1 (mkdir) or -1 (rmdir).
+func (i *InodeToPath) AdjustSubdirCount(dirPath util.FullPath, delta int32) {
+	i.Lock()
+	defer i.Unlock()
+	d := i.dirStateOf(dirPath)
+	if d == nil {
+		return
+	}
+	d.subdirCount += delta
+	if d.subdirCount < 0 {
+		d.subdirCount = 0
+	}
+}
+
+// GetSubdirCount returns the tracked subdirectory count for a directory.
+func (i *InodeToPath) GetSubdirCount(dirPath util.FullPath) int32 {
+	i.RLock()
+	defer i.RUnlock()
+	d := i.dirStateOf(dirPath)
+	if d == nil {
+		return 0
+	}
+	return d.subdirCount
+}
+
+// SetSubdirCount sets the subdirectory count for a directory (used after readdir).
+func (i *InodeToPath) SetSubdirCount(dirPath util.FullPath, count int32) {
+	i.Lock()
+	defer i.Unlock()
+	if d := i.dirStateOf(dirPath); d != nil {
+		d.subdirCount = count
+	}
+}
+
+func (i *InodeToPath) TouchDirectory(fullpath util.FullPath) {
+	i.Lock()
+	defer i.Unlock()
+	if d := i.dirStateOf(fullpath); d != nil {
+		d.lastAccess = time.Now()
+	}
+}
+
+func (i *InodeToPath) MarkDirectoryReadThrough(fullpath util.FullPath, now time.Time) bool {
+	i.Lock()
+	defer i.Unlock()
+	d := i.dirStateOf(fullpath)
+	if d == nil {
+		return false
+	}
+	d.isChildrenCached = false
+	d.readDirDirect = true
+	d.cachedExpiresTime = time.Time{}
+	d.lastAccess = now
+	d.lastRefresh = time.Time{}
+	return true
+}
+
+func (i *InodeToPath) ShouldReadDirectoryDirect(fullpath util.FullPath) bool {
+	i.RLock()
+	defer i.RUnlock()
+	d := i.dirStateOf(fullpath)
+	if d == nil {
+		return false
+	}
+	return d.readDirDirect
+}
+
+func (i *InodeToPath) MarkDirectoryRefreshed(fullpath util.FullPath, now time.Time) {
+	i.Lock()
+	defer i.Unlock()
+	d := i.dirStateOf(fullpath)
+	if d == nil {
+		return
+	}
+	d.lastRefresh = now
+	d.lastAccess = now
+	d.readDirDirect = false
+	if i.cacheMetaTtlSec > 0 {
+		d.cachedExpiresTime = now.Add(i.cacheMetaTtlSec)
+	}
+}
+
+func (i *InodeToPath) CollectEvictableDirs(now time.Time, idle time.Duration) []util.FullPath {
+	if idle <= 0 {
+		return nil
+	}
+	i.Lock()
+	defer i.Unlock()
+	var dirs []util.FullPath
+	for inode, d := range i.dirStates {
+		if !d.isChildrenCached {
+			continue
+		}
+		if d.lastAccess.IsZero() || now.Sub(d.lastAccess) < idle {
+			continue
+		}
+		d.resetCacheState()
+		if entry, ok := i.inode2path[inode]; ok {
+			dirs = entry.appendPaths(dirs)
+		}
+	}
+	return dirs
+}
+
+func (i *InodeToPath) AddPath(inode uint64, path util.FullPath) {
+	i.Lock()
+	defer i.Unlock()
+	i.path2inode[path] = inode
+
+	ie, found := i.inode2path[inode]
+	if found {
+		ie.addPath(path)
+		ie.nlookup++
+	} else {
+		i.inode2path[inode] = &InodeEntry{
+			path:    path,
+			nlookup: 1,
+		}
+	}
+}
+
+// RemovePath drops the name. onStillReferenced, if given, runs under the
+// table's lock when the kernel still holds lookup references to the inode:
+// such an inode keeps receiving requests until the final forget, and holding
+// the lock is what keeps that forget from racing whatever per-inode state the
+// callback installs — Forget releases under the same lock.
+func (i *InodeToPath) RemovePath(path util.FullPath, onStillReferenced func(inode uint64)) {
+	i.Lock()
+	defer i.Unlock()
+	inode, found := i.path2inode[path]
+	if found {
+		delete(i.path2inode, path)
+		i.dropDirPath(inode)
+		i.removePathFromInode2Path(inode, path)
+		if ie := i.inode2path[inode]; ie != nil && ie.nlookup > 0 && onStillReferenced != nil {
+			onStillReferenced(inode)
+		}
+	}
+}
+
+func (i *InodeToPath) removePathFromInode2Path(inode uint64, path util.FullPath) {
+	ie, found := i.inode2path[inode]
+	if !found {
+		return
+	}
+	if !ie.removeOnePath(path) {
+		return
+	}
+}
+
+func (i *InodeToPath) MovePath(sourcePath, targetPath util.FullPath) (sourceInode, targetInode uint64) {
+	i.Lock()
+	defer i.Unlock()
+	sourceInode, sourceFound := i.path2inode[sourcePath]
+	if !sourceFound {
+		// Nothing of ours to move: the source was never visited here, or a
+		// redelivery already moved it. Whatever sits at the target is not ours
+		// to take apart on the strength of an absent source, and deciding that
+		// outside this lock would race a concurrent move to the same target.
+		return 0, 0
+	}
+	targetInode, targetFound := i.path2inode[targetPath]
+	if targetFound {
+		i.removePathFromInode2Path(targetInode, targetPath)
+		delete(i.path2inode, targetPath)
+		i.dropDirPath(targetInode)
+	}
+	delete(i.path2inode, sourcePath)
+	i.path2inode[targetPath] = sourceInode
+	if entry, entryFound := i.inode2path[sourceInode]; entryFound {
+		entry.replacePath(sourcePath, targetPath)
+		if d := i.dirStates[sourceInode]; d != nil {
+			i.dropDirPath(sourceInode)
+			d.path = targetPath
+			i.dirPaths[targetPath] = d
+			d.resetCacheState()
+		}
+	} else {
+		glog.Errorf("MovePath %s to %s: sourceInode %d not found", sourcePath, targetPath, sourceInode)
+	}
+	return
+}
+
+// Forget drops nlookup references. onRelease, if given, runs at the moment the
+// inode is released and while the table is still locked: state keyed by the
+// inode number has to be dropped there, because the number is derived from the
+// path and a lookup arriving after the unlock would be handed the same one.
+func (i *InodeToPath) Forget(inode, nlookup uint64, onRelease func(inode uint64), onForgetDir func(dir util.FullPath)) {
+	var dirPaths []util.FullPath
+	callOnForgetDir := false
+
+	i.Lock()
+	path, found := i.inode2path[inode]
+	if found {
+		if nlookup > path.nlookup {
+			glog.Errorf("kernel forget over-decrement: inode %d path %v current %d forget %d", inode, path.path, path.nlookup, nlookup)
+			path.nlookup = 0
+		} else {
+			path.nlookup -= nlookup
+		}
+		glog.V(4).Infof("kernel forget: inode %d path %v nlookup %d", inode, path.path, path.nlookup)
+		if path.nlookup == 0 {
+			if onRelease != nil {
+				onRelease(inode)
+			}
+			if _, isDir := i.dirStates[inode]; isDir && onForgetDir != nil {
+				dirPaths = path.appendPaths(nil)
+				callOnForgetDir = true
+			}
+			delete(i.path2inode, path.path)
+			if path.extraPaths != nil {
+				for _, p := range *path.extraPaths {
+					delete(i.path2inode, p)
+				}
+			}
+			delete(i.inode2path, inode)
+			i.dropDirPath(inode)
+			delete(i.dirStates, inode)
+		} else {
+			glog.V(4).Infof("kernel forget but nlookup not zero: inode %d path %v nlookup %d", inode, path.path, path.nlookup)
+		}
+	} else {
+		glog.Warningf("kernel forget but inode not found: inode %d", inode)
+	}
+	i.Unlock()
+
+	if callOnForgetDir {
+		for _, p := range dirPaths {
+			onForgetDir(p)
+		}
+	}
+}

weed/mount/inode_to_path_dirstate_test.go

@@ -0,0 +1,92 @@
+package mount
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// The two indexes hold the same states, at the path the inode entry carries.
+func checkDirIndexes(t *testing.T, itp *InodeToPath) {
+	t.Helper()
+	itp.RLock()
+	defer itp.RUnlock()
+	for path, d := range itp.dirPaths {
+		if d.path != path {
+			t.Errorf("dirPaths[%s] holds a state for %s", path, d.path)
+		}
+	}
+	for inode, d := range itp.dirStates {
+		e := itp.inode2path[inode]
+		if e == nil || e.path == "" {
+			continue
+		}
+		if e.path != d.path {
+			t.Errorf("inode %d: entry at %s, directory state at %s", inode, e.path, d.path)
+		}
+		if itp.dirPaths[d.path] != d {
+			t.Errorf("inode %d: directory state at %s is missing from dirPaths", inode, d.path)
+		}
+	}
+}
+
+func TestDirStateIndexesStayInStep(t *testing.T) {
+	itp := NewInodeToPath(util.FullPath("/"), 0)
+	now := time.Now().Unix()
+
+	dirInode := itp.Lookup("/a", now, true, false, 0, true)
+	subInode := itp.Lookup("/a/sub", now, true, false, 0, true)
+	itp.Lookup("/a/sub/f.txt", now, false, false, 0, true)
+	checkDirIndexes(t, itp)
+
+	if itp.dirStateOf("/a/sub") == nil {
+		t.Fatal("/a/sub not indexed as a directory")
+	}
+	if itp.dirStateOf("/a/sub/f.txt") != nil {
+		t.Error("a file landed in the directory index")
+	}
+
+	itp.MovePath("/a/sub", "/a/moved")
+	checkDirIndexes(t, itp)
+	if itp.dirStateOf("/a/sub") != nil {
+		t.Error("pre-rename directory still indexed")
+	}
+	if itp.dirStateOf("/a/moved") == nil {
+		t.Error("renamed directory not indexed")
+	}
+
+	itp.RemovePath("/a/moved", nil)
+	if itp.dirStateOf("/a/moved") != nil {
+		t.Error("removed directory still indexed")
+	}
+	checkDirIndexes(t, itp)
+
+	itp.Forget(subInode, 1, nil, nil)
+	itp.Forget(dirInode, 1, nil, nil)
+	checkDirIndexes(t, itp)
+	if itp.dirStateOf("/a") != nil {
+		t.Error("forgotten directory still indexed")
+	}
+}
+
+// A released directory's state keeps its old path. Forgetting it must not take
+// the index entry of whatever holds that name now.
+func TestForgetDoesNotDropAReusedDirPath(t *testing.T) {
+	itp := NewInodeToPath(util.FullPath("/"), 0)
+	now := time.Now().Unix()
+
+	oldInode := itp.Lookup("/a", now, true, false, 0, true)
+	itp.RemovePath("/a", nil)
+	newInode := itp.Lookup("/a", now+1, true, false, 0, true)
+	if newInode == oldInode {
+		t.Fatalf("recreated directory reused inode %d", newInode)
+	}
+
+	itp.Forget(oldInode, 1, nil, nil)
+
+	if itp.dirStateOf("/a") == nil {
+		t.Error("live directory lost its index when the released one was forgotten")
+	}
+	checkDirIndexes(t, itp)
+}

weed/mount/inode_to_path_test.go

@@ -0,0 +1,210 @@
+package mount
+
+import (
+	"testing"
+	"time"
+	"unsafe"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// A mount with millions of files is mostly these, so guard the size class.
+func TestInodeEntryStaysInSizeClass(t *testing.T) {
+	if unsafe.Sizeof(uintptr(0)) != 8 {
+		t.Skip("size classes are 64-bit specific")
+	}
+	if got := unsafe.Sizeof(InodeEntry{}); got != 32 {
+		t.Errorf("sizeof(InodeEntry) = %d, want 32", got)
+	}
+}
+
+func inodeEntryWith(paths ...util.FullPath) InodeEntry {
+	var ie InodeEntry
+	for _, p := range paths {
+		ie.addPath(p)
+	}
+	return ie
+}
+
+func TestInodeEntry_removeOnePath(t *testing.T) {
+	tests := []struct {
+		name  string
+		entry InodeEntry
+		p     util.FullPath
+		want  bool
+		count int
+	}{
+		{
+			name:  "actual case",
+			entry: inodeEntryWith("/pjd/nx", "/pjd/n0"),
+			p:     "/pjd/nx",
+			want:  true,
+			count: 1,
+		},
+		{
+			name:  "empty",
+			entry: InodeEntry{},
+			p:     "x",
+			want:  false,
+			count: 0,
+		},
+		{
+			name:  "single",
+			entry: inodeEntryWith("/x"),
+			p:     "/x",
+			want:  true,
+			count: 0,
+		},
+		{
+			name:  "first",
+			entry: inodeEntryWith("/x", "/y", "/z"),
+			p:     "/x",
+			want:  true,
+			count: 2,
+		},
+		{
+			name:  "middle",
+			entry: inodeEntryWith("/x", "/y", "/z"),
+			p:     "/y",
+			want:  true,
+			count: 2,
+		},
+		{
+			name:  "last",
+			entry: inodeEntryWith("/x", "/y", "/z"),
+			p:     "/z",
+			want:  true,
+			count: 2,
+		},
+		{
+			name:  "not found",
+			entry: inodeEntryWith("/x", "/y", "/z"),
+			p:     "/t",
+			want:  false,
+			count: 3,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.entry.removeOnePath(tt.p); got != tt.want {
+				t.Errorf("removeOnePath() = %v, want %v", got, tt.want)
+			}
+			left := tt.entry.appendPaths(nil)
+			if tt.count != len(left) {
+				t.Errorf("removeOnePath path count = %v, want %v", len(left), tt.count)
+			}
+			for i, p := range left {
+				if p == tt.p {
+					t.Errorf("removeOnePath found path still exists at %v, %v", i, p)
+				}
+			}
+		})
+	}
+}
+
+// Only directory inodes carry dirState; files never do. This keeps the file
+// InodeEntry in the smaller size class, which is the memory win on large mounts.
+func TestOnlyDirectoriesGetDirState(t *testing.T) {
+	itp := NewInodeToPath(util.FullPath("/"), 60)
+
+	file := util.FullPath("/data/file.txt")
+	fileInode := itp.Lookup(file, time.Now().Unix(), false, false, 0, true)
+	if _, ok := itp.dirStates[fileInode]; ok {
+		t.Fatal("file inode must not have a dirState")
+	}
+	// dir queries against a file are no-ops and must not register one
+	itp.GetSubdirCount(file)
+	itp.IsChildrenCached(file)
+	itp.ShouldReadDirectoryDirect(file)
+	itp.MarkChildrenCached(file)
+	if _, ok := itp.dirStates[fileInode]; ok {
+		t.Fatal("dir queries on a file inode must not create a dirState")
+	}
+
+	dir := util.FullPath("/data")
+	dirInode := itp.Lookup(dir, time.Now().Unix(), true, false, 0, true)
+	if _, ok := itp.dirStates[dirInode]; !ok {
+		t.Fatal("directory inode should be registered in dirStates at creation")
+	}
+
+	// forgetting the directory drops its dirState
+	itp.Forget(dirInode, 1, nil, nil)
+	if _, ok := itp.dirStates[dirInode]; ok {
+		t.Fatal("forgotten directory must be removed from dirStates")
+	}
+}
+
+func TestMarkChildrenCachedClearsReadThroughMode(t *testing.T) {
+	root := util.FullPath("/")
+	dir := util.FullPath("/data")
+
+	inodeToPath := NewInodeToPath(root, 60)
+	inodeToPath.Lookup(dir, time.Now().Unix(), true, false, 0, true)
+
+	if !inodeToPath.MarkDirectoryReadThrough(dir, time.Now()) {
+		t.Fatal("expected read-through flag to be set")
+	}
+	inodeToPath.MarkChildrenCached(dir)
+
+	if !inodeToPath.IsChildrenCached(dir) {
+		t.Fatal("directory should be cached after MarkChildrenCached")
+	}
+	if inodeToPath.ShouldReadDirectoryDirect(dir) {
+		t.Fatal("directory should leave read-through mode after caching")
+	}
+}
+
+// State keyed by an inode number has to be dropped while the table is locked.
+// The numbers come from the path, so a lookup racing a forget is handed the
+// same one back, and a cleanup running after the unlock would wipe what that
+// lookup just stored.
+func TestForgetOnReleaseRunsUnderLock(t *testing.T) {
+	itp := NewInodeToPath(util.FullPath("/"), 60)
+	file := util.FullPath("/data/f.txt")
+	inode := itp.Lookup(file, time.Now().Unix(), false, false, 0, true)
+
+	itp.Lookup(file, time.Now().Unix(), false, false, 0, true) // nlookup is 2 now
+
+	calls := 0
+	onRelease := func(released uint64) {
+		calls++
+		if released != inode {
+			t.Errorf("released inode = %d, want %d", released, inode)
+		}
+		if itp.TryLock() {
+			itp.Unlock()
+			t.Error("onRelease ran outside the critical section")
+		}
+	}
+
+	itp.Forget(inode, 1, onRelease, nil)
+	if calls != 0 {
+		t.Fatalf("partial forget must not release: onRelease called %d times", calls)
+	}
+
+	itp.Forget(inode, 1, onRelease, nil)
+	if calls != 1 {
+		t.Fatalf("onRelease called %d times, want 1", calls)
+	}
+}
+
+// A move with no source is not a move. Deciding that inside the lock is what
+// keeps two invalidations racing on the same rename from unlinking each other's
+// work.
+func TestMovePathWithNoSourceLeavesTheTarget(t *testing.T) {
+	itp := NewInodeToPath(util.FullPath("/"), 0)
+	inode := itp.Lookup("/a/f.txt", time.Now().Unix(), false, false, 0, true)
+
+	itp.MovePath("/a/f.txt", "/a/g.txt")
+	if sourceInode, targetInode := itp.MovePath("/a/f.txt", "/a/g.txt"); sourceInode != 0 || targetInode != 0 {
+		t.Errorf("repeat move reported %d -> %d, want nothing moved", sourceInode, targetInode)
+	}
+
+	if got, status := itp.GetPath(inode); status != fuse.OK || got != "/a/g.txt" {
+		t.Errorf("inode resolves to %q (%v), want /a/g.txt", got, status)
+	}
+	if got, found := itp.GetInode("/a/g.txt"); !found || got != inode {
+		t.Errorf("/a/g.txt resolves to %d (found %v), want %d", got, found, inode)
+	}
+}

weed/mount/locked_entry.go

@@ -0,0 +1,45 @@
+package mount
+
+import (
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+type LockedEntry struct {
+	*filer_pb.Entry
+	sync.RWMutex
+}
+
+func (le *LockedEntry) GetEntry() *filer_pb.Entry {
+	le.RLock()
+	defer le.RUnlock()
+	return le.Entry
+}
+
+// SetEntry sets the entry of the LockedEntry
+// entry should never be nil
+func (le *LockedEntry) SetEntry(entry *filer_pb.Entry) {
+	le.Lock()
+	defer le.Unlock()
+	le.Entry = entry
+}
+
+func (le *LockedEntry) UpdateEntry(fn func(entry *filer_pb.Entry)) *filer_pb.Entry {
+	le.Lock()
+	defer le.Unlock()
+	fn(le.Entry)
+	return le.Entry
+}
+
+func (le *LockedEntry) GetChunks() []*filer_pb.FileChunk {
+	le.RLock()
+	defer le.RUnlock()
+	return le.Entry.Chunks
+}
+
+func (le *LockedEntry) AppendChunks(newChunks []*filer_pb.FileChunk) {
+	le.Lock()
+	defer le.Unlock()
+	le.Entry.Chunks = append(le.Entry.Chunks, newChunks...)
+}

weed/mount/meta_cache/cache_config.go

@@ -0,0 +1,32 @@
+package meta_cache
+
+import "github.com/seaweedfs/seaweedfs/weed/util"
+
+var (
+	_ = util.Configuration(&cacheConfig{})
+)
+
+// implementing util.Configuration
+type cacheConfig struct {
+	dir string
+}
+
+func (c cacheConfig) GetString(key string) string {
+	return c.dir
+}
+
+func (c cacheConfig) GetBool(key string) bool {
+	panic("implement me")
+}
+
+func (c cacheConfig) GetInt(key string) int {
+	panic("implement me")
+}
+
+func (c cacheConfig) GetStringSlice(key string) []string {
+	panic("implement me")
+}
+
+func (c cacheConfig) SetDefault(key string, value interface{}) {
+	panic("implement me")
+}

weed/mount/meta_cache/id_mapper.go

@@ -0,0 +1,101 @@
+package meta_cache
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+type UidGidMapper struct {
+	uidMapper *IdMapper
+	gidMapper *IdMapper
+}
+
+type IdMapper struct {
+	localToFiler map[uint32]uint32
+	filerToLocal map[uint32]uint32
+}
+
+// UidGidMapper translates local uid/gid to filer uid/gid
+// The local storage always persists the same as the filer.
+// The local->filer translation happens when updating the filer first and later saving to meta_cache.
+// And filer->local happens when reading from the meta_cache.
+func NewUidGidMapper(uidPairsStr, gidPairStr string) (*UidGidMapper, error) {
+	uidMapper, err := newIdMapper(uidPairsStr)
+	if err != nil {
+		return nil, err
+	}
+	gidMapper, err := newIdMapper(gidPairStr)
+	if err != nil {
+		return nil, err
+	}
+
+	return &UidGidMapper{
+		uidMapper: uidMapper,
+		gidMapper: gidMapper,
+	}, nil
+}
+
+func (m *UidGidMapper) LocalToFiler(uid, gid uint32) (uint32, uint32) {
+	return m.uidMapper.LocalToFiler(uid), m.gidMapper.LocalToFiler(gid)
+}
+func (m *UidGidMapper) FilerToLocal(uid, gid uint32) (uint32, uint32) {
+	return m.uidMapper.FilerToLocal(uid), m.gidMapper.FilerToLocal(gid)
+}
+
+func (m *IdMapper) LocalToFiler(id uint32) uint32 {
+	value, found := m.localToFiler[id]
+	if found {
+		return value
+	}
+	return id
+}
+func (m *IdMapper) FilerToLocal(id uint32) uint32 {
+	value, found := m.filerToLocal[id]
+	if found {
+		return value
+	}
+	return id
+}
+
+func newIdMapper(pairsStr string) (*IdMapper, error) {
+
+	localToFiler, filerToLocal, err := parseUint32Pairs(pairsStr)
+	if err != nil {
+		return nil, err
+	}
+
+	return &IdMapper{
+		localToFiler: localToFiler,
+		filerToLocal: filerToLocal,
+	}, nil
+
+}
+
+func parseUint32Pairs(pairsStr string) (localToFiler, filerToLocal map[uint32]uint32, err error) {
+
+	if pairsStr == "" {
+		return
+	}
+
+	localToFiler = make(map[uint32]uint32)
+	filerToLocal = make(map[uint32]uint32)
+	for _, pairStr := range strings.Split(pairsStr, ",") {
+		pair := strings.Split(pairStr, ":")
+		localUidStr, filerUidStr := pair[0], pair[1]
+		localUid, localUidErr := strconv.Atoi(localUidStr)
+		if localUidErr != nil {
+			err = fmt.Errorf("failed to parse local %s: %v", localUidStr, localUidErr)
+			return
+		}
+		filerUid, filerUidErr := strconv.Atoi(filerUidStr)
+		if filerUidErr != nil {
+			err = fmt.Errorf("failed to parse remote %s: %v", filerUidStr, filerUidErr)
+			return
+		}
+		localToFiler[uint32(localUid)] = uint32(filerUid)
+		filerToLocal[uint32(filerUid)] = uint32(localUid)
+	}
+
+	return
+}

weed/mount/meta_cache/meta_cache_apply_test.go

@@ -0,0 +1,931 @@
+package meta_cache
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestApplyMetadataResponseAppliesEventsInOrder(t *testing.T) {
+	mc, _, notifications, invalidations := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	createResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name: "file.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    1,
+					FileMode: 0100644,
+					FileSize: 11,
+				},
+			},
+		},
+	}
+	updateResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{
+				Name: "file.txt",
+			},
+			NewEntry: &filer_pb.Entry{
+				Name: "file.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    2,
+					FileMode: 0100644,
+					FileSize: 29,
+				},
+			},
+			NewParentPath: "/dir",
+		},
+	}
+	deleteResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{
+				Name: "file.txt",
+			},
+		},
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), createResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply create: %v", err)
+	}
+
+	entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt"))
+	if err != nil {
+		t.Fatalf("find created entry: %v", err)
+	}
+	if entry.FileSize != 11 {
+		t.Fatalf("created file size = %d, want 11", entry.FileSize)
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), updateResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply update: %v", err)
+	}
+
+	entry, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt"))
+	if err != nil {
+		t.Fatalf("find updated entry: %v", err)
+	}
+	if entry.FileSize != 29 {
+		t.Fatalf("updated file size = %d, want 29", entry.FileSize)
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), deleteResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply delete: %v", err)
+	}
+
+	entry, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt"))
+	if err != filer_pb.ErrNotFound {
+		t.Fatalf("find deleted entry error = %v, want %v", err, filer_pb.ErrNotFound)
+	}
+	if entry != nil {
+		t.Fatalf("deleted entry still cached: %+v", entry)
+	}
+
+	if got := countPath(notifications.paths(), util.FullPath("/dir")); got != 3 {
+		t.Fatalf("directory notifications for /dir = %d, want 3", got)
+	}
+	mc.WaitForEntryInvalidations()
+	if got := countPath(invalidations.paths(), util.FullPath("/dir/file.txt")); got != 3 {
+		t.Fatalf("invalidations for /dir/file.txt = %d, want 3 (create + update + delete)", got)
+	}
+}
+
+func TestApplyMetadataResponseRenamesAcrossCachedDirectories(t *testing.T) {
+	mc, _, notifications, invalidations := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/src": true,
+		"/dst": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/src/file.tmp",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0100644,
+			FileSize: 7,
+		},
+	}, 0); err != nil {
+		t.Fatalf("insert source entry: %v", err)
+	}
+
+	renameResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/src",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{
+				Name: "file.tmp",
+			},
+			NewEntry: &filer_pb.Entry{
+				Name: "file.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    2,
+					FileMode: 0100644,
+					FileSize: 41,
+				},
+			},
+			NewParentPath: "/dst",
+		},
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), renameResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply rename: %v", err)
+	}
+
+	oldEntry, _, err := mc.FindEntry(context.Background(), util.FullPath("/src/file.tmp"))
+	if err != filer_pb.ErrNotFound {
+		t.Fatalf("find old path error = %v, want %v", err, filer_pb.ErrNotFound)
+	}
+	if oldEntry != nil {
+		t.Fatalf("old path still cached: %+v", oldEntry)
+	}
+
+	newEntry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dst/file.txt"))
+	if err != nil {
+		t.Fatalf("find new path: %v", err)
+	}
+	if newEntry.FileSize != 41 {
+		t.Fatalf("renamed file size = %d, want 41", newEntry.FileSize)
+	}
+
+	if got := countPath(notifications.paths(), util.FullPath("/src")); got != 1 {
+		t.Fatalf("directory notifications for /src = %d, want 1", got)
+	}
+	if got := countPath(notifications.paths(), util.FullPath("/dst")); got != 1 {
+		t.Fatalf("directory notifications for /dst = %d, want 1", got)
+	}
+	mc.WaitForEntryInvalidations()
+	if got := countPath(invalidations.paths(), util.FullPath("/src/file.tmp")); got != 1 {
+		t.Fatalf("invalidations for /src/file.tmp = %d, want 1", got)
+	}
+	if got := countPath(invalidations.paths(), util.FullPath("/dst/file.txt")); got != 1 {
+		t.Fatalf("invalidations for /dst/file.txt = %d, want 1", got)
+	}
+}
+
+func TestApplyMetadataResponseLocalOptionsSkipInvalidations(t *testing.T) {
+	mc, _, notifications, invalidations := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/dir/file.txt",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0100644,
+			FileSize: 7,
+		},
+	}, 0); err != nil {
+		t.Fatalf("insert source entry: %v", err)
+	}
+
+	updateResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{
+				Name: "file.txt",
+			},
+			NewEntry: &filer_pb.Entry{
+				Name: "file.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    2,
+					FileMode: 0100644,
+					FileSize: 17,
+				},
+			},
+			NewParentPath: "/dir",
+		},
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), updateResp, LocalMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply local update: %v", err)
+	}
+
+	entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt"))
+	if err != nil {
+		t.Fatalf("find updated entry: %v", err)
+	}
+	if entry.FileSize != 17 {
+		t.Fatalf("updated file size = %d, want 17", entry.FileSize)
+	}
+	if got := countPath(notifications.paths(), util.FullPath("/dir")); got != 1 {
+		t.Fatalf("directory notifications for /dir = %d, want 1", got)
+	}
+	mc.WaitForEntryInvalidations()
+	if got := len(invalidations.paths()); got != 0 {
+		t.Fatalf("invalidations = %d, want 0", got)
+	}
+}
+
+func TestApplyMetadataResponseDeduplicatesRepeatedFilerEvent(t *testing.T) {
+	mc, _, notifications, invalidations := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/dir/file.txt",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0100644,
+			FileSize: 5,
+		},
+	}, 0); err != nil {
+		t.Fatalf("insert source entry: %v", err)
+	}
+
+	updateResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{
+				Name: "file.txt",
+			},
+			NewEntry: &filer_pb.Entry{
+				Name: "file.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    2,
+					FileMode: 0100644,
+					FileSize: 15,
+				},
+			},
+			NewParentPath: "/dir",
+			Signatures:    []int32{7},
+		},
+		TsNs: 99,
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), updateResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("first apply: %v", err)
+	}
+	if err := mc.ApplyMetadataResponse(context.Background(), updateResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("second apply: %v", err)
+	}
+
+	entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt"))
+	if err != nil {
+		t.Fatalf("find updated entry: %v", err)
+	}
+	if entry.FileSize != 15 {
+		t.Fatalf("updated file size = %d, want 15", entry.FileSize)
+	}
+	if got := countPath(notifications.paths(), util.FullPath("/dir")); got != 1 {
+		t.Fatalf("directory notifications for /dir = %d, want 1", got)
+	}
+	mc.WaitForEntryInvalidations()
+	if got := countPath(invalidations.paths(), util.FullPath("/dir/file.txt")); got != 1 {
+		t.Fatalf("invalidations for /dir/file.txt = %d, want 1", got)
+	}
+}
+
+func TestApplyMetadataResponseSkipsHiddenSystemEntryWhenDisabled(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/": true,
+	})
+	defer mc.Shutdown()
+
+	createResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/",
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name: "topics",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    1,
+					FileMode: uint32(os.ModeDir | 0o755),
+				},
+				IsDirectory: true,
+			},
+		},
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), createResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply create: %v", err)
+	}
+
+	entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/topics"))
+	if err != filer_pb.ErrNotFound {
+		t.Fatalf("find hidden entry error = %v, want %v", err, filer_pb.ErrNotFound)
+	}
+	if entry != nil {
+		t.Fatalf("hidden entry still cached: %+v", entry)
+	}
+}
+
+func TestApplyMetadataResponsePurgesHiddenDestinationPath(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/src": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/topics",
+		Attr: filer.Attr{
+			Crtime: time.Unix(1, 0),
+			Mtime:  time.Unix(1, 0),
+			Mode:   os.ModeDir | 0o755,
+		},
+	}, 0); err != nil {
+		t.Fatalf("insert stale hidden dir: %v", err)
+	}
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/topics/leaked.txt",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0o644,
+			FileSize: 7,
+		},
+	}, 0); err != nil {
+		t.Fatalf("insert leaked hidden child: %v", err)
+	}
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/src/visible",
+		Attr: filer.Attr{
+			Crtime: time.Unix(1, 0),
+			Mtime:  time.Unix(1, 0),
+			Mode:   os.ModeDir | 0o755,
+		},
+	}, 0); err != nil {
+		t.Fatalf("insert source dir: %v", err)
+	}
+
+	renameResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/src",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{
+				Name:        "visible",
+				IsDirectory: true,
+			},
+			NewEntry: &filer_pb.Entry{
+				Name: "topics",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   2,
+					Mtime:    2,
+					FileMode: uint32(os.ModeDir | 0o755),
+				},
+				IsDirectory: true,
+			},
+			NewParentPath: "/",
+		},
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), renameResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply rename: %v", err)
+	}
+
+	if entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/src/visible")); err != filer_pb.ErrNotFound || entry != nil {
+		t.Fatalf("source dir after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
+	}
+	if entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/topics")); err != filer_pb.ErrNotFound || entry != nil {
+		t.Fatalf("hidden destination after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
+	}
+	if entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/topics/leaked.txt")); err != filer_pb.ErrNotFound || entry != nil {
+		t.Fatalf("hidden child after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
+	}
+}
+
+// The entry attached to each invalidation is what an open file handle gets
+// refreshed with, so it must be the entry now at that path — or nil when the
+// path was vacated and the handle should keep its last entry — versioned by
+// the event's filer log timestamp.
+func TestCollectEntryInvalidationsCarryAuthoritativeEntries(t *testing.T) {
+	newEntry := &filer_pb.Entry{
+		Name:       "file.txt",
+		Attributes: &filer_pb.FuseAttributes{FileSize: 42},
+	}
+
+	update := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		TsNs:      77,
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: "file.txt"},
+			NewEntry:      newEntry,
+			NewParentPath: "/dir",
+		},
+	}
+	got := collectEntryInvalidations(update)
+	if len(got) != 1 || got[0].Path != "/dir/file.txt" || got[0].Entry != newEntry || got[0].TsNs != 77 {
+		t.Fatalf("in-place update invalidations = %+v, want [{/dir/file.txt NewEntry ts 77}]", got)
+	}
+
+	rename := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/src",
+		TsNs:      78,
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: "file.tmp"},
+			NewEntry:      newEntry,
+			NewParentPath: "/dst",
+		},
+	}
+	got = collectEntryInvalidations(rename)
+	if len(got) != 2 || got[0].Path != "/src/file.tmp" || got[0].Entry != nil || got[0].TsNs != 78 ||
+		got[1].Path != "/dst/file.txt" || got[1].Entry != newEntry || got[1].TsNs != 78 {
+		t.Fatalf("rename invalidations = %+v, want [{/src/file.tmp nil} {/dst/file.txt NewEntry}]", got)
+	}
+
+	create := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		TsNs:      79,
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: newEntry,
+		},
+	}
+	got = collectEntryInvalidations(create)
+	if len(got) != 1 || got[0].Path != "/dir/file.txt" || got[0].Entry != newEntry || got[0].TsNs != 79 {
+		t.Fatalf("create invalidations = %+v, want [{/dir/file.txt NewEntry ts 79}]", got)
+	}
+
+	deleteResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		TsNs:      80,
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{Name: "file.txt"},
+		},
+	}
+	got = collectEntryInvalidations(deleteResp)
+	if len(got) != 1 || got[0].Path != "/dir/file.txt" || got[0].Entry != nil || got[0].TsNs != 80 {
+		t.Fatalf("delete invalidations = %+v, want [{/dir/file.txt nil ts 80}]", got)
+	}
+}
+
+func newTestMetaCache(t *testing.T, cached map[util.FullPath]bool) (*MetaCache, map[util.FullPath]bool, *recordedPaths, *recordedPaths) {
+	t.Helper()
+
+	mapper, err := NewUidGidMapper("", "")
+	if err != nil {
+		t.Fatalf("uid/gid mapper: %v", err)
+	}
+
+	var cachedMu sync.Mutex
+	notifications := &recordedPaths{}
+	invalidations := &recordedPaths{}
+
+	mc := NewMetaCache(
+		filepath.Join(t.TempDir(), "meta"),
+		mapper,
+		util.FullPath("/"),
+		false,
+		func(path util.FullPath) {
+			cachedMu.Lock()
+			defer cachedMu.Unlock()
+			cached[path] = true
+		},
+		func(path util.FullPath) bool {
+			cachedMu.Lock()
+			defer cachedMu.Unlock()
+			return cached[path]
+		},
+		func(inv EntryInvalidation) {
+			invalidations.record(inv.Path)
+		},
+		func(dir util.FullPath) {
+			notifications.record(dir)
+		},
+	)
+
+	return mc, cached, notifications, invalidations
+}
+
+type recordedPaths struct {
+	mu    sync.Mutex
+	items []util.FullPath
+}
+
+func (r *recordedPaths) record(path util.FullPath) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.items = append(r.items, path)
+}
+
+func (r *recordedPaths) paths() []util.FullPath {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return append([]util.FullPath(nil), r.items...)
+}
+
+func countPath(paths []util.FullPath, target util.FullPath) int {
+	count := 0
+	for _, path := range paths {
+		if path == target {
+			count++
+		}
+	}
+	return count
+}
+
+// An unversioned local write replaces the content, so the previous version
+// claim no longer describes it and must be cleared — keeping it would fence
+// out events correcting the unversioned state.
+func TestUnversionedWriteClearsEntryVersion(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	versioned := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		TsNs:      1000,
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name:       "file.txt",
+				Attributes: &filer_pb.FuseAttributes{FileSize: 11, FileMode: 0100644},
+			},
+		},
+	}
+	if err := mc.ApplyMetadataResponse(context.Background(), versioned, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply versioned event: %v", err)
+	}
+	if _, versionTsNs, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt")); err != nil || versionTsNs != 1000 {
+		t.Fatalf("versioned entry = ts %d, %v; want 1000", versionTsNs, err)
+	}
+
+	if err := mc.UpdateEntry(context.Background(), &filer.Entry{
+		FullPath: "/dir/file.txt",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(2, 0),
+			Mode:     0100644,
+			FileSize: 22,
+		},
+	}); err != nil {
+		t.Fatalf("local update: %v", err)
+	}
+	if _, versionTsNs, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt")); err != nil || versionTsNs != 0 {
+		t.Fatalf("after unversioned write = ts %d, %v; want 0", versionTsNs, err)
+	}
+	mc.WaitForEntryInvalidations()
+}
+
+// A rebuild against a pre-upgrade filer returns no snapshot; the reinserted
+// entries must not inherit the version records their pre-rebuild
+// incarnations left behind, or valid events below the stale claim are
+// rejected.
+func TestUnversionedRebuildClearsStaleVersions(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	versioned := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		TsNs:      3000,
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name:       "file.txt",
+				Attributes: &filer_pb.FuseAttributes{FileSize: 11, FileMode: 0100644},
+			},
+		},
+	}
+	if err := mc.ApplyMetadataResponse(context.Background(), versioned, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply versioned event: %v", err)
+	}
+
+	if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil {
+		t.Fatalf("begin build: %v", err)
+	}
+	if err := mc.doBatchInsertEntries(context.Background(), []*filer.Entry{{
+		FullPath: "/dir/file.txt",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0100644,
+			FileSize: 22,
+		},
+	}}); err != nil {
+		t.Fatalf("batch insert: %v", err)
+	}
+	if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/dir"), 0, nil); err != nil {
+		t.Fatalf("complete unversioned build: %v", err)
+	}
+
+	if _, versionTsNs, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt")); err != nil || versionTsNs != 0 {
+		t.Fatalf("rebuilt entry version = %d, %v; want 0 (stale claim must not survive an unversioned rebuild)", versionTsNs, err)
+	}
+	mc.WaitForEntryInvalidations()
+}
+
+// Tombstones are scoped to directories whose cached state the fence
+// protects; an uncached parent never serves from the store nor applies the
+// resurrecting insert, so a tombstone there would only accumulate.
+func TestTombstonesScopedToCachedDirectories(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":       true,
+		"/cached": true,
+	})
+	defer mc.Shutdown()
+
+	deleteEventFor := func(dir, name string, tsNs int64) *filer_pb.SubscribeMetadataResponse {
+		return &filer_pb.SubscribeMetadataResponse{
+			Directory: dir,
+			TsNs:      tsNs,
+			EventNotification: &filer_pb.EventNotification{
+				OldEntry: &filer_pb.Entry{Name: name},
+			},
+		}
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), deleteEventFor("/cached", "file", 2000), SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply cached delete: %v", err)
+	}
+	if tsNs, tombstone := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/cached/file")); tsNs != 2000 || !tombstone {
+		t.Fatalf("cached-dir delete record = (%d, %v), want tombstone at 2000", tsNs, tombstone)
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), deleteEventFor("/uncached", "file", 2000), SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply uncached delete: %v", err)
+	}
+	if tsNs, _ := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/uncached/file")); tsNs != 0 {
+		t.Fatalf("uncached-dir delete record = %d, want none", tsNs)
+	}
+	mc.WaitForEntryInvalidations()
+}
+
+// A completed listing's absence floor supersedes the direct-child tombstones
+// at or below its snapshot; newer tombstones survive.
+func TestBuildCompletionPrunesSupersededTombstones(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	apply := func(resp *filer_pb.SubscribeMetadataResponse) {
+		t.Helper()
+		if err := mc.ApplyMetadataResponse(context.Background(), resp, SubscriberMetadataResponseApplyOptions); err != nil {
+			t.Fatalf("apply: %v", err)
+		}
+	}
+	createEvent := func(name string, tsNs int64) *filer_pb.SubscribeMetadataResponse {
+		return &filer_pb.SubscribeMetadataResponse{
+			Directory: "/dir",
+			TsNs:      tsNs,
+			EventNotification: &filer_pb.EventNotification{
+				NewEntry: &filer_pb.Entry{
+					Name:       name,
+					Attributes: &filer_pb.FuseAttributes{FileSize: 1, FileMode: 0100644},
+				},
+			},
+		}
+	}
+	deleteEvent := func(name string, tsNs int64) *filer_pb.SubscribeMetadataResponse {
+		return &filer_pb.SubscribeMetadataResponse{
+			Directory: "/dir",
+			TsNs:      tsNs,
+			EventNotification: &filer_pb.EventNotification{
+				OldEntry: &filer_pb.Entry{Name: name},
+			},
+		}
+	}
+
+	apply(createEvent("a", 900))
+	apply(deleteEvent("a", 1000))
+	apply(createEvent("b", 3900))
+	apply(deleteEvent("b", 4000))
+
+	// A deeper descendant's tombstone belongs to its own directory's floor.
+	mc.Lock()
+	mc.setEntryTombstoneLocked(context.Background(), util.FullPath("/dir/sub/x"), 1000)
+	mc.Unlock()
+
+	if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil {
+		t.Fatalf("begin build: %v", err)
+	}
+	if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/dir"), 3000, nil); err != nil {
+		t.Fatalf("complete build: %v", err)
+	}
+
+	if tsNs, _ := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/dir/a")); tsNs != 0 {
+		t.Fatalf("tombstone at 1000 should be pruned by the floor at 3000, got %d", tsNs)
+	}
+	if tsNs, tombstone := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/dir/b")); tsNs != 4000 || !tombstone {
+		t.Fatalf("tombstone at 4000 must survive the floor at 3000, got (%d, %v)", tsNs, tombstone)
+	}
+	if tsNs, tombstone := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/dir/sub/x")); tsNs != 1000 || !tombstone {
+		t.Fatalf("descendant tombstone must survive the parent's prune, got (%d, %v)", tsNs, tombstone)
+	}
+	mc.WaitForEntryInvalidations()
+}
+
+// Evicting a directory clears its children's version records (plain and
+// tombstone) so they cannot accumulate for the lifetime of a long-running
+// mount.
+func TestDirectoryEvictionClearsVersionRecords(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	create := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		TsNs:      1000,
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{Name: "keep", Attributes: &filer_pb.FuseAttributes{FileSize: 1, FileMode: 0100644}},
+		},
+	}
+	if err := mc.ApplyMetadataResponse(context.Background(), create, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply create: %v", err)
+	}
+	del := &filer_pb.SubscribeMetadataResponse{
+		Directory:         "/dir",
+		TsNs:              1100,
+		EventNotification: &filer_pb.EventNotification{OldEntry: &filer_pb.Entry{Name: "gone"}},
+	}
+	if err := mc.ApplyMetadataResponse(context.Background(), del, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply delete: %v", err)
+	}
+
+	if tsNs, _ := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/dir/keep")); tsNs != 1000 {
+		t.Fatalf("version record for /dir/keep = %d, want 1000 before eviction", tsNs)
+	}
+	if tsNs, tombstone := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/dir/gone")); tsNs != 1100 || !tombstone {
+		t.Fatalf("tombstone for /dir/gone = (%d,%v), want (1100,true) before eviction", tsNs, tombstone)
+	}
+
+	if err := mc.DeleteFolderChildren(context.Background(), util.FullPath("/dir")); err != nil {
+		t.Fatalf("evict directory: %v", err)
+	}
+
+	if tsNs, _ := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/dir/keep")); tsNs != 0 {
+		t.Fatalf("version record for /dir/keep survived eviction: %d", tsNs)
+	}
+	if tsNs, _ := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/dir/gone")); tsNs != 0 {
+		t.Fatalf("tombstone for /dir/gone survived eviction: %d", tsNs)
+	}
+	mc.WaitForEntryInvalidations()
+}
+
+// A completed build versions its children through one directory floor rather
+// than a version record per child, and the floor fences exactly as per-child
+// records did.
+func TestBuildFloorVersionsChildrenWithoutPerChildRecords(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil {
+		t.Fatalf("begin build: %v", err)
+	}
+	if err := mc.doBatchInsertEntries(context.Background(), []*filer.Entry{{
+		FullPath: "/dir/file",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0100644,
+			FileSize: 300,
+		},
+	}}); err != nil {
+		t.Fatalf("batch insert: %v", err)
+	}
+	if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/dir"), 2000, nil); err != nil {
+		t.Fatalf("complete build: %v", err)
+	}
+
+	// No per-child record was written...
+	if tsNs, _ := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/dir/file")); tsNs != 0 {
+		t.Fatalf("per-child version record = %d, want none (the floor versions the child)", tsNs)
+	}
+	// ...yet the child reads back at the listing snapshot.
+	if _, versionTsNs, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file")); err != nil || versionTsNs != 2000 {
+		t.Fatalf("child version = %d, %v; want 2000 from the directory floor", versionTsNs, err)
+	}
+
+	// And an event the snapshot already covered is still fenced out.
+	covered := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		TsNs:      1500,
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: "file"},
+			NewEntry:      &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{FileSize: 100, FileMode: 0100644}},
+			NewParentPath: "/dir",
+		},
+	}
+	if err := mc.ApplyMetadataResponse(context.Background(), covered, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply covered event: %v", err)
+	}
+	entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file"))
+	if err != nil || entry.FileSize != 300 {
+		t.Fatalf("entry = %+v, %v; want size 300 (floor must fence the covered event)", entry, err)
+	}
+	mc.WaitForEntryInvalidations()
+}
+
+// The presence probe applies the same TTL expiry the read path does, so a
+// logically-expired entry is judged by its directory's floor rather than by
+// the stale version record it still carries.
+func TestExpiredEntryIsJudgedByDirectoryFloor(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/dir/file",
+		Attr: filer.Attr{
+			Crtime:   time.Now().Add(-2 * time.Hour),
+			Mtime:    time.Now().Add(-2 * time.Hour),
+			Mode:     0100644,
+			FileSize: 7,
+			TtlSec:   1,
+		},
+	}, 0); err != nil {
+		t.Fatalf("insert expiring entry: %v", err)
+	}
+
+	mc.Lock()
+	// A record newer than the directory floor: it only governs while the entry
+	// it describes still exists. Once the entry has logically expired the floor
+	// is the accurate answer, and an event above the floor must apply.
+	mc.setEntryVersionLocked(context.Background(), util.FullPath("/dir/file"), 5000)
+	mc.dirVersionFloors[util.FullPath("/dir")] = 3000
+	blocks := mc.entryVersionBlocksLocked(context.Background(), util.FullPath("/dir/file"), 4000, false)
+	mc.Unlock()
+
+	if blocks {
+		t.Fatal("event at 4000 fenced by an expired entry's stale record at 5000; the floor at 3000 should govern")
+	}
+}
+
+// An unversioned local write leaves content no log position describes, so it
+// must not inherit the directory's listing floor — that would fence the very
+// events needed to correct it.
+func TestUnversionedWriteDoesNotInheritDirectoryFloor(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil {
+		t.Fatalf("begin build: %v", err)
+	}
+	if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/dir"), 2000, nil); err != nil {
+		t.Fatalf("complete build: %v", err)
+	}
+
+	// A local write with no log position behind it.
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/dir/file",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0100644,
+			FileSize: 7,
+		},
+	}, 0); err != nil {
+		t.Fatalf("local insert: %v", err)
+	}
+
+	if _, versionTsNs, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file")); err != nil || versionTsNs != 0 {
+		t.Fatalf("version = %d, %v; want 0 (unversioned content must not inherit the floor at 2000)", versionTsNs, err)
+	}
+
+	// And an event below the floor can still correct it.
+	correcting := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		TsNs:      1500,
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: "file"},
+			NewEntry:      &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{FileSize: 42, FileMode: 0100644}},
+			NewParentPath: "/dir",
+		},
+	}
+	if err := mc.ApplyMetadataResponse(context.Background(), correcting, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply correcting event: %v", err)
+	}
+	entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file"))
+	if err != nil || entry.FileSize != 42 {
+		t.Fatalf("entry = %+v, %v; want size 42 (the floor must not fence a correction of unversioned content)", entry, err)
+	}
+	mc.WaitForEntryInvalidations()
+}

weed/mount/meta_cache/meta_cache_build_test.go

@@ -0,0 +1,635 @@
+package meta_cache
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+)
+
+type buildListStream struct {
+	responses   []*filer_pb.ListEntriesResponse
+	onFirstRecv func()
+	once        sync.Once
+	index       int
+}
+
+func (s *buildListStream) Recv() (*filer_pb.ListEntriesResponse, error) {
+	s.once.Do(func() {
+		if s.onFirstRecv != nil {
+			s.onFirstRecv()
+		}
+	})
+	if s.index >= len(s.responses) {
+		return nil, io.EOF
+	}
+	resp := s.responses[s.index]
+	s.index++
+	return resp, nil
+}
+
+func (s *buildListStream) Header() (metadata.MD, error) { return metadata.MD{}, nil }
+func (s *buildListStream) Trailer() metadata.MD         { return metadata.MD{} }
+func (s *buildListStream) CloseSend() error             { return nil }
+func (s *buildListStream) Context() context.Context     { return context.Background() }
+func (s *buildListStream) SendMsg(any) error            { return nil }
+func (s *buildListStream) RecvMsg(any) error            { return nil }
+
+type buildListClient struct {
+	filer_pb.SeaweedFilerClient
+	responses   []*filer_pb.ListEntriesResponse
+	onFirstRecv func()
+}
+
+func (c *buildListClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
+	return &buildListStream{
+		responses:   c.responses,
+		onFirstRecv: c.onFirstRecv,
+	}, nil
+}
+
+type buildFilerAccessor struct {
+	client filer_pb.SeaweedFilerClient
+}
+
+func (a *buildFilerAccessor) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+	return fn(a.client)
+}
+
+func (a *buildFilerAccessor) AdjustedUrl(*filer_pb.Location) string { return "" }
+func (a *buildFilerAccessor) GetDataCenter() string                 { return "" }
+
+func TestEnsureVisitedReplaysBufferedEventsAfterSnapshot(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/": true,
+	})
+	defer mc.Shutdown()
+
+	var applyErr error
+	accessor := &buildFilerAccessor{
+		client: &buildListClient{
+			responses: []*filer_pb.ListEntriesResponse{
+				{
+					Entry: &filer_pb.Entry{
+						Name: "base.txt",
+						Attributes: &filer_pb.FuseAttributes{
+							Crtime:   1,
+							Mtime:    1,
+							FileMode: 0100644,
+							FileSize: 3,
+						},
+					},
+					SnapshotTsNs: 100,
+				},
+			},
+			onFirstRecv: func() {
+				applyErr = mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
+					Directory: "/dir",
+					EventNotification: &filer_pb.EventNotification{
+						NewEntry: &filer_pb.Entry{
+							Name: "after.txt",
+							Attributes: &filer_pb.FuseAttributes{
+								Crtime:   2,
+								Mtime:    2,
+								FileMode: 0100644,
+								FileSize: 9,
+							},
+						},
+					},
+					TsNs: 101,
+				}, SubscriberMetadataResponseApplyOptions)
+			},
+		},
+	}
+
+	if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 0); err != nil {
+		t.Fatalf("ensure visited: %v", err)
+	}
+	if applyErr != nil {
+		t.Fatalf("apply buffered event: %v", applyErr)
+	}
+	if !mc.IsDirectoryCached(util.FullPath("/dir")) {
+		t.Fatal("directory /dir should be cached after build completes")
+	}
+
+	baseEntry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/base.txt"))
+	if err != nil {
+		t.Fatalf("find base entry: %v", err)
+	}
+	if baseEntry.FileSize != 3 {
+		t.Fatalf("base entry size = %d, want 3", baseEntry.FileSize)
+	}
+
+	afterEntry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/after.txt"))
+	if err != nil {
+		t.Fatalf("find replayed entry: %v", err)
+	}
+	if afterEntry.FileSize != 9 {
+		t.Fatalf("replayed entry size = %d, want 9", afterEntry.FileSize)
+	}
+}
+
+// TestDirectoryNotificationsSuppressedDuringBuild verifies that metadata events
+// targeting a directory under active build do NOT fire onDirectoryUpdate for
+// that directory. In production, onDirectoryUpdate can trigger
+// markDirectoryReadThrough → DeleteFolderChildren, which would wipe entries
+// that EnsureVisited already inserted mid-build.
+func TestDirectoryNotificationsSuppressedDuringBuild(t *testing.T) {
+	mc, _, notifications, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/": true,
+	})
+	defer mc.Shutdown()
+
+	// Start building /dir (simulates the beginning of EnsureVisited)
+	if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil {
+		t.Fatalf("begin build: %v", err)
+	}
+
+	// Insert an entry as EnsureVisited would during the filer listing
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/dir/existing.txt",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0100644,
+			FileSize: 100,
+		},
+	}, 0); err != nil {
+		t.Fatalf("insert entry during build: %v", err)
+	}
+
+	// Simulate multiple metadata events arriving for /dir while the build
+	// is in progress. Each event would normally call noteDirectoryUpdate,
+	// which in production can trigger markDirectoryReadThrough and wipe entries.
+	for i := 0; i < 5; i++ {
+		resp := &filer_pb.SubscribeMetadataResponse{
+			Directory: "/dir",
+			EventNotification: &filer_pb.EventNotification{
+				NewEntry: &filer_pb.Entry{
+					Name: fmt.Sprintf("new-%d.txt", i),
+					Attributes: &filer_pb.FuseAttributes{
+						Crtime:   int64(10 + i),
+						Mtime:    int64(10 + i),
+						FileMode: 0100644,
+						FileSize: uint64(i + 1),
+					},
+				},
+			},
+			TsNs: int64(200 + i),
+		}
+		if err := mc.ApplyMetadataResponse(context.Background(), resp, SubscriberMetadataResponseApplyOptions); err != nil {
+			t.Fatalf("apply event %d: %v", i, err)
+		}
+	}
+
+	// The building directory /dir must NOT have received any notifications.
+	// If it did, markDirectoryReadThrough would wipe the cache mid-build.
+	for _, p := range notifications.paths() {
+		if p == util.FullPath("/dir") {
+			t.Fatal("onDirectoryUpdate was called for /dir during build; this would cause markDirectoryReadThrough to wipe entries mid-build")
+		}
+	}
+
+	// The entry inserted during the build must still be present
+	entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/existing.txt"))
+	if err != nil {
+		t.Fatalf("entry wiped during build: %v", err)
+	}
+	if entry.FileSize != 100 {
+		t.Fatalf("entry size = %d, want 100", entry.FileSize)
+	}
+
+	// Complete the build — buffered events should be replayed
+	if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/dir"), 150, nil); err != nil {
+		t.Fatalf("complete build: %v", err)
+	}
+
+	// After build completes, the entry from the listing should still exist
+	entry, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/existing.txt"))
+	if err != nil {
+		t.Fatalf("entry lost after build completion: %v", err)
+	}
+	if entry.FileSize != 100 {
+		t.Fatalf("entry size after build = %d, want 100", entry.FileSize)
+	}
+
+	// Buffered events with TsNs > snapshotTsNs (150) should have been replayed
+	for i := 0; i < 5; i++ {
+		name := fmt.Sprintf("new-%d.txt", i)
+		e, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/"+name))
+		if err != nil {
+			t.Fatalf("replayed entry %s not found: %v", name, err)
+		}
+		if e.FileSize != uint64(i+1) {
+			t.Fatalf("replayed entry %s size = %d, want %d", name, e.FileSize, i+1)
+		}
+	}
+}
+
+// TestEmptyDirectoryBuildReplaysAllBufferedEvents verifies that when a
+// directory build completes with snapshotTsNs=0 (empty directory — server
+// returned no entries and no snapshot), ALL buffered events are replayed
+// without any TsNs filtering. This prevents clock-skew between client and
+// filer from dropping legitimate mutations.
+func TestEmptyDirectoryBuildReplaysAllBufferedEvents(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/empty")); err != nil {
+		t.Fatalf("begin build: %v", err)
+	}
+
+	// Buffer events with a range of TsNs values — some very old, some recent.
+	// With a client-synthesized snapshot, old events could be incorrectly filtered.
+	tsValues := []int64{1, 50, 500, 5000, 50000}
+	for i, ts := range tsValues {
+		resp := &filer_pb.SubscribeMetadataResponse{
+			Directory: "/empty",
+			EventNotification: &filer_pb.EventNotification{
+				NewEntry: &filer_pb.Entry{
+					Name: fmt.Sprintf("file-%d.txt", i),
+					Attributes: &filer_pb.FuseAttributes{
+						Crtime:   ts,
+						Mtime:    ts,
+						FileMode: 0100644,
+						FileSize: uint64(i + 10),
+					},
+				},
+			},
+			TsNs: ts,
+		}
+		if err := mc.ApplyMetadataResponse(context.Background(), resp, SubscriberMetadataResponseApplyOptions); err != nil {
+			t.Fatalf("apply event %d: %v", i, err)
+		}
+	}
+
+	// Complete with snapshotTsNs=0 — simulates empty directory listing
+	if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/empty"), 0, nil); err != nil {
+		t.Fatalf("complete build: %v", err)
+	}
+
+	// Every buffered event must have been replayed, regardless of TsNs
+	for i := range tsValues {
+		name := fmt.Sprintf("file-%d.txt", i)
+		e, _, err := mc.FindEntry(context.Background(), util.FullPath("/empty/"+name))
+		if err != nil {
+			t.Fatalf("replayed entry %s not found: %v", name, err)
+		}
+		if e.FileSize != uint64(i+10) {
+			t.Fatalf("replayed entry %s size = %d, want %d", name, e.FileSize, i+10)
+		}
+	}
+
+	if !mc.IsDirectoryCached(util.FullPath("/empty")) {
+		t.Fatal("/empty should be marked cached after build completes")
+	}
+}
+
+// TestBuildCompletionSurvivesCallerCancellation verifies that once
+// CompleteDirectoryBuild is enqueued, a cancelled caller context does not
+// prevent the build from completing. The apply loop uses context.Background()
+// internally, so the operation finishes even if the caller gives up waiting.
+func TestBuildCompletionSurvivesCallerCancellation(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil {
+		t.Fatalf("begin build: %v", err)
+	}
+
+	// Insert an entry during the build (as EnsureVisited would)
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/dir/kept.txt",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0100644,
+			FileSize: 42,
+		},
+	}, 0); err != nil {
+		t.Fatalf("insert entry: %v", err)
+	}
+
+	// Buffer an event that should be replayed
+	if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name: "buffered.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   5,
+					Mtime:    5,
+					FileMode: 0100644,
+					FileSize: 77,
+				},
+			},
+		},
+		TsNs: 200,
+	}, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply event: %v", err)
+	}
+
+	// Complete with an already-cancelled context. The operation should still
+	// succeed because enqueueAndWait sets req.ctx = context.Background().
+	cancelledCtx, cancel := context.WithCancel(context.Background())
+	cancel() // cancel immediately
+
+	// CompleteDirectoryBuild may return ctx.Err() if the select picks
+	// ctx.Done() first, but the operation itself still completes in the
+	// apply loop. Poll for the observable side effect instead of using
+	// a fixed sleep.
+	_ = mc.CompleteDirectoryBuild(cancelledCtx, util.FullPath("/dir"), 100, nil)
+
+	// Poll until the build completes or a deadline elapses.
+	deadline := time.After(2 * time.Second)
+	for !mc.IsDirectoryCached(util.FullPath("/dir")) {
+		select {
+		case <-deadline:
+			t.Fatal("/dir should be cached — CompleteDirectoryBuild must have executed despite cancelled context")
+		default:
+			time.Sleep(5 * time.Millisecond)
+		}
+	}
+
+	// The pre-existing entry must survive
+	entry, _, findErr := mc.FindEntry(context.Background(), util.FullPath("/dir/kept.txt"))
+	if findErr != nil {
+		t.Fatalf("find kept entry: %v", findErr)
+	}
+	if entry.FileSize != 42 {
+		t.Fatalf("kept entry size = %d, want 42", entry.FileSize)
+	}
+
+	// The buffered event (TsNs 200 > snapshot 100) must have been replayed
+	buffered, _, findErr := mc.FindEntry(context.Background(), util.FullPath("/dir/buffered.txt"))
+	if findErr != nil {
+		t.Fatalf("find buffered entry: %v", findErr)
+	}
+	if buffered.FileSize != 77 {
+		t.Fatalf("buffered entry size = %d, want 77", buffered.FileSize)
+	}
+}
+
+func TestBufferedRenameUpdatesOtherDirectoryBeforeBuildCompletes(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/src": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/src/from.txt",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0100644,
+			FileSize: 7,
+		},
+	}, 0); err != nil {
+		t.Fatalf("insert source entry: %v", err)
+	}
+
+	if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dst")); err != nil {
+		t.Fatalf("begin build: %v", err)
+	}
+
+	renameResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/src",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{
+				Name: "from.txt",
+			},
+			NewEntry: &filer_pb.Entry{
+				Name: "to.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   2,
+					Mtime:    2,
+					FileMode: 0100644,
+					FileSize: 12,
+				},
+			},
+			NewParentPath: "/dst",
+		},
+		TsNs: 101,
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), renameResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply rename: %v", err)
+	}
+
+	oldEntry, _, err := mc.FindEntry(context.Background(), util.FullPath("/src/from.txt"))
+	if err != filer_pb.ErrNotFound {
+		t.Fatalf("find old path error = %v, want %v", err, filer_pb.ErrNotFound)
+	}
+	if oldEntry != nil {
+		t.Fatalf("old path should be removed before build completes: %+v", oldEntry)
+	}
+
+	newEntry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dst/to.txt"))
+	if err != filer_pb.ErrNotFound {
+		t.Fatalf("find buffered new path error = %v, want %v", err, filer_pb.ErrNotFound)
+	}
+	if newEntry != nil {
+		t.Fatalf("new path should stay hidden until build completes: %+v", newEntry)
+	}
+
+	if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/dst"), 100, nil); err != nil {
+		t.Fatalf("complete build: %v", err)
+	}
+
+	newEntry, _, err = mc.FindEntry(context.Background(), util.FullPath("/dst/to.txt"))
+	if err != nil {
+		t.Fatalf("find replayed new path: %v", err)
+	}
+	if newEntry.FileSize != 12 {
+		t.Fatalf("replayed new path size = %d, want 12", newEntry.FileSize)
+	}
+}
+
+// TestEnsureVisitedPreservesLocalOnlyEntry reproduces the residual coherence
+// gap behind the FUSE ConcurrentReadWrite ENOENT flake.
+//
+// A FUSE create on the writeback/deferFilerCreate path inserts the entry into
+// the local store directly (weedfs_file_mkrm.go createFile), off the metaCache
+// apply loop, before the filer holds it. A concurrent rebuild of the parent —
+// triggered when the directory falls out of cache (idle evict, hot-dir
+// read-through) — wipes the store and refills it from a filer listing that does
+// not yet include the un-flushed local create, then publishes the directory
+// authoritatively cached (markCachedFn). The local entry vanishes although the
+// client created it: lookupEntry then returns an authoritative ENOENT for it.
+func TestEnsureVisitedPreservesLocalOnlyEntry(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
+	defer mc.Shutdown()
+
+	// The mount pins the un-flushed create (open dirty handle / pending flush),
+	// keyed off the inode the entry carries so a kernel Forget that dropped the
+	// path→inode mapping cannot unpin an in-flight create.
+	mc.SetPinnedChildFn(func(e *filer.Entry) bool { return e.Attr.Inode == 42 })
+
+	// A deferred local create lands before the rebuild; /dir is not yet cached.
+	// It carries its allocated inode, as createFile's placeholder does.
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/dir/pending.txt",
+		Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1, Inode: 42},
+	}, 0); err != nil {
+		t.Fatalf("insert pending entry: %v", err)
+	}
+
+	// A concurrent rebuild lists the filer, whose snapshot pre-dates the
+	// un-flushed create, so it returns only the already-persisted sibling.
+	accessor := &buildFilerAccessor{client: &buildListClient{
+		responses: []*filer_pb.ListEntriesResponse{{
+			Entry: &filer_pb.Entry{
+				Name: "base.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    1,
+					FileMode: 0100644,
+					FileSize: 3,
+				},
+			},
+			SnapshotTsNs: 100,
+		}},
+	}}
+
+	if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 0); err != nil {
+		t.Fatalf("ensure visited: %v", err)
+	}
+	if !mc.IsDirectoryCached(util.FullPath("/dir")) {
+		t.Fatal("/dir should be cached after build completes")
+	}
+
+	// base.txt from the listing is present.
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/base.txt")); err != nil {
+		t.Fatalf("listed entry missing after build: %v", err)
+	}
+
+	// The un-flushed local create must survive the rebuild. With /dir now
+	// authoritatively cached, losing it is the file-vanishes flake.
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/pending.txt")); err != nil {
+		t.Fatalf("local-only entry lost across concurrent rebuild: %v", err)
+	}
+}
+
+// TestEnsureVisitedDropsUnpinnedStaleEntry guards the preservation's selectivity:
+// a cached child the filer listing no longer returns and that is NOT pinned must
+// still be wiped, so the rebuild can't resurrect a deleted/renamed entry.
+func TestEnsureVisitedDropsUnpinnedStaleEntry(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
+	defer mc.Shutdown()
+
+	mc.SetPinnedChildFn(func(*filer.Entry) bool { return false })
+
+	// A stale child sits in the cache; the filer no longer has it.
+	insertCacheEntry(t, mc, "/dir/stale.txt")
+
+	accessor := &buildFilerAccessor{client: &buildListClient{
+		responses: []*filer_pb.ListEntriesResponse{{
+			Entry: &filer_pb.Entry{
+				Name: "base.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    1,
+					FileMode: 0100644,
+					FileSize: 3,
+				},
+			},
+			SnapshotTsNs: 100,
+		}},
+	}}
+
+	if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 0); err != nil {
+		t.Fatalf("ensure visited: %v", err)
+	}
+	if entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/stale.txt")); err != filer_pb.ErrNotFound || entry != nil {
+		t.Fatalf("unpinned stale entry survived rebuild = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
+	}
+}
+
+// sequencedListClient returns a different ListEntries result per call (repeating
+// the last), modelling a filer that lists empty transiently then the real entries.
+type sequencedListClient struct {
+	filer_pb.SeaweedFilerClient
+	mu      sync.Mutex
+	perCall [][]*filer_pb.ListEntriesResponse
+	calls   int
+}
+
+func (c *sequencedListClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
+	c.mu.Lock()
+	idx := c.calls
+	if idx >= len(c.perCall) {
+		idx = len(c.perCall) - 1
+	}
+	c.calls++
+	resp := c.perCall[idx]
+	c.mu.Unlock()
+	return &buildListStream{responses: resp}, nil
+}
+
+// TestEnsureVisitedConfirmsTransientEmptyListing: a rebuild whose first filer
+// listing comes back empty must re-read and cache the real entries, not strand
+// the directory cached over an empty store (the ConcurrentReadWrite ENOENT flake).
+func TestEnsureVisitedConfirmsTransientEmptyListing(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
+	defer mc.Shutdown()
+
+	entry := &filer_pb.Entry{
+		Name:       "keep.txt",
+		Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 7},
+	}
+	accessor := &buildFilerAccessor{client: &sequencedListClient{
+		perCall: [][]*filer_pb.ListEntriesResponse{
+			{},                                  // first read: transient empty
+			{{Entry: entry, SnapshotTsNs: 100}}, // confirm read: the real entry
+		},
+	}}
+
+	if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 0); err != nil {
+		t.Fatalf("ensure visited: %v", err)
+	}
+	if !mc.IsDirectoryCached(util.FullPath("/dir")) {
+		t.Fatal("/dir should be cached after build completes")
+	}
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/keep.txt")); err != nil {
+		t.Fatalf("/dir/keep.txt stranded after transient empty listing: %v", err)
+	}
+}
+
+// TestEnsureVisitedCachesGenuinelyEmptyDirectory: a really-empty directory lists
+// empty on every confirm and must still end up cached.
+func TestEnsureVisitedCachesGenuinelyEmptyDirectory(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
+	defer mc.Shutdown()
+
+	client := &sequencedListClient{
+		perCall: [][]*filer_pb.ListEntriesResponse{{}}, // always empty
+	}
+	accessor := &buildFilerAccessor{client: client}
+
+	if err := EnsureVisited(mc, accessor, util.FullPath("/empty"), 0); err != nil {
+		t.Fatalf("ensure visited: %v", err)
+	}
+	if !mc.IsDirectoryCached(util.FullPath("/empty")) {
+		t.Fatal("/empty should be cached even though it has no entries")
+	}
+	// The empty result must have been confirmed, not trusted on the first read.
+	client.mu.Lock()
+	calls := client.calls
+	client.mu.Unlock()
+	if calls != emptyRebuildConfirmations+1 {
+		t.Fatalf("list calls = %d, want %d (initial + confirmations)", calls, emptyRebuildConfirmations+1)
+	}
+}

weed/mount/meta_cache/meta_cache_deadlock_test.go

@@ -0,0 +1,161 @@
+package meta_cache
+
+import (
+	"context"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// TestApplyLoopInvalidateDoesNotDeadlockWithLockHoldingEnqueuer reproduces the
+// mount hang seen in production (all readdirs stuck, FUSE waiting>0):
+//
+//   - A FUSE flush handler (flushMetadataToFiler) holds the file handle's
+//     exclusive lock in wfs.fhLockTable and then enqueues a local metadata
+//     event, waiting for the apply loop to process it.
+//   - Concurrently the apply loop processes a subscriber event whose
+//     invalidateFunc (weedfs.go) acquires the same file handle lock.
+//
+// The single-goroutine apply loop blocks on the fh lock while the fh lock
+// holder blocks on the apply loop: a permanent ABBA deadlock. Every later
+// EnsureVisited/BeginDirectoryBuild then backs up behind the dead loop, FUSE
+// handler goroutines exhaust, and the kernel queue (waiting) grows.
+//
+// The contract under test: the apply loop must never block on locks that can
+// be held by goroutines waiting on the apply loop.
+func TestApplyLoopInvalidateDoesNotDeadlockWithLockHoldingEnqueuer(t *testing.T) {
+	mapper, err := NewUidGidMapper("", "")
+	if err != nil {
+		t.Fatalf("uid/gid mapper: %v", err)
+	}
+
+	// Stand-in for wfs.fhLockTable, keyed like FileHandleId.
+	fhLockTable := util.NewLockTable[uint64]()
+	const fhKey uint64 = 42
+
+	invalidateEntered := make(chan struct{})
+	var enteredOnce sync.Once
+
+	var cachedMu sync.Mutex
+	cached := map[util.FullPath]bool{"/": true, "/dir": true}
+
+	mc := NewMetaCache(
+		filepath.Join(t.TempDir(), "meta"),
+		mapper,
+		util.FullPath("/"),
+		false,
+		func(path util.FullPath) {
+			cachedMu.Lock()
+			defer cachedMu.Unlock()
+			cached[path] = true
+		},
+		func(path util.FullPath) bool {
+			cachedMu.Lock()
+			defer cachedMu.Unlock()
+			return cached[path]
+		},
+		func(inv EntryInvalidation) {
+			// Mirrors the wfs invalidateFunc: it takes the open file
+			// handle's exclusive lock before refreshing the handle entry.
+			enteredOnce.Do(func() { close(invalidateEntered) })
+			lock := fhLockTable.AcquireLock("invalidateFunc", fhKey, util.ExclusiveLock)
+			fhLockTable.ReleaseLock(fhKey, lock)
+		},
+		func(dir util.FullPath) {},
+	)
+	defer func() {
+		// Only safe to shut down once the apply loop is unwedged.
+		mc.Shutdown()
+	}()
+
+	subscriberEvent := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name: "file.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    1,
+					FileMode: 0100644,
+				},
+			},
+		},
+	}
+	localEvent := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{Name: "file.txt"},
+			NewEntry: &filer_pb.Entry{
+				Name: "file.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    2,
+					FileMode: 0100644,
+				},
+			},
+			NewParentPath: "/dir",
+		},
+	}
+
+	// Step 1: the "flush handler" takes the fh exclusive lock
+	// (flushMetadataToFiler does this before talking to the filer).
+	flushLock := fhLockTable.AcquireLock("doFlush", fhKey, util.ExclusiveLock)
+	flushLockReleased := false
+	releaseFlushLock := func() {
+		if !flushLockReleased {
+			flushLockReleased = true
+			fhLockTable.ReleaseLock(fhKey, flushLock)
+		}
+	}
+	defer releaseFlushLock()
+
+	// Step 2: a subscriber event arrives; the apply loop will run
+	// invalidateFunc, which wants the fh lock held above.
+	subscriberDone := make(chan error, 1)
+	go func() {
+		subscriberDone <- mc.ApplyMetadataResponse(context.Background(), subscriberEvent, SubscriberMetadataResponseApplyOptions)
+	}()
+
+	select {
+	case <-invalidateEntered:
+	case <-time.After(5 * time.Second):
+		t.Fatal("invalidateFunc was never invoked for the subscriber event")
+	}
+	// Give the invalidate goroutine a moment to actually block on AcquireLock.
+	time.Sleep(50 * time.Millisecond)
+
+	// Step 3: still holding the fh lock, the flush handler applies its local
+	// metadata event and waits for the apply loop — exactly what
+	// flushMetadataToFiler does after streamCreateEntry succeeds.
+	flushApplyDone := make(chan error, 1)
+	go func() {
+		flushApplyDone <- mc.ApplyMetadataResponseOwned(context.Background(), localEvent, LocalMetadataResponseApplyOptions)
+	}()
+
+	select {
+	case err := <-flushApplyDone:
+		if err != nil {
+			t.Fatalf("local metadata apply: %v", err)
+		}
+	case <-time.After(3 * time.Second):
+		// Unwedge the apply loop so Shutdown in the deferred cleanup
+		// doesn't hang the whole test binary.
+		releaseFlushLock()
+		t.Fatal("deadlock: apply loop blocked on fh lock while fh lock holder waits on apply loop")
+	}
+
+	releaseFlushLock()
+
+	select {
+	case err := <-subscriberDone:
+		if err != nil {
+			t.Fatalf("subscriber apply: %v", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("subscriber apply never completed")
+	}
+}

weed/mount/meta_cache/meta_cache_init.go

@@ -0,0 +1,230 @@
+package meta_cache
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// DirectoryTooLargeError reports a directory the mount refuses to cache
+// locally. Its listings read through to the filer instead.
+type DirectoryTooLargeError struct {
+	Path util.FullPath
+}
+
+func (e *DirectoryTooLargeError) Error() string {
+	return fmt.Sprintf("directory %s is too large to cache locally", e.Path)
+}
+
+// maxCacheableEntries is the directory size above which a build gives up, or 0
+// to cache everything.
+func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.FullPath, maxCacheableEntries int) error {
+	// Collect all uncached paths from target directory up to root
+	var uncachedPaths []util.FullPath
+	currentPath := dirPath
+
+	for {
+		// If this path is cached, all ancestors are also cached
+		if mc.isCachedFn(currentPath) {
+			break
+		}
+		if mc.isOversized(currentPath) {
+			// The directory itself reads through; an ancestor is stepped over,
+			// or it would wedge every listing beneath it forever.
+			if currentPath == dirPath {
+				return &DirectoryTooLargeError{Path: currentPath}
+			}
+		} else {
+			uncachedPaths = append(uncachedPaths, currentPath)
+		}
+
+		// Continue to parent directory
+		if currentPath != mc.root {
+			parent, _ := currentPath.DirAndName()
+			currentPath = util.FullPath(parent)
+		} else {
+			break
+		}
+	}
+
+	if len(uncachedPaths) == 0 {
+		return nil
+	}
+
+	// Fetch all uncached directories in parallel with context for cancellation
+	// If one fetch fails, cancel the others to avoid unnecessary work
+	g, ctx := errgroup.WithContext(context.Background())
+	for _, p := range uncachedPaths {
+		path := p // capture for closure
+		g.Go(func() error {
+			err := doEnsureVisited(ctx, mc, client, path, maxCacheableEntries)
+			var tooLarge *DirectoryTooLargeError
+			if errors.As(err, &tooLarge) && path != dirPath {
+				// An ancestor found oversized just reads through; failing the
+				// group here would cancel the builds of its cacheable
+				// descendants, and the caller would treat the refusal as the
+				// listed directory's own.
+				return nil
+			}
+			return err
+		})
+	}
+	return g.Wait()
+}
+
+// batchInsertSize is the number of entries to accumulate before flushing to LevelDB.
+// 100 provides a balance between memory usage (~100 Entry pointers) and write efficiency
+// (fewer disk syncs). Larger values reduce I/O overhead but increase memory and latency.
+const batchInsertSize = 100
+
+const (
+	emptyRebuildConfirmations = 2
+	emptyRebuildConfirmDelay  = 50 * time.Millisecond
+)
+
+func doEnsureVisited(ctx context.Context, mc *MetaCache, client filer_pb.FilerClient, path util.FullPath, maxCacheableEntries int) error {
+	// Use singleflight to deduplicate concurrent requests for the same path
+	_, err, _ := mc.visitGroup.Do(string(path), func() (interface{}, error) {
+		// Check for cancellation before starting
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+
+		// Double-check if already cached (another goroutine may have completed)
+		if mc.isCachedFn(path) {
+			return nil, nil
+		}
+
+		glog.V(4).Infof("ReadDirAllEntries %s ...", path)
+
+		// Use context.Background() for build lifecycle calls so that
+		// errgroup cancellation of ctx doesn't cause enqueueAndWait to
+		// return early, which would trigger cleanupBuild while the
+		// operation is still queued.
+		if err := mc.BeginDirectoryBuild(context.Background(), path); err != nil {
+			return nil, fmt.Errorf("begin build %s: %w", path, err)
+		}
+		cleanupDone := false
+		cleanupBuild := func(reason string) {
+			if cleanupDone {
+				return
+			}
+			cleanupDone = true
+			if deleteErr := mc.deleteFolderChildrenForRebuild(context.Background(), path); deleteErr != nil {
+				glog.V(2).Infof("clear %s build %s: %v", reason, path, deleteErr)
+			}
+			if abortErr := mc.AbortDirectoryBuild(context.Background(), path); abortErr != nil {
+				glog.V(2).Infof("abort %s build %s: %v", reason, path, abortErr)
+			}
+		}
+		defer func() {
+			if !cleanupDone && ctx.Err() != nil {
+				cleanupBuild("canceled")
+			}
+		}()
+
+		// reloadFromFiler wipes the cached children and reloads them from the filer.
+		reloadFromFiler := func() (entryCount int, snapshotTsNs int64, sections sectionBoundsCollector, err error) {
+			err = util.Retry("ReadDirAllEntries", func() error {
+				entryCount = 0
+				sections = sectionBoundsCollector{}
+				var batch []*filer.Entry // reset on retry, allow GC of previous entries
+				if err := mc.deleteFolderChildrenForRebuild(ctx, path); err != nil {
+					return fmt.Errorf("clear existing entries for %s: %w", path, err)
+				}
+				var listErr error
+				snapshotTsNs, listErr = filer_pb.ReadDirAllEntriesWithSnapshot(ctx, client, path, "", func(pbEntry *filer_pb.Entry, isLast bool) error {
+					entry := filer.FromPbEntry(string(path), pbEntry)
+					if !mc.includeSystemEntries && IsHiddenSystemEntry(string(path), entry.Name()) {
+						return nil
+					}
+
+					if maxCacheableEntries > 0 && entryCount >= maxCacheableEntries {
+						return &DirectoryTooLargeError{Path: path}
+					}
+					sections.note(entry.Name())
+					batch = append(batch, entry)
+					entryCount++
+
+					// flush by size, not isLast: hidden entries can return early
+					if len(batch) >= batchInsertSize {
+						if err := mc.doBatchInsertEntries(ctx, batch); err != nil {
+							return fmt.Errorf("batch insert for %s: %w", path, err)
+						}
+						batch = make([]*filer.Entry, 0, batchInsertSize)
+					}
+					return nil
+				})
+				if listErr != nil {
+					return listErr
+				}
+				if len(batch) > 0 {
+					if err := mc.doBatchInsertEntries(ctx, batch); err != nil {
+						return fmt.Errorf("batch insert remaining for %s: %w", path, err)
+					}
+				}
+				return nil
+			})
+			return entryCount, snapshotTsNs, sections, err
+		}
+
+		entryCount, snapshotTsNs, sections, fetchErr := reloadFromFiler()
+		if fetchErr != nil {
+			var tooLarge *DirectoryTooLargeError
+			if errors.As(fetchErr, &tooLarge) {
+				// Remember the refusal so the next visit fails fast instead of
+				// streaming up to the limit again to rediscover it.
+				mc.markOversized(path)
+				glog.V(0).Infof("directory %s exceeds %d entries, reading it through instead of caching", path, maxCacheableEntries)
+				cleanupBuild("oversized")
+				return nil, fetchErr
+			}
+			cleanupBuild("failed")
+			return nil, fmt.Errorf("list %s: %w", path, fetchErr)
+		}
+
+		// A transient empty listing would strand a populated directory cached over
+		// an empty store; re-read to confirm before trusting it. First re-read is
+		// immediate (a clean-EOF stream glitch clears at once), later ones space out.
+		// On cancellation the deferred cleanup aborts the build.
+		for attempt := 0; entryCount == 0 && attempt < emptyRebuildConfirmations; attempt++ {
+			if ctx.Err() != nil {
+				return nil, ctx.Err()
+			}
+			if attempt > 0 {
+				select {
+				case <-time.After(emptyRebuildConfirmDelay):
+				case <-ctx.Done():
+					return nil, ctx.Err()
+				}
+			}
+			if entryCount, snapshotTsNs, sections, fetchErr = reloadFromFiler(); fetchErr != nil {
+				cleanupBuild("failed")
+				return nil, fmt.Errorf("confirm empty list %s: %w", path, fetchErr)
+			}
+			if entryCount > 0 {
+				glog.Warningf("rebuild of %s saw a transient empty listing, recovered %d entries on confirmation", path, entryCount)
+			}
+		}
+
+		if err := mc.CompleteDirectoryBuild(context.Background(), path, snapshotTsNs, sections.bounds); err != nil {
+			cleanupBuild("unreplayed")
+			return nil, fmt.Errorf("complete build for %s: %w", path, err)
+		}
+		cleanupDone = true // Prevent deferred cleanup after successful publish
+		return nil, nil
+	})
+	return err
+}
+
+func IsHiddenSystemEntry(dir, name string) bool {
+	return dir == "/" && (name == "topics" || name == "etc")
+}

weed/mount/meta_cache/meta_cache_purge_test.go

@@ -0,0 +1,98 @@
+package meta_cache
+
+import (
+	"context"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func insertCacheEntry(t *testing.T, mc *MetaCache, path util.FullPath) {
+	t.Helper()
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: path,
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0100644,
+			FileSize: 1,
+		},
+	}, 0); err != nil {
+		t.Fatalf("insert %s: %v", path, err)
+	}
+}
+
+// barrier flushes the apply loop: enqueued before it, PurgeDirectoryChildren is
+// asynchronous, so a synchronous apply-loop call afterward guarantees the purge
+// has been processed (the loop is FIFO and single-threaded).
+func barrier(t *testing.T, mc *MetaCache) {
+	t.Helper()
+	if err := mc.AbortDirectoryBuild(context.Background(), util.FullPath("/__barrier__")); err != nil {
+		t.Fatalf("barrier: %v", err)
+	}
+}
+
+// TestPurgeSkippedWhileDirectoryBuilding is the core regression guard: a purge
+// (idle eviction / kernel Forget / copy-range fallback) that lands while a
+// directory is being rebuilt must NOT wipe the entries the build just inserted.
+// Otherwise CompleteDirectoryBuild marks the directory cached over an empty
+// store, and every file in it vanishes from the mount though it is safe on the
+// filer.
+func TestPurgeSkippedWhileDirectoryBuilding(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
+	defer mc.Shutdown()
+
+	dir := util.FullPath("/dir")
+	if err := mc.BeginDirectoryBuild(context.Background(), dir); err != nil {
+		t.Fatalf("begin build: %v", err)
+	}
+	insertCacheEntry(t, mc, "/dir/a.txt")
+	insertCacheEntry(t, mc, "/dir/b.txt")
+
+	// A concurrent eviction tries to purge the directory mid-build. It is
+	// enqueued before CompleteDirectoryBuild, so the apply loop processes it
+	// first — while /dir is still building.
+	var resetCalls int32
+	mc.PurgeDirectoryChildren(dir, func() { atomic.AddInt32(&resetCalls, 1) })
+
+	if err := mc.CompleteDirectoryBuild(context.Background(), dir, 0, nil); err != nil {
+		t.Fatalf("complete build: %v", err)
+	}
+
+	if got := atomic.LoadInt32(&resetCalls); got != 0 {
+		t.Fatalf("resetFn ran %d times during build; purge must be skipped", got)
+	}
+	if !mc.IsDirectoryCached(dir) {
+		t.Fatal("/dir should be cached after build completes")
+	}
+	for _, name := range []string{"/dir/a.txt", "/dir/b.txt"} {
+		if _, _, err := mc.FindEntry(context.Background(), util.FullPath(name)); err != nil {
+			t.Fatalf("%s missing after mid-build purge: %v", name, err)
+		}
+	}
+}
+
+// TestPurgeClearsWhenNotBuilding verifies the eviction path still works off the
+// apply loop: with no build in flight, the purge resets the cached flag and
+// wipes the store.
+func TestPurgeClearsWhenNotBuilding(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true, "/dir": true})
+	defer mc.Shutdown()
+
+	dir := util.FullPath("/dir")
+	insertCacheEntry(t, mc, "/dir/a.txt")
+
+	var resetCalls int32
+	mc.PurgeDirectoryChildren(dir, func() { atomic.AddInt32(&resetCalls, 1) })
+	barrier(t, mc)
+
+	if got := atomic.LoadInt32(&resetCalls); got != 1 {
+		t.Fatalf("resetFn ran %d times; want 1", got)
+	}
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/a.txt")); err == nil {
+		t.Fatal("/dir/a.txt should have been purged")
+	}
+}

weed/mount/meta_cache/meta_cache_rename_gate_test.go

@@ -0,0 +1,95 @@
+package meta_cache
+
+import (
+	"context"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// A rename stamps its source and takes the name away at the same log position,
+// so the removal arrives at exactly the version the source already records.
+// Read as a write that is already reflected, it was dropped and the source
+// stayed visible next to the destination.
+func TestRenameRemovesSourceAtItsOwnVersion(t *testing.T) {
+	const version = 1787761708173717200
+
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true, "/dir": true})
+	defer mc.Shutdown()
+	ctx := context.Background()
+
+	write := func(name string, tsNs int64) *filer_pb.SubscribeMetadataResponse {
+		return &filer_pb.SubscribeMetadataResponse{
+			Directory: "/dir",
+			EventNotification: &filer_pb.EventNotification{
+				NewEntry: &filer_pb.Entry{
+					Name:       name,
+					Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 3},
+				},
+			},
+			TsNs: tsNs,
+		}
+	}
+	for _, event := range []*filer_pb.SubscribeMetadataResponse{write("src", version), write("dst", version-1000)} {
+		if err := mc.ApplyMetadataResponseOwned(ctx, event, LocalMetadataResponseApplyOptions); err != nil {
+			t.Fatalf("seed %s: %v", event.EventNotification.NewEntry.Name, err)
+		}
+	}
+
+	rename := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{Name: "src"},
+			NewEntry: &filer_pb.Entry{
+				Name:       "dst",
+				Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 3},
+			},
+			NewParentPath: "/dir",
+		},
+		TsNs: version,
+	}
+	if err := mc.ApplyMetadataResponseOwned(ctx, rename, LocalMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("rename: %v", err)
+	}
+
+	if entry, _, err := mc.FindEntry(ctx, "/dir/src"); err == nil && entry != nil {
+		t.Error("the renamed-away source is still cached")
+	}
+	if entry, _, err := mc.FindEntry(ctx, "/dir/dst"); err != nil || entry == nil {
+		t.Errorf("the rename destination is missing: %v", err)
+	}
+}
+
+// A removal older than the entry's record is still stale and must not apply:
+// the record describes a write the removal never saw.
+func TestRemovalOlderThanTheRecordStillFences(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true, "/dir": true})
+	defer mc.Shutdown()
+	ctx := context.Background()
+
+	if err := mc.ApplyMetadataResponseOwned(ctx, &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name:       "file",
+				Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 3},
+			},
+		},
+		TsNs: 2000,
+	}, LocalMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("seed: %v", err)
+	}
+
+	if err := mc.ApplyMetadataResponseOwned(ctx, &filer_pb.SubscribeMetadataResponse{
+		Directory:         "/dir",
+		EventNotification: &filer_pb.EventNotification{OldEntry: &filer_pb.Entry{Name: "file"}},
+		TsNs:              1000,
+	}, LocalMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("stale removal: %v", err)
+	}
+
+	if entry, _, err := mc.FindEntry(ctx, "/dir/file"); err != nil || entry == nil {
+		t.Errorf("a removal older than the entry's record deleted it: %v", err)
+	}
+}

weed/mount/meta_cache/meta_cache_sections.go

@@ -0,0 +1,422 @@
+package meta_cache
+
+import (
+	"context"
+	"errors"
+	"math"
+	"sort"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// A cached directory's listing is split into contiguous name-range sections so
+// a burst of remote changes invalidates one section, not the whole listing.
+// Events keep applying to a stale section; staleness only means the next
+// listing re-validates that range against the filer before serving it, and
+// lookups in it read through until then.
+const (
+	// dirSectionSize is the target entries per section, fixed when a listing
+	// is built and re-derived when a re-listed section has outgrown it.
+	dirSectionSize = 1024
+	// sectionHotThreshold remote changes within sectionHotWindow invalidate
+	// the section they land in.
+	sectionHotThreshold = 64
+	sectionHotWindow    = 2 * time.Second
+	// sectionRefreshTimeout bounds how long a readdir waits on re-validating
+	// a section before serving the maintained-but-unverified cache instead.
+	sectionRefreshTimeout = 5 * time.Second
+	// sectionRefreshMaxEntries is the most one refresh will carry; a section
+	// grown past it is cheaper to re-tile with a full directory rebuild.
+	sectionRefreshMaxEntries = 4 * dirSectionSize
+)
+
+// ErrRefreshRangeTooLarge reports a section that outgrew one refresh; the
+// caller should drop the directory cache so a full rebuild re-tiles it.
+var ErrRefreshRangeTooLarge = errors.New("section outgrew one refresh")
+
+// sectionList: bounds[i] is the first name of section i+1; section 0 starts at
+// the beginning of the namespace, the last section runs to the end. It is a
+// plain state machine — no locking, no store; MetaCache drives it under its
+// own mutex.
+type sectionList struct {
+	bounds   []string
+	sections []sectionState
+}
+
+type sectionState struct {
+	stale       bool
+	updateCount int
+	windowStart time.Time
+	// unverifiable marks a stale section whose filer stamps no listing
+	// snapshots: re-listing it can never vouch for it, so stop trying and
+	// leave its lookups reading through.
+	unverifiable bool
+	// floorTsNs is the section's own listing snapshot: a refresh at it covered
+	// every name in the range, present or absent, so it fences like the
+	// directory floor but for this range alone.
+	floorTsNs int64
+}
+
+func newSectionTable(bounds []string) *sectionList {
+	return &sectionList{bounds: bounds, sections: make([]sectionState, len(bounds)+1)}
+}
+
+// sectionBoundsCollector derives section boundaries from an ordered listing:
+// every dirSectionSize-th name starts a new section.
+type sectionBoundsCollector struct {
+	count  int
+	bounds []string
+}
+
+func (c *sectionBoundsCollector) note(name string) {
+	if c.count > 0 && c.count%dirSectionSize == 0 {
+		c.bounds = append(c.bounds, name)
+	}
+	c.count++
+}
+
+// sectionRefresh carries one section's re-listing to the apply loop.
+type sectionRefresh struct {
+	lo, hi       string
+	entries      []*filer.Entry
+	snapshotTsNs int64
+}
+
+func (sl *sectionList) sectionOf(name string) int {
+	idx := sort.SearchStrings(sl.bounds, name)
+	if idx < len(sl.bounds) && sl.bounds[idx] == name {
+		idx++
+	}
+	return idx
+}
+
+// sectionRange returns the [lo, hi) name range of section idx; "" is unbounded.
+func (sl *sectionList) sectionRange(idx int) (lo, hi string) {
+	if idx > 0 {
+		lo = sl.bounds[idx-1]
+	}
+	if idx < len(sl.bounds) {
+		hi = sl.bounds[idx]
+	}
+	return
+}
+
+// noteChange counts one change against the section it lands in, marking the
+// section stale when a burst crosses the threshold.
+func (sl *sectionList) noteChange(name string, now time.Time) {
+	s := &sl.sections[sl.sectionOf(name)]
+	if s.stale {
+		return
+	}
+	if s.windowStart.IsZero() || now.Sub(s.windowStart) > sectionHotWindow {
+		s.windowStart = now
+		s.updateCount = 0
+	}
+	s.updateCount++
+	if s.updateCount >= sectionHotThreshold {
+		s.stale = true
+	}
+}
+
+func (sl *sectionList) isFresh(name string) bool {
+	return !sl.sections[sl.sectionOf(name)].stale
+}
+
+// floorOf returns the refresh snapshot covering this name, or zero when its
+// section has never been re-listed. A stale section keeps fencing: what its
+// last listing established stays established.
+func (sl *sectionList) floorOf(name string) int64 {
+	return sl.sections[sl.sectionOf(name)].floorTsNs
+}
+
+type nameRange struct {
+	lo, hi string
+}
+
+// staleRangesAhead returns the invalidated ranges worth re-listing from the
+// section holding startName to the end of the directory.
+func (sl *sectionList) staleRangesAhead(startName string) (ranges []nameRange) {
+	for i := sl.sectionOf(startName); i < len(sl.sections); i++ {
+		if sl.sections[i].stale && !sl.sections[i].unverifiable {
+			lo, hi := sl.sectionRange(i)
+			ranges = append(ranges, nameRange{lo: lo, hi: hi})
+		}
+	}
+	return
+}
+
+// hasRange reports whether the table still has a section covering exactly
+// [lo, hi); a rebuild or re-split since a listing was taken retires the range
+// it described.
+func (sl *sectionList) hasRange(lo, hi string) bool {
+	curLo, curHi := sl.sectionRange(sl.sectionOf(lo))
+	return curLo == lo && curHi == hi
+}
+
+// completeRefresh marks the section covering exactly [lo, hi) fresh after a
+// re-listing that fetched names at snapshotTsNs, re-splitting a section that
+// outgrew twice its target size. The snapshot becomes the section's floor. A
+// range the table no longer has is ignored — splicing bounds from a stale
+// range could leave the table unsorted — and an unversioned listing vouches
+// for nothing: the section stays stale, remembered as not worth re-listing.
+func (sl *sectionList) completeRefresh(lo, hi string, names []string, snapshotTsNs int64) bool {
+	idx := sl.sectionOf(lo)
+	if curLo, curHi := sl.sectionRange(idx); curLo != lo || curHi != hi {
+		return false
+	}
+	if snapshotTsNs == 0 {
+		sl.sections[idx].unverifiable = true
+		return false
+	}
+	if len(names) > 2*dirSectionSize {
+		var newBounds []string
+		for i := dirSectionSize; i < len(names); i += dirSectionSize {
+			newBounds = append(newBounds, names[i])
+		}
+		bounds := make([]string, 0, len(sl.bounds)+len(newBounds))
+		bounds = append(bounds, sl.bounds[:idx]...)
+		bounds = append(bounds, newBounds...)
+		bounds = append(bounds, sl.bounds[idx:]...)
+		sections := make([]sectionState, 0, len(bounds)+1)
+		sections = append(sections, sl.sections[:idx]...)
+		for i := 0; i <= len(newBounds); i++ {
+			sections = append(sections, sectionState{floorTsNs: snapshotTsNs})
+		}
+		sections = append(sections, sl.sections[idx+1:]...)
+		sl.bounds, sl.sections = bounds, sections
+	} else {
+		sl.sections[idx] = sectionState{floorTsNs: snapshotTsNs}
+	}
+	return true
+}
+
+// noteSectionChangeLocked counts one remote change against the section of the
+// directory it lands in.
+func (mc *MetaCache) noteSectionChangeLocked(fp util.FullPath, now time.Time) {
+	dir, name := fp.DirAndName()
+	if sl := mc.dirSections[util.FullPath(dir)]; sl != nil {
+		sl.noteChange(name, now)
+	}
+}
+
+// IsNameFresh reports whether the cached listing still vouches for this name.
+// A directory without section state vouches for all of it.
+func (mc *MetaCache) IsNameFresh(fp util.FullPath) bool {
+	dir, name := fp.DirAndName()
+	mc.RLock()
+	defer mc.RUnlock()
+	sl := mc.dirSections[util.FullPath(dir)]
+	return sl == nil || sl.isFresh(name)
+}
+
+func (mc *MetaCache) staleRangesAhead(dirPath util.FullPath, startName string) []nameRange {
+	mc.RLock()
+	defer mc.RUnlock()
+	sl := mc.dirSections[dirPath]
+	if sl == nil {
+		return nil
+	}
+	return sl.staleRangesAhead(startName)
+}
+
+func (mc *MetaCache) rangeStale(dirPath util.FullPath, lo string) bool {
+	mc.RLock()
+	defer mc.RUnlock()
+	sl := mc.dirSections[dirPath]
+	return sl != nil && !sl.isFresh(lo)
+}
+
+// EnsureListingFresh re-validates every invalidated section from startName to
+// the end of the directory before a listing pages through them. A listing's
+// reach is unknowable up front — a resumed handle can skip far ahead, and
+// shrunken sections let one batch span many — so all of them are covered.
+func EnsureListingFresh(ctx context.Context, mc *MetaCache, client filer_pb.FilerClient, dirPath util.FullPath, startName string) error {
+	ranges := mc.staleRangesAhead(dirPath, startName)
+	if len(ranges) == 0 {
+		return nil
+	}
+	ctx, cancel := context.WithTimeout(ctx, sectionRefreshTimeout)
+	defer cancel()
+	for _, r := range ranges {
+		if err := mc.refreshSection(ctx, client, dirPath, r.lo, r.hi); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (mc *MetaCache) refreshSection(ctx context.Context, client filer_pb.FilerClient, dirPath util.FullPath, lo, hi string) error {
+	_, err, _ := mc.visitGroup.Do(string(dirPath)+"\x00section\x00"+lo, func() (interface{}, error) {
+		if !mc.rangeStale(dirPath, lo) {
+			return nil, nil
+		}
+		entries, snapshotTsNs, err := mc.listFilerRange(ctx, client, dirPath, lo, hi)
+		if err != nil {
+			return nil, err
+		}
+		return nil, mc.enqueueAndWait(ctx, metadataApplyRequest{
+			kind:      metadataSectionRefresh,
+			buildPath: dirPath,
+			refresh:   &sectionRefresh{lo: lo, hi: hi, entries: entries, snapshotTsNs: snapshotTsNs},
+		})
+	})
+	return err
+}
+
+// listFilerRange reads [lo, hi) from the filer at one snapshot, paging by
+// section-sized batches.
+func (mc *MetaCache) listFilerRange(ctx context.Context, client filer_pb.FilerClient, dirPath util.FullPath, lo, hi string) (entries []*filer.Entry, snapshotTsNs int64, err error) {
+	startFrom, includeStart := lo, lo != ""
+	for {
+		var page []*filer.Entry
+		var pageCount int
+		var last string
+		done := false
+		err = client.WithFilerClient(false, func(sc filer_pb.SeaweedFilerClient) error {
+			// reset in case a failover retry re-runs a partly streamed page
+			page, pageCount, last, done = nil, 0, "", false
+			ts, listErr := filer_pb.DoSeaweedListWithSnapshot(ctx, sc, dirPath, "", func(pbEntry *filer_pb.Entry, isLast bool) error {
+				pageCount++
+				last = pbEntry.Name
+				if hi != "" && pbEntry.Name >= hi {
+					done = true
+				}
+				if done {
+					return nil
+				}
+				if !mc.includeSystemEntries && IsHiddenSystemEntry(string(dirPath), pbEntry.Name) {
+					return nil
+				}
+				page = append(page, filer.FromPbEntry(string(dirPath), pbEntry))
+				return nil
+			}, startFrom, includeStart, dirSectionSize, snapshotTsNs)
+			if listErr != nil {
+				return listErr
+			}
+			if snapshotTsNs == 0 {
+				snapshotTsNs = ts
+			}
+			return nil
+		})
+		if err != nil {
+			return nil, 0, err
+		}
+		entries = append(entries, page...)
+		if len(entries) > sectionRefreshMaxEntries {
+			return nil, 0, ErrRefreshRangeTooLarge
+		}
+		if done || pageCount < dirSectionSize {
+			return entries, snapshotTsNs, nil
+		}
+		startFrom, includeStart = last, false
+	}
+}
+
+// applySectionRefreshNow reconciles one section against a filer listing of its
+// range, then marks it fresh. Runs on the apply loop; mutations go through the
+// version gate so the listing cannot roll back a newer applied event, and
+// pinned local-only entries (deferred creates not yet on the filer) survive.
+func (mc *MetaCache) applySectionRefreshNow(ctx context.Context, dirPath util.FullPath, r *sectionRefresh) error {
+	lo, hi, snapshotTsNs := r.lo, r.hi, r.snapshotTsNs
+
+	// A build wipes and repopulates the store off-loop; reconciling against
+	// it would sweep children the build already inserted and publish the
+	// directory incomplete. The staleness dies with the build's fresh table.
+	if mc.isBuildingDir(dirPath) {
+		return nil
+	}
+
+	mc.Lock()
+	defer mc.Unlock()
+
+	// With no per-entry versions, only the section floor fences this work; a
+	// range the rebuilt or re-split table no longer has gets no floor, so it
+	// must not touch the store either. The lock is held through the floor
+	// install below, so the check cannot go stale.
+	sl := mc.dirSections[dirPath]
+	if sl == nil || !sl.hasRange(lo, hi) {
+		return nil
+	}
+
+	fetchedNames := make([]string, 0, len(r.entries))
+	fetched := make(map[string]struct{}, len(r.entries))
+	for _, entry := range r.entries {
+		fetchedNames = append(fetchedNames, entry.Name())
+		fetched[entry.Name()] = struct{}{}
+		if snapshotTsNs == 0 {
+			// A pre-upgrade filer stamps no snapshot, leaving nothing to
+			// order against: only fill gaps, so a concurrently applied event
+			// can never be rolled back.
+			if mc.entryExistsLocked(ctx, entry.FullPath) {
+				continue
+			}
+			if _, tombstone := mc.getEntryVersionRecordLocked(ctx, entry.FullPath); tombstone {
+				continue
+			}
+			if err := mc.localStore.InsertEntry(ctx, entry); err != nil {
+				return err
+			}
+			mc.setEntryVersionLocked(ctx, entry.FullPath, 0)
+			continue
+		}
+		if mc.entryVersionBlocksLocked(ctx, entry.FullPath, snapshotTsNs, false) {
+			continue
+		}
+		// An unversioned marker would bypass the section floor, so it cannot
+		// outlive the snapshot write that replaces its content — but pinned
+		// local-only state stays authoritative and is not replaced at all.
+		_, _, unversioned := mc.entryVersionRecordLocked(ctx, entry.FullPath)
+		if unversioned {
+			if existing, findErr := mc.localStore.FindEntry(ctx, entry.FullPath); findErr == nil && existing != nil && mc.pinnedChildFn != nil && mc.pinnedChildFn(existing) {
+				continue
+			}
+		}
+		// no per-entry version: the section floor set below covers the range
+		if err := mc.localStore.InsertEntry(ctx, entry); err != nil {
+			return err
+		}
+		if unversioned {
+			// only once the write landed: if it fails, the old content keeps
+			// the marker, and with it the right to be corrected by any event
+			mc.clearEntryVersionLocked(ctx, entry.FullPath)
+		}
+	}
+
+	// Deletions need the snapshot as an ordering reference; without one a
+	// name created after the listing would be swept away.
+	if snapshotTsNs != 0 {
+		var vanished []*filer.Entry
+		if _, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, lo, true, math.MaxInt64, func(entry *filer.Entry) (bool, error) {
+			if hi != "" && entry.Name() >= hi {
+				return false, nil
+			}
+			if _, found := fetched[entry.Name()]; !found {
+				vanished = append(vanished, entry)
+			}
+			return true, nil
+		}); err != nil {
+			return err
+		}
+		for _, entry := range vanished {
+			if mc.pinnedChildFn != nil && mc.pinnedChildFn(entry) {
+				continue
+			}
+			if mc.entryVersionBlocksLocked(ctx, entry.FullPath, snapshotTsNs, true) {
+				continue
+			}
+			if err := mc.localStore.DeleteEntry(ctx, entry.FullPath); err != nil {
+				return err
+			}
+			mc.clearEntryVersionLocked(ctx, entry.FullPath)
+		}
+	}
+
+	// The floor fences the whole range, absent names included; an unversioned
+	// listing sets none and the section stays stale, its lookups reading
+	// through, with no further re-listing attempts.
+	sl.completeRefresh(lo, hi, fetchedNames, snapshotTsNs)
+	return nil
+}

weed/mount/meta_cache/meta_cache_sections_test.go

@@ -0,0 +1,974 @@
+package meta_cache
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"sync"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestSectionOf(t *testing.T) {
+	sl := newSectionTable([]string{"g", "p"})
+	for name, want := range map[string]int{
+		"a": 0, "f": 0,
+		"g": 1, "h": 1, "o": 1,
+		"p": 2, "z": 2,
+	} {
+		if got := sl.sectionOf(name); got != want {
+			t.Errorf("sectionOf(%q) = %d, want %d", name, got, want)
+		}
+	}
+	if lo, hi := sl.sectionRange(0); lo != "" || hi != "g" {
+		t.Errorf("sectionRange(0) = %q..%q, want ..g", lo, hi)
+	}
+	if lo, hi := sl.sectionRange(1); lo != "g" || hi != "p" {
+		t.Errorf("sectionRange(1) = %q..%q, want g..p", lo, hi)
+	}
+	if lo, hi := sl.sectionRange(2); lo != "p" || hi != "" {
+		t.Errorf("sectionRange(2) = %q..%q, want p..", lo, hi)
+	}
+}
+
+func buildSectionedDir(t *testing.T, mc *MetaCache, dir util.FullPath, snapshotTsNs int64, bounds []string) {
+	t.Helper()
+	if err := mc.BeginDirectoryBuild(context.Background(), dir); err != nil {
+		t.Fatalf("begin build: %v", err)
+	}
+	if err := mc.CompleteDirectoryBuild(context.Background(), dir, snapshotTsNs, bounds); err != nil {
+		t.Fatalf("complete build: %v", err)
+	}
+}
+
+func applyChurn(t *testing.T, mc *MetaCache, dir, prefix string, n int, baseTsNs int64, options MetadataResponseApplyOptions) {
+	t.Helper()
+	for i := 0; i < n; i++ {
+		resp := &filer_pb.SubscribeMetadataResponse{
+			Directory: dir,
+			EventNotification: &filer_pb.EventNotification{
+				NewEntry: &filer_pb.Entry{
+					Name: fmt.Sprintf("%s%03d", prefix, i),
+					Attributes: &filer_pb.FuseAttributes{
+						Crtime:   1,
+						Mtime:    1,
+						FileMode: 0100644,
+						FileSize: 1,
+					},
+				},
+			},
+			TsNs: baseTsNs + int64(i),
+		}
+		if err := mc.ApplyMetadataResponse(context.Background(), resp, options); err != nil {
+			t.Fatalf("apply churn %d: %v", i, err)
+		}
+	}
+}
+
+func TestSectionBurstInvalidatesOnlyItsSection(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})
+
+	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
+
+	if mc.IsNameFresh(util.FullPath("/dir/a-000")) {
+		t.Fatal("burst section should be invalidated")
+	}
+	if !mc.IsNameFresh(util.FullPath("/dir/zzz")) {
+		t.Fatal("the other section must stay fresh")
+	}
+}
+
+func TestLocalChangesDoNotInvalidateSections(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, nil)
+
+	applyChurn(t, mc, "/dir", "a-", 2*sectionHotThreshold, 2000, LocalMetadataResponseApplyOptions)
+
+	if !mc.IsNameFresh(util.FullPath("/dir/a-000")) {
+		t.Fatal("this mount's own writes must not invalidate its cache")
+	}
+}
+
+func TestSectionRefreshReconciles(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+	mc.SetPinnedChildFn(func(entry *filer.Entry) bool {
+		return entry.Name() == "b-pinned"
+	})
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})
+
+	for _, name := range []string{"b-keep", "b-vanish", "b-pinned"} {
+		if err := mc.InsertEntry(context.Background(), &filer.Entry{
+			FullPath: util.FullPath("/dir/" + name),
+			Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1},
+		}, 0); err != nil {
+			t.Fatalf("seed %s: %v", name, err)
+		}
+	}
+	// applied ahead of the refresh snapshot; the listing must not roll it back
+	if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name:       "b-newer",
+				Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 9, FileMode: 0100644, FileSize: 9},
+			},
+		},
+		TsNs: 9000,
+	}, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply newer event: %v", err)
+	}
+
+	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
+	if mc.IsNameFresh(util.FullPath("/dir/b-keep")) {
+		t.Fatal("section should be stale before the refresh")
+	}
+
+	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
+		kind:      metadataSectionRefresh,
+		buildPath: util.FullPath("/dir"),
+		refresh: &sectionRefresh{
+			hi: "m",
+			entries: []*filer.Entry{
+				{
+					FullPath: util.FullPath("/dir/b-keep"),
+					Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(2, 0), Mode: 0100644, FileSize: 7},
+				},
+				{
+					FullPath: util.FullPath("/dir/b-new"),
+					Attr:     filer.Attr{Crtime: time.Unix(2, 0), Mtime: time.Unix(2, 0), Mode: 0100644, FileSize: 3},
+				},
+			},
+			snapshotTsNs: 5000,
+		},
+	}); err != nil {
+		t.Fatalf("refresh: %v", err)
+	}
+
+	if !mc.IsNameFresh(util.FullPath("/dir/b-keep")) {
+		t.Fatal("section should be fresh after the refresh")
+	}
+	entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-keep"))
+	if err != nil || entry.FileSize != 7 {
+		t.Fatalf("b-keep = %+v, %v; want size 7", entry, err)
+	}
+	if entry, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/b-new")); err != nil || entry.FileSize != 3 {
+		t.Fatalf("b-new = %+v, %v; want size 3", entry, err)
+	}
+	if _, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/b-vanish")); err != filer_pb.ErrNotFound {
+		t.Fatalf("b-vanish error = %v, want not found", err)
+	}
+	if _, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/a-000")); err != filer_pb.ErrNotFound {
+		t.Fatalf("churned a-000 error = %v, want not found", err)
+	}
+	if entry, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/b-newer")); err != nil || entry.FileSize != 9 {
+		t.Fatalf("b-newer = %+v, %v; want size 9 kept", entry, err)
+	}
+	if entry, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/b-pinned")); err != nil || entry == nil {
+		t.Fatalf("pinned local-only entry must survive a refresh: %v", err)
+	}
+
+	// a redelivered create older than the snapshot must not resurrect b-vanish
+	if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name:       "b-vanish",
+				Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 1},
+			},
+		},
+		TsNs: 4000,
+	}, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply stale event: %v", err)
+	}
+	if _, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/b-vanish")); err != filer_pb.ErrNotFound {
+		t.Fatalf("b-vanish resurrected by a pre-snapshot event: %v", err)
+	}
+}
+
+func TestSectionRefreshSplitsOvergrownSection(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, nil)
+
+	count := 2*dirSectionSize + 1
+	entries := make([]*filer.Entry, 0, count)
+	for i := 0; i < count; i++ {
+		entries = append(entries, &filer.Entry{
+			FullPath: util.FullPath(fmt.Sprintf("/dir/f-%05d", i)),
+			Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1},
+		})
+	}
+	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
+		kind:      metadataSectionRefresh,
+		buildPath: util.FullPath("/dir"),
+		refresh:   &sectionRefresh{entries: entries, snapshotTsNs: 5000},
+	}); err != nil {
+		t.Fatalf("refresh: %v", err)
+	}
+
+	mc.RLock()
+	sl := mc.dirSections[util.FullPath("/dir")]
+	bounds := append([]string(nil), sl.bounds...)
+	sections := len(sl.sections)
+	mc.RUnlock()
+	want := []string{fmt.Sprintf("f-%05d", dirSectionSize), fmt.Sprintf("f-%05d", 2*dirSectionSize)}
+	if len(bounds) != len(want) || bounds[0] != want[0] || bounds[1] != want[1] {
+		t.Fatalf("bounds = %v, want %v", bounds, want)
+	}
+	if sections != len(bounds)+1 {
+		t.Fatalf("sections = %d, want %d", sections, len(bounds)+1)
+	}
+}
+
+type sectionFilerServer struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+	mu       sync.Mutex
+	names    []string // sorted; all under one directory
+	snapshot int64
+	requests []*filer_pb.ListEntriesRequest
+}
+
+func (s *sectionFilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream filer_pb.SeaweedFiler_ListEntriesServer) error {
+	s.mu.Lock()
+	s.requests = append(s.requests, req)
+	names := s.names
+	snapshot := s.snapshot
+	s.mu.Unlock()
+
+	sent := uint32(0)
+	first := true
+	for _, name := range names {
+		if name < req.StartFromFileName || (name == req.StartFromFileName && !req.InclusiveStartFrom) {
+			continue
+		}
+		resp := &filer_pb.ListEntriesResponse{Entry: &filer_pb.Entry{
+			Name:       name,
+			Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 1},
+		}}
+		if first {
+			resp.SnapshotTsNs = snapshot
+			first = false
+		}
+		if err := stream.Send(resp); err != nil {
+			return err
+		}
+		sent++
+		if req.Limit > 0 && sent >= req.Limit {
+			break
+		}
+	}
+	return nil
+}
+
+func (s *sectionFilerServer) listRequests() []*filer_pb.ListEntriesRequest {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]*filer_pb.ListEntriesRequest(nil), s.requests...)
+}
+
+type sectionTestFilerClient struct {
+	addr string
+}
+
+func (c *sectionTestFilerClient) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+	conn, err := grpc.NewClient(c.addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+	return fn(filer_pb.NewSeaweedFilerClient(conn))
+}
+
+func (c *sectionTestFilerClient) AdjustedUrl(location *filer_pb.Location) string { return location.Url }
+
+func (c *sectionTestFilerClient) GetDataCenter() string { return "" }
+
+func startSectionFilerServer(t *testing.T, s *sectionFilerServer) filer_pb.FilerClient {
+	t.Helper()
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	srv := grpc.NewServer()
+	filer_pb.RegisterSeaweedFilerServer(srv, s)
+	go srv.Serve(listener)
+	t.Cleanup(srv.Stop)
+	return &sectionTestFilerClient{addr: listener.Addr().String()}
+}
+
+func TestEnsureListingFreshRefreshesFromFiler(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	server := &sectionFilerServer{snapshot: 5000}
+	for i := 0; i < 1500; i++ {
+		server.names = append(server.names, fmt.Sprintf("b-%04d", i))
+	}
+	// at and beyond the section's end, must not be applied
+	server.names = append(server.names, "m", "z-1")
+	client := startSectionFilerServer(t, server)
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})
+	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
+	if mc.IsNameFresh(util.FullPath("/dir/a-000")) {
+		t.Fatal("section should be stale before the refresh")
+	}
+
+	if err := EnsureListingFresh(context.Background(), mc, client, util.FullPath("/dir"), ""); err != nil {
+		t.Fatalf("refresh: %v", err)
+	}
+
+	if !mc.IsNameFresh(util.FullPath("/dir/a-000")) {
+		t.Fatal("section should be fresh after the refresh")
+	}
+	for _, name := range []string{"b-0000", "b-1499"} {
+		if entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/"+name)); err != nil || entry == nil {
+			t.Fatalf("%s missing after refresh: %v", name, err)
+		}
+	}
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/a-000")); err != filer_pb.ErrNotFound {
+		t.Fatalf("churned a-000 error = %v, want not found", err)
+	}
+	for _, name := range []string{"m", "z-1"} {
+		if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/"+name)); err != filer_pb.ErrNotFound {
+			t.Fatalf("%s at or beyond the section's end was applied: %v", name, err)
+		}
+	}
+
+	requests := server.listRequests()
+	if len(requests) < 2 {
+		t.Fatalf("multi-page refresh made %d requests, want at least 2", len(requests))
+	}
+	if requests[0].SnapshotTsNs != 0 || requests[1].SnapshotTsNs != 5000 {
+		t.Fatalf("snapshot not pinned across pages: %d then %d", requests[0].SnapshotTsNs, requests[1].SnapshotTsNs)
+	}
+
+	// a fresh section costs no filer calls
+	if err := EnsureListingFresh(context.Background(), mc, client, util.FullPath("/dir"), ""); err != nil {
+		t.Fatalf("second refresh: %v", err)
+	}
+	if got := len(server.listRequests()); got != len(requests) {
+		t.Fatalf("fresh section still listed the filer: %d requests, was %d", got, len(requests))
+	}
+}
+
+func TestEnsureListingFreshGivesUpOnOvergrownRange(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	server := &sectionFilerServer{snapshot: 5000}
+	for i := 0; i <= sectionRefreshMaxEntries; i++ {
+		server.names = append(server.names, fmt.Sprintf("c-%05d", i))
+	}
+	client := startSectionFilerServer(t, server)
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, nil)
+	applyChurn(t, mc, "/dir", "x-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
+
+	err := EnsureListingFresh(context.Background(), mc, client, util.FullPath("/dir"), "")
+	if !errors.Is(err, ErrRefreshRangeTooLarge) {
+		t.Fatalf("refresh error = %v, want ErrRefreshRangeTooLarge", err)
+	}
+	if mc.IsNameFresh(util.FullPath("/dir/x-000")) {
+		t.Fatal("an aborted refresh must leave the section stale")
+	}
+}
+
+func TestSectionNoteChangeWindowExpiry(t *testing.T) {
+	sl := newSectionTable(nil)
+	t0 := time.Unix(100, 0)
+	for i := 0; i < sectionHotThreshold-1; i++ {
+		sl.noteChange(fmt.Sprintf("f-%03d", i), t0)
+	}
+	if !sl.isFresh("f-000") {
+		t.Fatal("below the threshold the section must stay fresh")
+	}
+
+	// the burst never completed inside one window, so the count restarts
+	t1 := t0.Add(sectionHotWindow + time.Second)
+	for i := 0; i < sectionHotThreshold-1; i++ {
+		sl.noteChange(fmt.Sprintf("f-%03d", i), t1)
+	}
+	if !sl.isFresh("f-000") {
+		t.Fatal("an expired window must not carry its count forward")
+	}
+
+	sl.noteChange("f-999", t1)
+	if sl.isFresh("f-000") {
+		t.Fatal("a full burst within one window must invalidate the section")
+	}
+}
+
+func TestSectionCompleteRefreshGuardsChangedTable(t *testing.T) {
+	sl := newSectionTable([]string{"g", "p"})
+	sl.sections[1].stale = true
+
+	if sl.completeRefresh("g", "q", []string{"h"}, 5000) {
+		t.Fatal("a range the table no longer has must be ignored")
+	}
+	if sl.isFresh("h") {
+		t.Fatal("an ignored refresh must not mark anything fresh")
+	}
+	if !sl.completeRefresh("g", "p", []string{"h"}, 5000) {
+		t.Fatal("the matching range must be accepted")
+	}
+	if !sl.isFresh("h") {
+		t.Fatal("section should be fresh after the refresh")
+	}
+}
+
+func TestSectionCompleteRefreshSplitsMiddleSection(t *testing.T) {
+	sl := newSectionTable([]string{"g", "p"})
+	sl.sections[1].stale = true
+
+	names := make([]string, 0, 2*dirSectionSize+1)
+	for i := 0; i <= 2*dirSectionSize; i++ {
+		names = append(names, fmt.Sprintf("g-%05d", i))
+	}
+	if !sl.completeRefresh("g", "p", names, 5000) {
+		t.Fatal("refresh of the middle section must be accepted")
+	}
+
+	wantBounds := []string{"g", names[dirSectionSize], names[2*dirSectionSize], "p"}
+	if len(sl.bounds) != len(wantBounds) {
+		t.Fatalf("bounds = %v, want %v", sl.bounds, wantBounds)
+	}
+	for i, b := range wantBounds {
+		if sl.bounds[i] != b {
+			t.Fatalf("bounds = %v, want %v", sl.bounds, wantBounds)
+		}
+	}
+	if len(sl.sections) != len(sl.bounds)+1 {
+		t.Fatalf("sections = %d, want %d", len(sl.sections), len(sl.bounds)+1)
+	}
+	for _, name := range []string{"a", "g-00000", names[dirSectionSize], "z"} {
+		if !sl.isFresh(name) {
+			t.Fatalf("%q should be fresh after the split", name)
+		}
+	}
+	if got := sl.sectionOf(names[dirSectionSize+1]); got != 2 {
+		t.Fatalf("sectionOf(%q) = %d, want 2", names[dirSectionSize+1], got)
+	}
+}
+
+func TestSectionFloorFencesAbsentNames(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})
+	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
+
+	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
+		kind:      metadataSectionRefresh,
+		buildPath: util.FullPath("/dir"),
+		refresh:   &sectionRefresh{hi: "m", snapshotTsNs: 5000},
+	}); err != nil {
+		t.Fatalf("refresh: %v", err)
+	}
+
+	// a delayed create between the directory floor and the refresh snapshot,
+	// for a name neither the cache nor the listing ever held
+	ghost := func(tsNs int64) {
+		t.Helper()
+		if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
+			Directory: "/dir",
+			EventNotification: &filer_pb.EventNotification{
+				NewEntry: &filer_pb.Entry{
+					Name:       "b-ghost",
+					Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 1},
+				},
+			},
+			TsNs: tsNs,
+		}, SubscriberMetadataResponseApplyOptions); err != nil {
+			t.Fatalf("apply ghost event: %v", err)
+		}
+	}
+	ghost(4500)
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-ghost")); err != filer_pb.ErrNotFound {
+		t.Fatalf("pre-snapshot event resurrected an absent name: %v", err)
+	}
+	ghost(6000)
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-ghost")); err != nil {
+		t.Fatalf("post-snapshot event must apply: %v", err)
+	}
+}
+
+func TestUnversionedRefreshStaysStale(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, nil)
+	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
+
+	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
+		kind:      metadataSectionRefresh,
+		buildPath: util.FullPath("/dir"),
+		refresh: &sectionRefresh{entries: []*filer.Entry{{
+			FullPath: util.FullPath("/dir/b-gap"),
+			Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1},
+		}}},
+	}); err != nil {
+		t.Fatalf("refresh: %v", err)
+	}
+
+	if mc.IsNameFresh(util.FullPath("/dir/a-000")) {
+		t.Fatal("an unversioned refresh vouches for nothing and must stay stale")
+	}
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-gap")); err != nil {
+		t.Fatalf("gap fill should still land: %v", err)
+	}
+	if ranges := mc.staleRangesAhead(util.FullPath("/dir"), ""); len(ranges) != 0 {
+		t.Fatalf("an unverifiable section must not be re-listed, got %d ranges", len(ranges))
+	}
+}
+
+func TestEnsureListingFreshCoversAllStaleSectionsAhead(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	server := &sectionFilerServer{snapshot: 5000, names: []string{"b-1", "n-1"}}
+	client := startSectionFilerServer(t, server)
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})
+	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
+	applyChurn(t, mc, "/dir", "n-", sectionHotThreshold, 3000, SubscriberMetadataResponseApplyOptions)
+
+	if err := EnsureListingFresh(context.Background(), mc, client, util.FullPath("/dir"), ""); err != nil {
+		t.Fatalf("refresh: %v", err)
+	}
+	if !mc.IsNameFresh(util.FullPath("/dir/a-000")) || !mc.IsNameFresh(util.FullPath("/dir/n-000")) {
+		t.Fatal("one call must re-validate every stale section ahead of the start")
+	}
+}
+
+func TestSectionCompleteRefreshSetsFloors(t *testing.T) {
+	sl := newSectionTable([]string{"g", "p"})
+	sl.sections[1].stale = true
+
+	names := make([]string, 0, 2*dirSectionSize+1)
+	for i := 0; i <= 2*dirSectionSize; i++ {
+		names = append(names, fmt.Sprintf("g-%05d", i))
+	}
+	if !sl.completeRefresh("g", "p", names, 7000) {
+		t.Fatal("refresh must be accepted")
+	}
+	for _, name := range []string{"g", "g-99999", names[dirSectionSize+1]} {
+		if got := sl.floorOf(name); got != 7000 {
+			t.Fatalf("floorOf(%q) = %d, want 7000", name, got)
+		}
+	}
+	if got := sl.floorOf("a"); got != 0 {
+		t.Fatalf("floorOf(a) = %d, want 0 for a never-refreshed section", got)
+	}
+}
+
+func TestSectionFloorOutranksOlderTombstone(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, nil)
+
+	event := func(tsNs int64, oldEntry, newEntry *filer_pb.Entry) {
+		t.Helper()
+		if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
+			Directory:         "/dir",
+			EventNotification: &filer_pb.EventNotification{OldEntry: oldEntry, NewEntry: newEntry},
+			TsNs:              tsNs,
+		}, SubscriberMetadataResponseApplyOptions); err != nil {
+			t.Fatalf("apply event at %d: %v", tsNs, err)
+		}
+	}
+	pbFile := &filer_pb.Entry{
+		Name:       "b-x",
+		Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 1},
+	}
+	event(1500, nil, pbFile)                       // create
+	event(2000, &filer_pb.Entry{Name: "b-x"}, nil) // delete, leaves a tombstone at 2000
+
+	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2500, SubscriberMetadataResponseApplyOptions)
+	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
+		kind:      metadataSectionRefresh,
+		buildPath: util.FullPath("/dir"),
+		refresh:   &sectionRefresh{snapshotTsNs: 5000},
+	}); err != nil {
+		t.Fatalf("refresh: %v", err)
+	}
+
+	// the floor at 5000 outranks the tombstone at 2000
+	event(4000, nil, pbFile)
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-x")); err != filer_pb.ErrNotFound {
+		t.Fatalf("pre-floor event applied over an older tombstone: %v", err)
+	}
+	event(6000, nil, pbFile)
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-x")); err != nil {
+		t.Fatalf("post-floor event must apply: %v", err)
+	}
+}
+
+func TestMismatchedRefreshLeavesStoreAlone(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: util.FullPath("/dir/b-keep"),
+		Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1},
+	}, 0); err != nil {
+		t.Fatalf("seed: %v", err)
+	}
+	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
+
+	// a range the table does not have: the reconcile must not touch the store
+	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
+		kind:      metadataSectionRefresh,
+		buildPath: util.FullPath("/dir"),
+		refresh: &sectionRefresh{hi: "q", snapshotTsNs: 5000, entries: []*filer.Entry{{
+			FullPath: util.FullPath("/dir/b-new"),
+			Attr:     filer.Attr{Crtime: time.Unix(2, 0), Mtime: time.Unix(2, 0), Mode: 0100644, FileSize: 3},
+		}}},
+	}); err != nil {
+		t.Fatalf("refresh: %v", err)
+	}
+
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-new")); err != filer_pb.ErrNotFound {
+		t.Fatalf("mismatched refresh inserted an entry: %v", err)
+	}
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-keep")); err != nil {
+		t.Fatalf("mismatched refresh swept an entry: %v", err)
+	}
+	if mc.IsNameFresh(util.FullPath("/dir/a-000")) {
+		t.Fatal("mismatched refresh must not mark anything fresh")
+	}
+}
+
+func TestRefreshClearsUnversionedMarker(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+	mc.SetPinnedChildFn(func(entry *filer.Entry) bool {
+		return entry.Name() == "b-pin"
+	})
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, nil)
+	for _, name := range []string{"b-local", "b-pin"} {
+		if err := mc.InsertEntry(context.Background(), &filer.Entry{
+			FullPath: util.FullPath("/dir/" + name),
+			Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1},
+		}, 0); err != nil { // versionTsNs 0 leaves the unversioned marker
+			t.Fatalf("seed %s: %v", name, err)
+		}
+	}
+	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
+
+	refreshed := func(name string) *filer.Entry {
+		return &filer.Entry{
+			FullPath: util.FullPath("/dir/" + name),
+			Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(2, 0), Mode: 0100644, FileSize: 7},
+		}
+	}
+	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
+		kind:      metadataSectionRefresh,
+		buildPath: util.FullPath("/dir"),
+		refresh:   &sectionRefresh{snapshotTsNs: 5000, entries: []*filer.Entry{refreshed("b-local"), refreshed("b-pin")}},
+	}); err != nil {
+		t.Fatalf("refresh: %v", err)
+	}
+
+	entry, versionTsNs, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-local"))
+	if err != nil || entry.FileSize != 7 {
+		t.Fatalf("b-local = %+v, %v; want the snapshot's size 7", entry, err)
+	}
+	if versionTsNs != 5000 {
+		t.Fatalf("b-local version = %d, want the floor 5000 once the marker is cleared", versionTsNs)
+	}
+
+	// a delayed event below the floor must not roll the snapshot write back
+	if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name:       "b-local",
+				Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 3, FileMode: 0100644, FileSize: 9},
+			},
+		},
+		TsNs: 4000,
+	}, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply delayed event: %v", err)
+	}
+	if entry, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/b-local")); err != nil || entry.FileSize != 7 {
+		t.Fatalf("b-local rolled back to %+v, %v", entry, err)
+	}
+
+	// pinned local-only state is neither replaced nor unmarked
+	entry, versionTsNs, err = mc.FindEntry(context.Background(), util.FullPath("/dir/b-pin"))
+	if err != nil || entry.FileSize != 1 {
+		t.Fatalf("b-pin = %+v, %v; want local size 1 kept", entry, err)
+	}
+	if versionTsNs != 0 {
+		t.Fatalf("b-pin version = %d, want 0 while pinned", versionTsNs)
+	}
+}
+
+func TestRefreshSkipsBuildingDirectory(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, nil)
+	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
+
+	// a rebuild is streaming: its inserts must survive a refresh's sweep
+	if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil {
+		t.Fatalf("begin build: %v", err)
+	}
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: util.FullPath("/dir/b-built"),
+		Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1},
+	}, 0); err != nil {
+		t.Fatalf("insert mid-build: %v", err)
+	}
+
+	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
+		kind:      metadataSectionRefresh,
+		buildPath: util.FullPath("/dir"),
+		refresh:   &sectionRefresh{snapshotTsNs: 5000},
+	}); err != nil {
+		t.Fatalf("refresh: %v", err)
+	}
+
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-built")); err != nil {
+		t.Fatalf("refresh swept a mid-build insert: %v", err)
+	}
+	if mc.IsNameFresh(util.FullPath("/dir/a-000")) {
+		t.Fatal("a skipped refresh must not mark the section fresh")
+	}
+	if err := mc.AbortDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil {
+		t.Fatalf("abort build: %v", err)
+	}
+}
+
+// TestSectionBorderEntries pins the border semantics: a bound-named entry
+// belongs to the section starting at the bound, a refresh sweeps [lo, hi)
+// inclusive of lo and exclusive of hi, and the neighboring sections' entries
+// come through untouched.
+func TestSectionBorderEntries(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"g", "p"})
+	for _, name := range []string{"g", "h", "p"} {
+		if err := mc.InsertEntry(context.Background(), &filer.Entry{
+			FullPath: util.FullPath("/dir/" + name),
+			Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1},
+		}, 0); err != nil {
+			t.Fatalf("seed %s: %v", name, err)
+		}
+	}
+	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
+	applyChurn(t, mc, "/dir", "h-", sectionHotThreshold, 3000, SubscriberMetadataResponseApplyOptions)
+
+	// the bound name is part of the section starting at it
+	if mc.IsNameFresh(util.FullPath("/dir/a-000")) || mc.IsNameFresh(util.FullPath("/dir/g")) {
+		t.Fatal("both churned sections should be stale")
+	}
+
+	// refreshing [ , g) sweeps its churn but must not reach the bound entry
+	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
+		kind:      metadataSectionRefresh,
+		buildPath: util.FullPath("/dir"),
+		refresh:   &sectionRefresh{hi: "g", snapshotTsNs: 5000},
+	}); err != nil {
+		t.Fatalf("refresh section 0: %v", err)
+	}
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/a-000")); err != filer_pb.ErrNotFound {
+		t.Fatalf("churned a-000 should be swept: %v", err)
+	}
+	for _, name := range []string{"g", "h", "p"} {
+		if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/"+name)); err != nil {
+			t.Fatalf("%s swept by the neighboring refresh: %v", name, err)
+		}
+	}
+	if !mc.IsNameFresh(util.FullPath("/dir/a-000")) || mc.IsNameFresh(util.FullPath("/dir/g")) {
+		t.Fatal("only the refreshed section should be fresh")
+	}
+
+	// refreshing [g, p) covers the bound entry itself and stops before p
+	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
+		kind:      metadataSectionRefresh,
+		buildPath: util.FullPath("/dir"),
+		refresh: &sectionRefresh{lo: "g", hi: "p", snapshotTsNs: 5001, entries: []*filer.Entry{{
+			FullPath: util.FullPath("/dir/h"),
+			Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(2, 0), Mode: 0100644, FileSize: 7},
+		}}},
+	}); err != nil {
+		t.Fatalf("refresh section 1: %v", err)
+	}
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/g")); err != filer_pb.ErrNotFound {
+		t.Fatalf("vanished bound entry should be swept by its own section: %v", err)
+	}
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/h-000")); err != filer_pb.ErrNotFound {
+		t.Fatalf("churned h-000 should be swept: %v", err)
+	}
+	if entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/h")); err != nil || entry.FileSize != 7 {
+		t.Fatalf("h = %+v, %v; want size 7", entry, err)
+	}
+	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/p")); err != nil {
+		t.Fatalf("the next bound's entry is outside the range and must survive: %v", err)
+	}
+}
+
+func TestChurnBeyondBuiltEntriesLandsInEdgeSection(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})
+
+	// names sorting past everything the build saw land in the last section
+	applyChurn(t, mc, "/dir", "z-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
+
+	if mc.IsNameFresh(util.FullPath("/dir/z-000")) {
+		t.Fatal("the tail section should be invalidated")
+	}
+	if !mc.IsNameFresh(util.FullPath("/dir/a")) {
+		t.Fatal("the head section must stay fresh")
+	}
+}
+
+func TestRenameAcrossSectionsCountsBoth(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/dir": true,
+	})
+	defer mc.Shutdown()
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})
+
+	for i := 0; i < sectionHotThreshold; i++ {
+		resp := &filer_pb.SubscribeMetadataResponse{
+			Directory: "/dir",
+			EventNotification: &filer_pb.EventNotification{
+				OldEntry: &filer_pb.Entry{Name: fmt.Sprintf("a-%03d", i)},
+				NewEntry: &filer_pb.Entry{
+					Name:       fmt.Sprintf("n-%03d", i),
+					Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 1},
+				},
+				NewParentPath: "/dir",
+			},
+			TsNs: 2000 + int64(i),
+		}
+		if err := mc.ApplyMetadataResponse(context.Background(), resp, SubscriberMetadataResponseApplyOptions); err != nil {
+			t.Fatalf("apply rename %d: %v", i, err)
+		}
+	}
+
+	if mc.IsNameFresh(util.FullPath("/dir/a-000")) {
+		t.Fatal("the vacated side's section should be invalidated")
+	}
+	if mc.IsNameFresh(util.FullPath("/dir/n-000")) {
+		t.Fatal("the landing side's section should be invalidated")
+	}
+}
+
+// A refresh sweeps the names its listing did not return. One recorded at
+// exactly the snapshot has its write reflected but not its removal, so reading
+// the gate as a write left it cached for good.
+func TestSectionRefreshSweepsANameRecordedAtTheSnapshot(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true, "/dir": true})
+	defer mc.Shutdown()
+
+	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})
+
+	const snapshotTsNs = 5000
+	if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name:       "b-vanish",
+				Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 3},
+			},
+		},
+		TsNs: snapshotTsNs,
+	}, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("seed at the snapshot version: %v", err)
+	}
+
+	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
+		kind:      metadataSectionRefresh,
+		buildPath: util.FullPath("/dir"),
+		refresh: &sectionRefresh{
+			hi: "m",
+			entries: []*filer.Entry{
+				{
+					FullPath: util.FullPath("/dir/b-keep"),
+					Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(2, 0), Mode: 0100644, FileSize: 7},
+				},
+			},
+			snapshotTsNs: snapshotTsNs,
+		},
+	}); err != nil {
+		t.Fatalf("refresh: %v", err)
+	}
+
+	if entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-vanish")); err == nil && entry != nil {
+		t.Error("a name the refresh did not list is still cached")
+	}
+}

weed/mount/meta_cache/meta_cache_subscribe.go

@@ -0,0 +1,102 @@
+package meta_cache
+
+import (
+	"context"
+	"io"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+)
+
+type MetadataFollower struct {
+	PathPrefixToWatch string
+	ProcessEventFn    func(resp *filer_pb.SubscribeMetadataResponse) error
+}
+
+func mergeProcessors(mainProcessor func(resp *filer_pb.SubscribeMetadataResponse) error, followers ...*MetadataFollower) func(resp *filer_pb.SubscribeMetadataResponse) error {
+	return func(resp *filer_pb.SubscribeMetadataResponse) error {
+
+		// build the full path
+		entry := resp.EventNotification.NewEntry
+		if entry == nil {
+			entry = resp.EventNotification.OldEntry
+		}
+		if entry != nil {
+			dir := resp.Directory
+			if resp.EventNotification.NewParentPath != "" {
+				dir = resp.EventNotification.NewParentPath
+			}
+			fp := util.NewFullPath(dir, entry.Name)
+
+			for _, follower := range followers {
+				if strings.HasPrefix(string(fp), follower.PathPrefixToWatch) {
+					if err := follower.ProcessEventFn(resp); err != nil {
+						return err
+					}
+				}
+			}
+		}
+		return mainProcessor(resp)
+	}
+}
+
+func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.FilerClient, lookupFn wdclient.LookupFileIdFunctionType, dir string, lastTsNs int64, skipSelfEvents bool, onRetry func(lastTsNs int64, err error), followers ...*MetadataFollower) error {
+
+	var prefixes []string
+	for _, follower := range followers {
+		prefixes = append(prefixes, follower.PathPrefixToWatch)
+	}
+
+	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
+		if skipSelfEvents && resp.EventNotification != nil {
+			for _, sig := range resp.EventNotification.Signatures {
+				if sig == selfSignature {
+					glog.V(4).Infof("skip self-originated event %s", resp.Directory)
+					return nil
+				}
+			}
+		}
+		return mc.ApplyMetadataResponse(context.Background(), resp, SubscriberMetadataResponseApplyOptions)
+	}
+
+	prefix := dir
+	if !strings.HasSuffix(prefix, "/") {
+		prefix = prefix + "/"
+	}
+
+	// Replaying the persisted log chunks here keeps the cost off the filer's heap.
+	// The caller's lookup says where to read them from, so a mount that cannot
+	// reach volume servers directly still replays through its filer.
+	metadataFollowOption := &pb.MetadataFollowOption{
+		ClientName:             "mount",
+		ClientId:               selfSignature,
+		ClientEpoch:            1,
+		SelfSignature:          selfSignature,
+		PathPrefix:             prefix,
+		AdditionalPathPrefixes: prefixes,
+		DirectoriesToWatch:     nil,
+		StartTsNs:              lastTsNs,
+		StopTsNs:               0,
+		EventErrorType:         pb.FatalOnError,
+		LogFileReaderFn: func(chunks []*filer_pb.FileChunk) (io.ReadCloser, error) {
+			return filer.NewChunkStreamReaderFromLookup(context.Background(), lookupFn, chunks), nil
+		},
+	}
+	util.RetryUntil("followMetaUpdates", func() error {
+		metadataFollowOption.ClientEpoch++
+		return pb.WithFilerClientFollowMetadata(client, metadataFollowOption, mergeProcessors(processEventFn, followers...))
+	}, func(err error) bool {
+		if onRetry != nil {
+			onRetry(metadataFollowOption.StartTsNs, err)
+		}
+		glog.Errorf("follow metadata updates: %v", err)
+		return true
+	})
+
+	return nil
+}

weed/mount/meta_cache/oversized_dir_test.go

@@ -0,0 +1,138 @@
+package meta_cache
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+)
+
+func listResponses(n int) []*filer_pb.ListEntriesResponse {
+	responses := make([]*filer_pb.ListEntriesResponse, 0, n)
+	for i := 0; i < n; i++ {
+		responses = append(responses, &filer_pb.ListEntriesResponse{
+			Entry: &filer_pb.Entry{
+				Name: fmt.Sprintf("f%05d", i),
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime: 1, Mtime: 1, FileMode: 0o644, FileSize: 3,
+				},
+			},
+		})
+	}
+	return responses
+}
+
+// TestEnsureVisitedRefusesOversizedDirectory checks that a directory past the
+// limit is not cached, that the refusal is remembered, and that the partial
+// build leaves nothing behind in the local store.
+func TestEnsureVisitedRefusesOversizedDirectory(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
+	defer mc.Shutdown()
+
+	accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(10)}}
+
+	err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 5)
+	var tooLarge *DirectoryTooLargeError
+	if !errors.As(err, &tooLarge) {
+		t.Fatalf("EnsureVisited = %v, want DirectoryTooLargeError", err)
+	}
+	if tooLarge.Path != util.FullPath("/dir") {
+		t.Fatalf("oversized path = %s, want /dir", tooLarge.Path)
+	}
+	if mc.IsDirectoryCached(util.FullPath("/dir")) {
+		t.Error("oversized directory reported cached")
+	}
+	// The aborted build must leave no partial children to be served later.
+	count := 0
+	if _, err := mc.ListDirectoryEntries(context.Background(), util.FullPath("/dir"), "", false, 100, func(e *filer.Entry) (bool, error) {
+		count++
+		return true, nil
+	}); err != nil {
+		t.Fatalf("list: %v", err)
+	}
+	if count != 0 {
+		t.Errorf("local store still holds %d children of the aborted build", count)
+	}
+
+	// A second visit fails fast without streaming to the limit again.
+	if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 5); !errors.As(err, &tooLarge) {
+		t.Fatalf("second EnsureVisited = %v, want DirectoryTooLargeError", err)
+	}
+}
+
+// TestEnsureVisitedStepsOverOversizedAncestor checks a huge ancestor does not
+// wedge the caching of its subdirectories.
+func TestEnsureVisitedStepsOverOversizedAncestor(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
+	defer mc.Shutdown()
+	mc.markOversized(util.FullPath("/huge"))
+
+	accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(3)}}
+	if err := EnsureVisited(mc, accessor, util.FullPath("/huge/sub"), 5); err != nil {
+		t.Fatalf("EnsureVisited under an oversized ancestor: %v", err)
+	}
+	if !mc.IsDirectoryCached(util.FullPath("/huge/sub")) {
+		t.Error("subdirectory of an oversized ancestor was not cached")
+	}
+	if mc.IsDirectoryCached(util.FullPath("/huge")) {
+		t.Error("oversized ancestor became cached")
+	}
+}
+
+// TestEnsureVisitedUnderTheLimitStillCaches pins that the gate does not change
+// behaviour for ordinary directories.
+func TestEnsureVisitedUnderTheLimitStillCaches(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
+	defer mc.Shutdown()
+
+	accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(5)}}
+	if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 5); err != nil {
+		t.Fatalf("EnsureVisited: %v", err)
+	}
+	if !mc.IsDirectoryCached(util.FullPath("/dir")) {
+		t.Error("directory at the limit was not cached")
+	}
+}
+
+// pathListClient serves canned listings per directory, so one visit can see
+// directories of different sizes.
+type pathListClient struct {
+	filer_pb.SeaweedFilerClient
+	perDir map[string][]*filer_pb.ListEntriesResponse
+}
+
+func (c *pathListClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
+	return &buildListStream{responses: c.perDir[in.Directory]}, nil
+}
+
+// TestEnsureVisitedAncestorFoundOversizedMidVisit covers the first discovery:
+// the ancestor's refusal must neither cancel the descendant's build nor be
+// reported as the descendant's own.
+func TestEnsureVisitedAncestorFoundOversizedMidVisit(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
+	defer mc.Shutdown()
+
+	accessor := &buildFilerAccessor{client: &pathListClient{perDir: map[string][]*filer_pb.ListEntriesResponse{
+		"/huge":     listResponses(10),
+		"/huge/sub": listResponses(3),
+	}}}
+
+	if err := EnsureVisited(mc, accessor, util.FullPath("/huge/sub"), 5); err != nil {
+		t.Fatalf("EnsureVisited: %v", err)
+	}
+	if !mc.IsDirectoryCached(util.FullPath("/huge/sub")) {
+		t.Error("descendant of a just-discovered oversized ancestor was not cached")
+	}
+	if mc.IsDirectoryCached(util.FullPath("/huge")) {
+		t.Error("oversized ancestor became cached")
+	}
+	if !mc.isOversized(util.FullPath("/huge")) {
+		t.Error("oversized ancestor was not remembered")
+	}
+}

weed/mount/metadata_events.go

@@ -0,0 +1,86 @@
+package mount
+
+import (
+	"context"
+
+	"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"google.golang.org/protobuf/proto"
+)
+
+// filerAckResponse is any mutation response carrying the log position of the
+// state it acknowledged.
+type filerAckResponse interface {
+	GetMetadataEvent() *filer_pb.SubscribeMetadataResponse
+	GetLogTsNs() int64
+}
+
+// ackVersionTsNs is the log position an ack confirmed: its event's timestamp,
+// or the response's fence when the ack carried no event.
+func ackVersionTsNs(resp filerAckResponse) int64 {
+	if ts := resp.GetMetadataEvent().GetTsNs(); ts != 0 {
+		return ts
+	}
+	return resp.GetLogTsNs()
+}
+
+func (wfs *WFS) applyLocalMetadataEvent(ctx context.Context, event *filer_pb.SubscribeMetadataResponse) error {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	return wfs.metaCache.ApplyMetadataResponseOwned(ctx, event, meta_cache.LocalMetadataResponseApplyOptions)
+}
+
+func (wfs *WFS) applyLocalMetadataEventAsync(event *filer_pb.SubscribeMetadataResponse) {
+	wfs.metaCache.ApplyMetadataResponseOwnedAsync(event, meta_cache.LocalMetadataResponseApplyOptions)
+}
+
+func metadataDeleteEvent(directory, name string, isDirectory bool) *filer_pb.SubscribeMetadataResponse {
+	if name == "" {
+		return nil
+	}
+	return &filer_pb.SubscribeMetadataResponse{
+		Directory: directory,
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{Name: name, IsDirectory: isDirectory},
+		},
+	}
+}
+
+func metadataCreateEvent(directory string, entry *filer_pb.Entry) *filer_pb.SubscribeMetadataResponse {
+	if entry == nil {
+		return nil
+	}
+	return &filer_pb.SubscribeMetadataResponse{
+		Directory: directory,
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry:      proto.Clone(entry).(*filer_pb.Entry),
+			NewParentPath: directory,
+		},
+	}
+}
+
+func metadataUpdateEvent(directory string, entry *filer_pb.Entry) *filer_pb.SubscribeMetadataResponse {
+	if entry == nil {
+		return nil
+	}
+	return &filer_pb.SubscribeMetadataResponse{
+		Directory: directory,
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: entry.Name},
+			NewEntry:      proto.Clone(entry).(*filer_pb.Entry),
+			NewParentPath: directory,
+		},
+	}
+}
+
+func metadataEventFromRenameResponse(resp *filer_pb.StreamRenameEntryResponse) *filer_pb.SubscribeMetadataResponse {
+	if resp == nil || resp.EventNotification == nil {
+		return nil
+	}
+	return &filer_pb.SubscribeMetadataResponse{
+		Directory:         resp.Directory,
+		EventNotification: proto.Clone(resp.EventNotification).(*filer_pb.EventNotification),
+		TsNs:              resp.TsNs,
+	}
+}

weed/mount/metadata_flush_retry.go

@@ -0,0 +1,55 @@
+package mount
+
+import (
+	"context"
+	"time"
+)
+
+const metadataFlushRetries = 3
+
+// metadataFlushSleep waits for d or until ctx is cancelled. Overridable in tests.
+var metadataFlushSleep = func(ctx context.Context, d time.Duration) {
+	timer := time.NewTimer(d)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+	case <-timer.C:
+	}
+}
+
+func retryMetadataFlush(ctx context.Context, flush func() error, onRetry func(nextAttempt, totalAttempts int, backoff time.Duration, err error)) error {
+	return retryMetadataFlushIf(ctx, flush, nil, onRetry)
+}
+
+// retryMetadataFlushIf retries flush with exponential backoff, stopping early
+// when shouldRetry returns false (clearly permanent errors) or when ctx is
+// cancelled (the FUSE request was interrupted, e.g. the process was killed).
+func retryMetadataFlushIf(ctx context.Context, flush func() error, shouldRetry func(error) bool, onRetry func(nextAttempt, totalAttempts int, backoff time.Duration, err error)) error {
+	totalAttempts := metadataFlushRetries + 1
+	var err error
+	for attempt := 1; attempt <= totalAttempts; attempt++ {
+		err = flush()
+		if err == nil {
+			break
+		}
+		if attempt == totalAttempts {
+			break
+		}
+		if shouldRetry != nil && !shouldRetry(err) {
+			break
+		}
+		if ctx.Err() != nil {
+			break
+		}
+
+		backoff := time.Duration(1<<uint(attempt-1)) * time.Second
+		if onRetry != nil {
+			onRetry(attempt+1, totalAttempts, backoff, err)
+		}
+		metadataFlushSleep(ctx, backoff)
+		if ctx.Err() != nil {
+			break
+		}
+	}
+	return err
+}

weed/mount/metadata_flush_retry_test.go

@@ -0,0 +1,118 @@
+package mount
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+)
+
+func TestRetryMetadataFlushEventuallySucceeds(t *testing.T) {
+	originalSleep := metadataFlushSleep
+	t.Cleanup(func() {
+		metadataFlushSleep = originalSleep
+	})
+
+	var sleeps []time.Duration
+	metadataFlushSleep = func(ctx context.Context, d time.Duration) {
+		sleeps = append(sleeps, d)
+	}
+
+	attempts := 0
+	err := retryMetadataFlush(context.Background(), func() error {
+		attempts++
+		if attempts < 3 {
+			return errors.New("temporary failure")
+		}
+		return nil
+	}, nil)
+	if err != nil {
+		t.Fatalf("retryMetadataFlush returned error: %v", err)
+	}
+
+	if attempts != 3 {
+		t.Fatalf("attempts = %d, want 3", attempts)
+	}
+
+	wantSleeps := []time.Duration{time.Second, 2 * time.Second}
+	if len(sleeps) != len(wantSleeps) {
+		t.Fatalf("sleep count = %d, want %d", len(sleeps), len(wantSleeps))
+	}
+	for i, want := range wantSleeps {
+		if sleeps[i] != want {
+			t.Fatalf("sleep[%d] = %v, want %v", i, sleeps[i], want)
+		}
+	}
+}
+
+func TestRetryMetadataFlushReturnsLastError(t *testing.T) {
+	originalSleep := metadataFlushSleep
+	t.Cleanup(func() {
+		metadataFlushSleep = originalSleep
+	})
+
+	var sleeps []time.Duration
+	metadataFlushSleep = func(ctx context.Context, d time.Duration) {
+		sleeps = append(sleeps, d)
+	}
+
+	expectedErr := errors.New("permanent failure")
+	attempts := 0
+	err := retryMetadataFlush(context.Background(), func() error {
+		attempts++
+		return expectedErr
+	}, nil)
+	if !errors.Is(err, expectedErr) {
+		t.Fatalf("retryMetadataFlush error = %v, want %v", err, expectedErr)
+	}
+
+	if attempts != metadataFlushRetries+1 {
+		t.Fatalf("attempts = %d, want %d", attempts, metadataFlushRetries+1)
+	}
+
+	wantSleeps := []time.Duration{time.Second, 2 * time.Second, 4 * time.Second}
+	if len(sleeps) != len(wantSleeps) {
+		t.Fatalf("sleep count = %d, want %d", len(sleeps), len(wantSleeps))
+	}
+	for i, want := range wantSleeps {
+		if sleeps[i] != want {
+			t.Fatalf("sleep[%d] = %v, want %v", i, sleeps[i], want)
+		}
+	}
+}
+
+// TestRetryMetadataFlushStopsOnCancel verifies that once the flush context is
+// done (the flush deadline elapsed against an overwhelmed filer) the retry loop
+// abandons its retries immediately instead of sleeping out the backoff, so
+// close() is not held longer than the deadline.
+func TestRetryMetadataFlushStopsOnCancel(t *testing.T) {
+	originalSleep := metadataFlushSleep
+	t.Cleanup(func() {
+		metadataFlushSleep = originalSleep
+	})
+
+	var sleeps []time.Duration
+	metadataFlushSleep = func(ctx context.Context, d time.Duration) {
+		sleeps = append(sleeps, d)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // flush deadline already elapsed before the first attempt completes
+
+	attempts := 0
+	flushErr := errors.New("filer overwhelmed")
+	err := retryMetadataFlush(ctx, func() error {
+		attempts++
+		return flushErr
+	}, nil)
+	if !errors.Is(err, flushErr) {
+		t.Fatalf("retryMetadataFlush error = %v, want %v", err, flushErr)
+	}
+
+	if attempts != 1 {
+		t.Fatalf("attempts = %d, want 1 (no retries after cancel)", attempts)
+	}
+	if len(sleeps) != 0 {
+		t.Fatalf("sleeps = %v, want none (no backoff after cancel)", sleeps)
+	}
+}

weed/mount/page_writer.go

@@ -0,0 +1,107 @@
+package mount
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mount/page_writer"
+)
+
+type PageWriter struct {
+	fh            *FileHandle
+	collection    string
+	replication   string
+	chunkSize     int64
+	writerPattern *WriterPattern
+
+	randomWriter page_writer.DirtyPages
+}
+
+var (
+	_ = page_writer.DirtyPages(&PageWriter{})
+)
+
+func newPageWriter(fh *FileHandle, chunkSize int64) *PageWriter {
+	pw := &PageWriter{
+		fh:            fh,
+		chunkSize:     chunkSize,
+		writerPattern: NewWriterPattern(chunkSize),
+		randomWriter:  newMemoryChunkPages(fh, chunkSize),
+	}
+	return pw
+}
+
+func (pw *PageWriter) AddPage(offset int64, data []byte, isSequential bool, tsNs int64) error {
+
+	glog.V(4).Infof("%v AddPage [%d, %d)", pw.fh.fh, offset, offset+int64(len(data)))
+
+	chunkIndex := offset / pw.chunkSize
+	for i := chunkIndex; len(data) > 0; i++ {
+		writeSize := min(int64(len(data)), (i+1)*pw.chunkSize-offset)
+		if err := pw.addToOneChunk(i, offset, data[:writeSize], isSequential, tsNs); err != nil {
+			return err
+		}
+		offset += writeSize
+		data = data[writeSize:]
+	}
+	return nil
+}
+
+func (pw *PageWriter) addToOneChunk(chunkIndex, offset int64, data []byte, isSequential bool, tsNs int64) error {
+	return pw.randomWriter.AddPage(offset, data, isSequential, tsNs)
+}
+
+func (pw *PageWriter) FlushData() error {
+	return pw.randomWriter.FlushData()
+}
+
+func (pw *PageWriter) HasWrites() bool {
+	return pw.randomWriter.HasWrites()
+}
+
+func (pw *PageWriter) ReadDirtyDataAt(data []byte, offset int64, tsNs int64) (maxStop int64) {
+	glog.V(4).Infof("ReadDirtyDataAt %v [%d, %d)", pw.fh.inode, offset, offset+int64(len(data)))
+
+	chunkIndex := offset / pw.chunkSize
+	for i := chunkIndex; len(data) > 0; i++ {
+		readSize := min(int64(len(data)), (i+1)*pw.chunkSize-offset)
+
+		maxStop = pw.randomWriter.ReadDirtyDataAt(data[:readSize], offset, tsNs)
+
+		offset += readSize
+		data = data[readSize:]
+	}
+
+	return
+}
+
+func (pw *PageWriter) LockForRead(startOffset, stopOffset int64) {
+	pw.randomWriter.LockForRead(startOffset, stopOffset)
+}
+
+func (pw *PageWriter) UnlockForRead(startOffset, stopOffset int64) {
+	pw.randomWriter.UnlockForRead(startOffset, stopOffset)
+}
+
+func (pw *PageWriter) Destroy() {
+	pw.randomWriter.Destroy()
+}
+
+func (pw *PageWriter) EvictOneWritableChunk() bool {
+	return pw.randomWriter.EvictOneWritableChunk()
+}
+
+func (pw *PageWriter) ProactiveFlush(nowNs, idleThresholdNs, maxHoldNs, fillRatio int64, frontierLag int, isSequential bool) bool {
+	return pw.randomWriter.ProactiveFlush(nowNs, idleThresholdNs, maxHoldNs, fillRatio, frontierLag, isSequential)
+}
+
+func max(x, y int64) int64 {
+	if x > y {
+		return x
+	}
+	return y
+}
+func min(x, y int64) int64 {
+	if x < y {
+		return x
+	}
+	return y
+}

weed/mount/page_writer/activity_score.go

@@ -0,0 +1,52 @@
+package page_writer
+
+import (
+	"sync/atomic"
+	"time"
+)
+
+type ActivityScore struct {
+	lastActiveTsNs         int64 // atomic
+	decayedActivenessScore int64 // atomic
+}
+
+func NewActivityScore() *ActivityScore {
+	return &ActivityScore{}
+}
+
+func (as *ActivityScore) MarkRead() {
+	now := time.Now().UnixNano()
+	last := atomic.SwapInt64(&as.lastActiveTsNs, now)
+	deltaTime := (now - last) >> 30 // about number of seconds
+	for {
+		old := atomic.LoadInt64(&as.decayedActivenessScore)
+		score := old>>deltaTime + 256
+		if score < 0 {
+			score = 0
+		}
+		if atomic.CompareAndSwapInt64(&as.decayedActivenessScore, old, score) {
+			break
+		}
+	}
+}
+
+func (as *ActivityScore) MarkWrite() {
+	now := time.Now().UnixNano()
+	last := atomic.SwapInt64(&as.lastActiveTsNs, now)
+	deltaTime := (now - last) >> 30 // about number of seconds
+	for {
+		old := atomic.LoadInt64(&as.decayedActivenessScore)
+		score := old>>deltaTime + 1024
+		if score < 0 {
+			score = 0
+		}
+		if atomic.CompareAndSwapInt64(&as.decayedActivenessScore, old, score) {
+			break
+		}
+	}
+}
+
+func (as *ActivityScore) ActivityScore() int64 {
+	deltaTime := (time.Now().UnixNano() - atomic.LoadInt64(&as.lastActiveTsNs)) >> 30
+	return atomic.LoadInt64(&as.decayedActivenessScore) >> deltaTime
+}

weed/mount/page_writer/chunk_interval_list.go

@@ -0,0 +1,165 @@
+package page_writer
+
+import (
+	"math"
+)
+
+// ChunkWrittenInterval mark one written interval within one page chunk
+type ChunkWrittenInterval struct {
+	StartOffset int64
+	stopOffset  int64
+	TsNs        int64
+	prev        *ChunkWrittenInterval
+	next        *ChunkWrittenInterval
+}
+
+func (interval *ChunkWrittenInterval) Size() int64 {
+	return interval.stopOffset - interval.StartOffset
+}
+
+func (interval *ChunkWrittenInterval) isComplete(chunkSize int64) bool {
+	return interval.stopOffset-interval.StartOffset == chunkSize
+}
+
+// ChunkWrittenIntervalList mark written intervals within one page chunk
+type ChunkWrittenIntervalList struct {
+	head *ChunkWrittenInterval
+	tail *ChunkWrittenInterval
+}
+
+func newChunkWrittenIntervalList() *ChunkWrittenIntervalList {
+	list := &ChunkWrittenIntervalList{
+		head: &ChunkWrittenInterval{
+			StartOffset: -1,
+			stopOffset:  -1,
+		},
+		tail: &ChunkWrittenInterval{
+			StartOffset: math.MaxInt64,
+			stopOffset:  math.MaxInt64,
+		},
+	}
+	list.head.next = list.tail
+	list.tail.prev = list.head
+	return list
+}
+
+func (list *ChunkWrittenIntervalList) MarkWritten(startOffset, stopOffset, tsNs int64) {
+	if startOffset >= stopOffset {
+		return
+	}
+	interval := &ChunkWrittenInterval{
+		StartOffset: startOffset,
+		stopOffset:  stopOffset,
+		TsNs:        tsNs,
+	}
+	list.addInterval(interval)
+}
+
+// IsComplete reports whether every byte of [0, chunkSize) has been
+// written, possibly via multiple adjacent or overlapping intervals.
+// addInterval does not merge adjacent intervals — a chunk filled by
+// two 1 MiB writes ends up as {[0,1M], [1M,2M]}, not one [0,2M] —
+// so checking list.size()==1 misses the "filled by adjacent writes"
+// case, leaving the chunk pinned in writableChunks even though all
+// its bytes are present. That latent bug became a hard deadlock once
+// -writeBufferSizeMB started reserving a global slot per writable
+// chunk: the chunks never got sealed, no uploader ran, no slot was
+// ever released, and the FUSE writer blocked in Reserve forever
+// (seaweedfs issue #8777 / PR #9066). Walking the list and tracking
+// the furthest covered offset detects adjacency correctly.
+func (list *ChunkWrittenIntervalList) IsComplete(chunkSize int64) bool {
+	var covered int64
+	for t := list.head.next; t != list.tail; t = t.next {
+		if t.StartOffset > covered {
+			return false // gap before this interval
+		}
+		if t.stopOffset > covered {
+			covered = t.stopOffset
+		}
+	}
+	return covered >= chunkSize
+}
+func (list *ChunkWrittenIntervalList) WrittenSize() (writtenByteCount int64) {
+	for t := list.head; t != nil; t = t.next {
+		writtenByteCount += t.Size()
+	}
+	return
+}
+
+// IsContiguouslyWritten reports whether the written bytes form one
+// unbroken run starting at offset 0. Pressure-driven sealing uses this
+// to avoid racing in-flight FUSE writeback on a gap range — sealing a
+// gappy chunk would emit volume chunks with no coverage for the gap and
+// reads would silently zero-fill it (issue #9330).
+func (list *ChunkWrittenIntervalList) IsContiguouslyWritten() bool {
+	first := list.head.next
+	if first == list.tail || first.StartOffset != 0 {
+		return false
+	}
+	for t := first; t.next != list.tail; t = t.next {
+		if t.stopOffset != t.next.StartOffset {
+			return false
+		}
+	}
+	return true
+}
+
+func (list *ChunkWrittenIntervalList) addInterval(interval *ChunkWrittenInterval) {
+
+	//t := list.head
+	//for ; t.next != nil; t = t.next {
+	//	if t.TsNs > interval.TsNs {
+	//		println("writes is out of order", t.TsNs-interval.TsNs, "ns")
+	//	}
+	//}
+
+	p := list.head
+	for ; p.next != nil && p.next.stopOffset <= interval.StartOffset; p = p.next {
+	}
+	q := list.tail
+	for ; q.prev != nil && q.prev.StartOffset >= interval.stopOffset; q = q.prev {
+	}
+
+	// left side
+	// interval after p.next start
+	if p.next.StartOffset < interval.StartOffset {
+		t := &ChunkWrittenInterval{
+			StartOffset: p.next.StartOffset,
+			stopOffset:  interval.StartOffset,
+			TsNs:        p.next.TsNs,
+		}
+		p.next = t
+		t.prev = p
+		t.next = interval
+		interval.prev = t
+	} else {
+		p.next = interval
+		interval.prev = p
+	}
+
+	// right side
+	// interval ends before p.prev
+	if interval.stopOffset < q.prev.stopOffset {
+		t := &ChunkWrittenInterval{
+			StartOffset: interval.stopOffset,
+			stopOffset:  q.prev.stopOffset,
+			TsNs:        q.prev.TsNs,
+		}
+		q.prev = t
+		t.next = q
+		interval.next = t
+		t.prev = interval
+	} else {
+		q.prev = interval
+		interval.next = q
+	}
+
+}
+
+func (list *ChunkWrittenIntervalList) size() int {
+	var count int
+	for t := list.head; t != nil; t = t.next {
+		count++
+	}
+	return count - 2
+}

weed/mount/page_writer/chunk_interval_list_test.go

@@ -0,0 +1,189 @@
+package page_writer
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_PageChunkWrittenIntervalList(t *testing.T) {
+	list := newChunkWrittenIntervalList()
+
+	assert.Equal(t, 0, list.size(), "empty list")
+
+	list.MarkWritten(0, 5, 1)
+	assert.Equal(t, 1, list.size(), "one interval")
+
+	list.MarkWritten(0, 5, 2)
+	assert.Equal(t, 1, list.size(), "duplicated interval2")
+
+	list.MarkWritten(95, 100, 3)
+	assert.Equal(t, 2, list.size(), "two intervals")
+
+	list.MarkWritten(50, 60, 4)
+	assert.Equal(t, 3, list.size(), "three intervals")
+
+	list.MarkWritten(50, 55, 5)
+	assert.Equal(t, 4, list.size(), "three intervals merge")
+
+	list.MarkWritten(40, 50, 6)
+	assert.Equal(t, 5, list.size(), "three intervals grow forward")
+
+	list.MarkWritten(50, 65, 7)
+	assert.Equal(t, 4, list.size(), "three intervals grow backward")
+
+	list.MarkWritten(70, 80, 8)
+	assert.Equal(t, 5, list.size(), "four intervals")
+
+	list.MarkWritten(60, 70, 9)
+	assert.Equal(t, 6, list.size(), "three intervals merged")
+
+	list.MarkWritten(59, 71, 10)
+	assert.Equal(t, 6, list.size(), "covered three intervals")
+
+	list.MarkWritten(5, 59, 11)
+	assert.Equal(t, 5, list.size(), "covered two intervals")
+
+	list.MarkWritten(70, 99, 12)
+	assert.Equal(t, 5, list.size(), "covered one intervals")
+
+}
+
+type interval struct {
+	start    int64
+	stop     int64
+	expected bool
+}
+
+func Test_PageChunkWrittenIntervalList1(t *testing.T) {
+	list := newChunkWrittenIntervalList()
+	inputs := []interval{
+		{1, 5, true},
+		{2, 3, true},
+	}
+	for i, input := range inputs {
+		list.MarkWritten(input.start, input.stop, int64(i)+1)
+		actual := hasData(list, 0, 4)
+		if actual != input.expected {
+			t.Errorf("input [%d,%d) expected %v actual %v", input.start, input.stop, input.expected, actual)
+		}
+	}
+}
+
+func hasData(usage *ChunkWrittenIntervalList, chunkStartOffset, x int64) bool {
+	for t := usage.head.next; t != usage.tail; t = t.next {
+		logicStart := chunkStartOffset + t.StartOffset
+		logicStop := chunkStartOffset + t.stopOffset
+		if logicStart <= x && x < logicStop {
+			return true
+		}
+	}
+	return false
+}
+
+func TestIsContiguouslyWritten(t *testing.T) {
+	const chunkSize int64 = 2 * 1024 * 1024
+	cases := []struct {
+		name     string
+		writes   [][2]int64
+		expected bool
+	}{
+		{name: "empty", writes: nil, expected: false},
+		{name: "single full", writes: [][2]int64{{0, chunkSize}}, expected: true},
+		{name: "single partial at start", writes: [][2]int64{{0, chunkSize / 4}}, expected: true},
+		{name: "two adjacent halves in order", writes: [][2]int64{{0, chunkSize / 2}, {chunkSize / 2, chunkSize}}, expected: true},
+		{name: "two adjacent halves out of order", writes: [][2]int64{{chunkSize / 2, chunkSize}, {0, chunkSize / 2}}, expected: true},
+		{name: "overlapping covers everything", writes: [][2]int64{{0, chunkSize*3/4 + 1}, {chunkSize / 2, chunkSize}}, expected: true},
+		{name: "three adjacent thirds", writes: [][2]int64{{0, chunkSize / 3}, {chunkSize / 3, 2 * chunkSize / 3}, {2 * chunkSize / 3, chunkSize}}, expected: true},
+		{name: "single partial in middle", writes: [][2]int64{{chunkSize / 4, chunkSize / 2}}, expected: false},
+		{name: "first byte missing", writes: [][2]int64{{1, chunkSize}}, expected: false},
+		{name: "gap in middle", writes: [][2]int64{{0, chunkSize / 4}, {chunkSize / 2, chunkSize}}, expected: false},
+		{name: "two intervals with 1-byte gap", writes: [][2]int64{{0, 100}, {101, 200}}, expected: false},
+		{name: "three intervals with middle gap", writes: [][2]int64{{0, chunkSize / 4}, {chunkSize / 3, chunkSize / 2}, {chunkSize / 2, chunkSize}}, expected: false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			list := newChunkWrittenIntervalList()
+			for i, w := range tc.writes {
+				list.MarkWritten(w[0], w[1], int64(i+1))
+			}
+			assert.Equal(t, tc.expected, list.IsContiguouslyWritten(), "IsContiguouslyWritten mismatch")
+		})
+	}
+}
+
+func TestIsComplete_AdjacentIntervals(t *testing.T) {
+	// Linux FUSE delivers writes up to FUSE_MAX_PAGES_PER_REQ
+	// (typically 1 MiB) per op, so a 2 MiB chunk filled by sequential
+	// writes arrives as two adjacent 1 MiB writes. addInterval does
+	// not merge adjacent intervals into one, but IsComplete must
+	// still report the chunk as fully covered — otherwise
+	// maybeMoveToSealed never fires, the chunk stays writable
+	// forever, and the write buffer cap cannot drain. Regression for
+	// the TestWriteBufferCap Linux CI deadlock on PR #9066.
+	const chunkSize int64 = 2 * 1024 * 1024
+
+	cases := []struct {
+		name     string
+		writes   [][2]int64
+		expected bool
+	}{
+		{
+			name:     "single full write",
+			writes:   [][2]int64{{0, chunkSize}},
+			expected: true,
+		},
+		{
+			name:     "two adjacent halves in order",
+			writes:   [][2]int64{{0, chunkSize / 2}, {chunkSize / 2, chunkSize}},
+			expected: true,
+		},
+		{
+			name:     "two adjacent halves out of order",
+			writes:   [][2]int64{{chunkSize / 2, chunkSize}, {0, chunkSize / 2}},
+			expected: true,
+		},
+		{
+			name: "eight adjacent eighths",
+			writes: func() [][2]int64 {
+				const n = 8
+				step := chunkSize / n
+				out := make([][2]int64, 0, n)
+				for i := int64(0); i < n; i++ {
+					out = append(out, [2]int64{i * step, (i + 1) * step})
+				}
+				return out
+			}(),
+			expected: true,
+		},
+		{
+			name:     "gap in the middle",
+			writes:   [][2]int64{{0, chunkSize / 4}, {chunkSize / 2, chunkSize}},
+			expected: false,
+		},
+		{
+			name:     "missing last byte",
+			writes:   [][2]int64{{0, chunkSize - 1}},
+			expected: false,
+		},
+		{
+			name:     "missing first byte",
+			writes:   [][2]int64{{1, chunkSize}},
+			expected: false,
+		},
+		{
+			name:     "overlapping intervals covering everything",
+			writes:   [][2]int64{{0, chunkSize*3/4 + 1}, {chunkSize / 2, chunkSize}},
+			expected: true,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			list := newChunkWrittenIntervalList()
+			for i, w := range tc.writes {
+				list.MarkWritten(w[0], w[1], int64(i+1))
+			}
+			assert.Equal(t, tc.expected, list.IsComplete(chunkSize), "IsComplete mismatch")
+		})
+	}
+}

weed/mount/page_writer/dirty_pages.go

@@ -0,0 +1,26 @@
+package page_writer
+
+type DirtyPages interface {
+	AddPage(offset int64, data []byte, isSequential bool, tsNs int64) error
+	FlushData() error
+	HasWrites() bool
+	ReadDirtyDataAt(data []byte, startOffset int64, tsNs int64) (maxStop int64)
+	Destroy()
+	LockForRead(startOffset, stopOffset int64)
+	UnlockForRead(startOffset, stopOffset int64)
+	EvictOneWritableChunk() bool
+	ProactiveFlush(nowNs, idleThresholdNs, maxHoldNs, fillRatio int64, frontierLag int, isSequential bool) bool
+}
+
+func max(x, y int64) int64 {
+	if x > y {
+		return x
+	}
+	return y
+}
+func min(x, y int64) int64 {
+	if x < y {
+		return x
+	}
+	return y
+}

weed/mount/page_writer/page_chunk.go

@@ -0,0 +1,19 @@
+package page_writer
+
+import (
+	"io"
+)
+
+type SaveToStorageFunc func(reader io.Reader, offset int64, size int64, modifiedTsNs int64, cleanupFn func())
+
+type PageChunk interface {
+	FreeResource()
+	WriteDataAt(src []byte, offset int64, tsNs int64) (n int)
+	ReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64)
+	IsComplete() bool
+	IsContiguouslyWritten() bool
+	ActivityScore() int64
+	WrittenSize() int64
+	LastWriteTsNs() int64
+	SaveContent(saveFn SaveToStorageFunc)
+}

weed/mount/page_writer/page_chunk_mem.go

@@ -0,0 +1,127 @@
+package page_writer
+
+import (
+	"sync"
+	"sync/atomic"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/mem"
+)
+
+var (
+	_ = PageChunk(&MemChunk{})
+
+	memChunkCounter int64
+)
+
+type MemChunk struct {
+	sync.RWMutex
+	buf             []byte
+	usage           *ChunkWrittenIntervalList
+	chunkSize       int64
+	logicChunkIndex LogicChunkIndex
+	activityScore   *ActivityScore
+	lastWriteTsNs   atomic.Int64
+}
+
+func NewMemChunk(logicChunkIndex LogicChunkIndex, chunkSize int64) *MemChunk {
+	atomic.AddInt64(&memChunkCounter, 1)
+	return &MemChunk{
+		logicChunkIndex: logicChunkIndex,
+		chunkSize:       chunkSize,
+		buf:             mem.Allocate(int(chunkSize)),
+		usage:           newChunkWrittenIntervalList(),
+		activityScore:   NewActivityScore(),
+	}
+}
+
+func (mc *MemChunk) FreeResource() {
+	mc.Lock()
+	defer mc.Unlock()
+
+	atomic.AddInt64(&memChunkCounter, -1)
+	mem.Free(mc.buf)
+}
+
+func (mc *MemChunk) WriteDataAt(src []byte, offset int64, tsNs int64) (n int) {
+	mc.Lock()
+	defer mc.Unlock()
+
+	innerOffset := offset % mc.chunkSize
+	n = copy(mc.buf[innerOffset:], src)
+	mc.usage.MarkWritten(innerOffset, innerOffset+int64(n), tsNs)
+	mc.activityScore.MarkWrite()
+	mc.lastWriteTsNs.Store(tsNs)
+
+	return
+}
+
+func (mc *MemChunk) ReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64) {
+	mc.RLock()
+	defer mc.RUnlock()
+
+	memChunkBaseOffset := int64(mc.logicChunkIndex) * mc.chunkSize
+	for t := mc.usage.head.next; t != mc.usage.tail; t = t.next {
+		logicStart := max(off, memChunkBaseOffset+t.StartOffset)
+		logicStop := min(off+int64(len(p)), memChunkBaseOffset+t.stopOffset)
+		if logicStart < logicStop {
+			copy(p[logicStart-off:logicStop-off], mc.buf[logicStart-memChunkBaseOffset:logicStop-memChunkBaseOffset])
+			maxStop = max(maxStop, logicStop)
+		}
+	}
+	mc.activityScore.MarkRead()
+
+	return
+}
+
+func (mc *MemChunk) IsComplete() bool {
+	mc.RLock()
+	defer mc.RUnlock()
+
+	return mc.usage.IsComplete(mc.chunkSize)
+}
+
+func (mc *MemChunk) IsContiguouslyWritten() bool {
+	mc.RLock()
+	defer mc.RUnlock()
+
+	return mc.usage.IsContiguouslyWritten()
+}
+
+func (mc *MemChunk) ActivityScore() int64 {
+	return mc.activityScore.ActivityScore()
+}
+
+func (mc *MemChunk) WrittenSize() int64 {
+	mc.RLock()
+	defer mc.RUnlock()
+
+	return mc.usage.WrittenSize()
+}
+
+func (mc *MemChunk) LastWriteTsNs() int64 {
+	return mc.lastWriteTsNs.Load()
+}
+
+func (mc *MemChunk) SaveContent(saveFn SaveToStorageFunc) {
+	mc.RLock()
+	defer mc.RUnlock()
+
+	if saveFn == nil {
+		return
+	}
+
+	for t := mc.usage.head.next; t != mc.usage.tail; t = t.next {
+		startOffset := t.StartOffset
+		stopOffset := t.stopOffset
+		tsNs := t.TsNs
+		for t != mc.usage.tail && t.next.StartOffset == stopOffset {
+			stopOffset = t.next.stopOffset
+			t = t.next
+			tsNs = max(tsNs, t.TsNs)
+		}
+		reader := util.NewBytesReader(mc.buf[startOffset:stopOffset])
+		saveFn(reader, int64(mc.logicChunkIndex)*mc.chunkSize+startOffset, stopOffset-startOffset, tsNs, func() {
+		})
+	}
+}

weed/mount/page_writer/page_chunk_swapfile.go

@@ -0,0 +1,223 @@
+package page_writer
+
+import (
+	"io"
+	"os"
+	"sync"
+	"sync/atomic"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/mem"
+)
+
+var (
+	_ = PageChunk(&SwapFileChunk{})
+)
+
+type ActualChunkIndex int
+
+type SwapFile struct {
+	dir                 string
+	file                *os.File
+	chunkSize           int64
+	chunkTrackingLock   sync.Mutex
+	activeChunkCount    int
+	freeActualChunkList []ActualChunkIndex
+}
+
+type SwapFileChunk struct {
+	sync.RWMutex
+	swapfile         *SwapFile
+	usage            *ChunkWrittenIntervalList
+	logicChunkIndex  LogicChunkIndex
+	actualChunkIndex ActualChunkIndex
+	activityScore    *ActivityScore
+	lastWriteTsNs    atomic.Int64
+	//memChunk         *MemChunk
+}
+
+func NewSwapFile(dir string, chunkSize int64) *SwapFile {
+	return &SwapFile{
+		dir:       dir,
+		file:      nil,
+		chunkSize: chunkSize,
+	}
+}
+func (sf *SwapFile) FreeResource() {
+	sf.chunkTrackingLock.Lock()
+	defer sf.chunkTrackingLock.Unlock()
+	if sf.file != nil {
+		sf.file.Close()
+		os.Remove(sf.file.Name())
+		sf.file = nil
+	}
+}
+
+func (sf *SwapFile) NewSwapFileChunk(logicChunkIndex LogicChunkIndex) (tc *SwapFileChunk) {
+	sf.chunkTrackingLock.Lock()
+	defer sf.chunkTrackingLock.Unlock()
+
+	if sf.file == nil {
+		var err error
+		sf.file, err = os.CreateTemp(sf.dir, "")
+		if os.IsNotExist(err) {
+			if mkdirErr := os.MkdirAll(sf.dir, 0700); mkdirErr != nil {
+				glog.Errorf("create/recreate swap directory %s: %v", sf.dir, mkdirErr)
+				return nil
+			}
+			sf.file, err = os.CreateTemp(sf.dir, "")
+		}
+		if err != nil {
+			glog.Errorf("create swap file in %s: %v", sf.dir, err)
+			return nil
+		}
+	}
+
+	sf.activeChunkCount++
+
+	// assign a new physical chunk
+	var actualChunkIndex ActualChunkIndex
+	if len(sf.freeActualChunkList) > 0 {
+		actualChunkIndex = sf.freeActualChunkList[0]
+		sf.freeActualChunkList = sf.freeActualChunkList[1:]
+	} else {
+		actualChunkIndex = ActualChunkIndex(sf.activeChunkCount)
+	}
+
+	swapFileChunk := &SwapFileChunk{
+		swapfile:         sf,
+		usage:            newChunkWrittenIntervalList(),
+		logicChunkIndex:  logicChunkIndex,
+		actualChunkIndex: actualChunkIndex,
+		activityScore:    NewActivityScore(),
+		// memChunk:         NewMemChunk(logicChunkIndex, sf.chunkSize),
+	}
+
+	// println(logicChunkIndex, "|", "++++", swapFileChunk.actualChunkIndex, swapFileChunk, sf)
+	return swapFileChunk
+}
+
+func (sc *SwapFileChunk) FreeResource() {
+
+	sc.Lock()
+	defer sc.Unlock()
+
+	sc.swapfile.chunkTrackingLock.Lock()
+	defer sc.swapfile.chunkTrackingLock.Unlock()
+
+	sc.swapfile.freeActualChunkList = append(sc.swapfile.freeActualChunkList, sc.actualChunkIndex)
+	sc.swapfile.activeChunkCount--
+	// println(sc.logicChunkIndex, "|", "----", sc.actualChunkIndex, sc, sc.swapfile)
+}
+
+func (sc *SwapFileChunk) WriteDataAt(src []byte, offset int64, tsNs int64) (n int) {
+	sc.Lock()
+	defer sc.Unlock()
+
+	// println(sc.logicChunkIndex, "|", tsNs, "write at", offset, len(src), sc.actualChunkIndex)
+
+	innerOffset := offset % sc.swapfile.chunkSize
+	var err error
+	n, err = sc.swapfile.file.WriteAt(src, int64(sc.actualChunkIndex)*sc.swapfile.chunkSize+innerOffset)
+	sc.usage.MarkWritten(innerOffset, innerOffset+int64(n), tsNs)
+	if err != nil {
+		glog.Errorf("failed to write swap file %s: %v", sc.swapfile.file.Name(), err)
+	}
+	//sc.memChunk.WriteDataAt(src, offset, tsNs)
+	sc.activityScore.MarkWrite()
+	sc.lastWriteTsNs.Store(tsNs)
+
+	return
+}
+
+func (sc *SwapFileChunk) ReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64) {
+	sc.RLock()
+	defer sc.RUnlock()
+
+	// println(sc.logicChunkIndex, "|", tsNs, "read at", off, len(p), sc.actualChunkIndex)
+
+	//memCopy := make([]byte, len(p))
+	//copy(memCopy, p)
+
+	chunkStartOffset := int64(sc.logicChunkIndex) * sc.swapfile.chunkSize
+	for t := sc.usage.head.next; t != sc.usage.tail; t = t.next {
+		logicStart := max(off, chunkStartOffset+t.StartOffset)
+		logicStop := min(off+int64(len(p)), chunkStartOffset+t.stopOffset)
+		if logicStart < logicStop {
+			actualStart := logicStart - chunkStartOffset + int64(sc.actualChunkIndex)*sc.swapfile.chunkSize
+			if n, err := sc.swapfile.file.ReadAt(p[logicStart-off:logicStop-off], actualStart); err != nil {
+				if err == io.EOF && n == int(logicStop-logicStart) {
+					err = nil
+				}
+				glog.Errorf("failed to reading swap file %s: %v", sc.swapfile.file.Name(), err)
+				break
+			}
+			maxStop = max(maxStop, logicStop)
+		}
+	}
+	//sc.memChunk.ReadDataAt(memCopy, off, tsNs)
+	//if bytes.Compare(memCopy, p) != 0 {
+	//	println("read wrong data from swap file", off, sc.logicChunkIndex)
+	//}
+
+	sc.activityScore.MarkRead()
+
+	return
+}
+
+func (sc *SwapFileChunk) IsComplete() bool {
+	sc.RLock()
+	defer sc.RUnlock()
+	return sc.usage.IsComplete(sc.swapfile.chunkSize)
+}
+
+func (sc *SwapFileChunk) IsContiguouslyWritten() bool {
+	sc.RLock()
+	defer sc.RUnlock()
+	return sc.usage.IsContiguouslyWritten()
+}
+
+func (sc *SwapFileChunk) ActivityScore() int64 {
+	return sc.activityScore.ActivityScore()
+}
+
+func (sc *SwapFileChunk) WrittenSize() int64 {
+	sc.RLock()
+	defer sc.RUnlock()
+	return sc.usage.WrittenSize()
+}
+
+func (sc *SwapFileChunk) LastWriteTsNs() int64 {
+	return sc.lastWriteTsNs.Load()
+}
+
+func (sc *SwapFileChunk) SaveContent(saveFn SaveToStorageFunc) {
+	sc.RLock()
+	defer sc.RUnlock()
+
+	if saveFn == nil {
+		return
+	}
+	// println(sc.logicChunkIndex, "|", "save")
+	for t := sc.usage.head.next; t != sc.usage.tail; t = t.next {
+		startOffset := t.StartOffset
+		stopOffset := t.stopOffset
+		tsNs := t.TsNs
+		for t != sc.usage.tail && t.next.StartOffset == stopOffset {
+			stopOffset = t.next.stopOffset
+			t = t.next
+			tsNs = max(tsNs, t.TsNs)
+		}
+
+		data := mem.Allocate(int(stopOffset - startOffset))
+		n, _ := sc.swapfile.file.ReadAt(data, startOffset+int64(sc.actualChunkIndex)*sc.swapfile.chunkSize)
+		if n > 0 {
+			reader := util.NewBytesReader(data[:n])
+			saveFn(reader, int64(sc.logicChunkIndex)*sc.swapfile.chunkSize+startOffset, int64(n), tsNs, func() {
+			})
+		}
+		mem.Free(data)
+	}
+
+}

weed/mount/page_writer/page_chunk_swapfile_test.go

@@ -0,0 +1,105 @@
+package page_writer
+
+import (
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"testing"
+)
+
+func TestSwapFile_NewSwapFileChunk_Concurrent(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", "seaweedfs_swap_test")
+	if err != nil {
+		t.Fatalf("failed to create temp dir: %v", err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	sf := NewSwapFile(filepath.Join(tempDir, "swap"), 1024)
+	defer sf.FreeResource()
+
+	var wg sync.WaitGroup
+	var failures atomic.Uint32
+	numConcurrent := 10
+	for i := 0; i < numConcurrent; i++ {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			chunk := sf.NewSwapFileChunk(LogicChunkIndex(idx))
+			if chunk == nil {
+				failures.Add(1)
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	if failures.Load() > 0 {
+		t.Errorf("failed to create %d chunks", failures.Load())
+	}
+
+	if sf.file == nil {
+		t.Error("sf.file should not be nil")
+	}
+}
+
+func TestSwapFile_MkdirAll_Permissions(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", "seaweedfs_swap_perm_test")
+	if err != nil {
+		t.Fatalf("failed to create temp dir: %v", err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	swapDir := filepath.Join(tempDir, "swap_subdir")
+	sf := NewSwapFile(swapDir, 1024)
+	defer sf.FreeResource()
+
+	// This should trigger os.MkdirAll
+	if sf.NewSwapFileChunk(0) == nil {
+		t.Fatal("NewSwapFileChunk failed to create a chunk")
+	}
+
+	info, err := os.Stat(swapDir)
+	if err != nil {
+		t.Fatalf("failed to stat swap dir: %v", err)
+	}
+
+	if !info.IsDir() {
+		t.Errorf("expected %s to be a directory", swapDir)
+	}
+
+	// Check permissions - should be 0700
+	if info.Mode().Perm() != 0700 {
+		t.Errorf("expected permissions 0700, got %o", info.Mode().Perm())
+	}
+}
+
+func TestSwapFile_RecreateDir(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", "seaweedfs_swap_recreate_test")
+	if err != nil {
+		t.Fatalf("failed to create temp dir: %v", err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	swapDir := filepath.Join(tempDir, "swap_recreate")
+	sf := NewSwapFile(swapDir, 1024)
+
+	// Create first chunk
+	if sf.NewSwapFileChunk(0) == nil {
+		t.Fatal("NewSwapFileChunk failed to create the first chunk")
+	}
+	sf.FreeResource()
+
+	// Delete the directory
+	os.RemoveAll(swapDir)
+
+	// Create second chunk - should recreate directory
+	chunk := sf.NewSwapFileChunk(1)
+	if chunk == nil {
+		t.Error("failed to create chunk after directory deletion")
+	}
+
+	if _, err := os.Stat(swapDir); os.IsNotExist(err) {
+		t.Error("swap directory was not recreated")
+	}
+	sf.FreeResource()
+}

weed/mount/page_writer/upload_pipeline.go

@@ -0,0 +1,402 @@
+package page_writer
+
+import (
+	"fmt"
+	"math"
+	"sync"
+	"sync/atomic"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type LogicChunkIndex int
+
+type UploadPipeline struct {
+	uploaderCount       int32
+	uploaderCountCond   *sync.Cond
+	filepath            util.FullPath
+	ChunkSize           int64
+	uploaders           *util.LimitedConcurrentExecutor
+	saveToStorageFn     SaveToStorageFunc
+	writableChunkLimit  int
+	concurrentWriterMax int32
+	swapFile            *SwapFile
+	chunksLock          sync.Mutex
+	writableChunks      map[LogicChunkIndex]PageChunk
+	sealedChunks        map[LogicChunkIndex]*SealedChunk
+	activeReadChunks    map[LogicChunkIndex]int
+	readerCountCond     *sync.Cond
+	accountant          *WriteBufferAccountant
+	lastWriteChunkIndex int64 // atomic: highest LogicChunkIndex written
+}
+
+type SealedChunk struct {
+	chunk            PageChunk
+	referenceCounter int // track uploading or reading processes
+	// accountant and chunkSize are captured when the chunk is sealed so
+	// FreeReference can release the global write-budget slot exactly once
+	// regardless of which code path triggers the final deref (normal
+	// upload completion, Shutdown, or replace-in-place in moveToSealed).
+	accountant *WriteBufferAccountant
+	chunkSize  int64
+}
+
+func (sc *SealedChunk) FreeReference(messageOnFree string) {
+	// Early-return guard so repeated calls (Shutdown racing the async
+	// uploader, or any future caller that loses track of ownership) are
+	// strict no-ops rather than driving referenceCounter negative.
+	if sc.referenceCounter <= 0 {
+		return
+	}
+	sc.referenceCounter--
+	if sc.referenceCounter == 0 {
+		glog.V(4).Infof("Free sealed chunk: %s", messageOnFree)
+		sc.chunk.FreeResource()
+		sc.accountant.Release(sc.chunkSize)
+	}
+}
+
+// NewUploadPipeline constructs an UploadPipeline. accountant may be nil,
+// in which case no global write-buffer cap is enforced. When non-nil,
+// creating a new page chunk (memory or swap) first reserves ChunkSize
+// bytes against it, blocking the writer if the global cap is reached.
+// The accountant is captured at construction so the pipeline's hot paths
+// (SaveDataAt, moveToSealed, Shutdown) can read up.accountant without
+// any synchronization.
+func NewUploadPipeline(writers *util.LimitedConcurrentExecutor, chunkSize int64, saveToStorageFn SaveToStorageFunc, bufferChunkLimit int, swapFileDir string, accountant *WriteBufferAccountant) *UploadPipeline {
+	t := &UploadPipeline{
+		ChunkSize:           chunkSize,
+		writableChunks:      make(map[LogicChunkIndex]PageChunk),
+		sealedChunks:        make(map[LogicChunkIndex]*SealedChunk),
+		uploaders:           writers,
+		uploaderCountCond:   sync.NewCond(&sync.Mutex{}),
+		saveToStorageFn:     saveToStorageFn,
+		activeReadChunks:    make(map[LogicChunkIndex]int),
+		writableChunkLimit:  bufferChunkLimit,
+		concurrentWriterMax: int32(bufferChunkLimit),
+		swapFile:            NewSwapFile(swapFileDir, chunkSize),
+		accountant:          accountant,
+	}
+	t.readerCountCond = sync.NewCond(&t.chunksLock)
+	return t
+}
+
+func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool, tsNs int64) (n int, err error) {
+
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+
+	logicChunkIndex := LogicChunkIndex(off / up.ChunkSize)
+
+	// track write frontier for proactive flushing (CAS to avoid regression)
+	for {
+		old := atomic.LoadInt64(&up.lastWriteChunkIndex)
+		if int64(logicChunkIndex) <= old {
+			break
+		}
+		if atomic.CompareAndSwapInt64(&up.lastWriteChunkIndex, old, int64(logicChunkIndex)) {
+			break
+		}
+	}
+
+	pageChunk, found := up.writableChunks[logicChunkIndex]
+	if !found {
+		// Reserve a chunk-sized slot against the global write budget before
+		// allocating. Reserve may block when volumes are full and sealed
+		// chunks can't drain, so we must release chunksLock first — the
+		// uploader goroutines that eventually call Release take chunksLock.
+		if up.accountant != nil {
+			up.chunksLock.Unlock()
+			up.accountant.Reserve(up.ChunkSize)
+			up.chunksLock.Lock()
+			// Re-check: another writer on the same file may have created
+			// the chunk while we were blocked. If so, give the slot back.
+			if pageChunk, found = up.writableChunks[logicChunkIndex]; found {
+				up.accountant.Release(up.ChunkSize)
+			}
+		}
+	}
+	if !found {
+		if len(up.writableChunks) > up.writableChunkLimit {
+			// Per-pipeline soft cap. Seal the fullest gap-free chunk;
+			// gappy chunks must wait so in-flight FUSE writeback can fill
+			// the gap (issue #9330). If none qualifies, fall through —
+			// the memChunkCounter ceiling below redirects to swap.
+			candidateChunkIndex, fullness := LogicChunkIndex(-1), int64(0)
+			for lci, mc := range up.writableChunks {
+				if !mc.IsContiguouslyWritten() {
+					continue
+				}
+				if b := mc.WrittenSize(); b > fullness {
+					candidateChunkIndex = lci
+					fullness = b
+				}
+			}
+			if candidateChunkIndex >= 0 {
+				up.moveToSealed(up.writableChunks[candidateChunkIndex], candidateChunkIndex)
+			}
+		}
+		// fmt.Printf("isSequential:%v len(up.writableChunks):%v memChunkCounter:%v", isSequential, len(up.writableChunks), memChunkCounter)
+		if isSequential &&
+			len(up.writableChunks) < up.writableChunkLimit &&
+			atomic.LoadInt64(&memChunkCounter) < 4*int64(up.writableChunkLimit) {
+			pageChunk = NewMemChunk(logicChunkIndex, up.ChunkSize)
+			// fmt.Printf(" create mem  chunk %d\n", logicChunkIndex)
+		} else {
+			pageChunk = up.swapFile.NewSwapFileChunk(logicChunkIndex)
+			// fmt.Printf(" create file chunk %d\n", logicChunkIndex)
+			if pageChunk == nil {
+				up.accountant.Release(up.ChunkSize)
+				return 0, fmt.Errorf("failed to create swap file chunk")
+			}
+		}
+		up.writableChunks[logicChunkIndex] = pageChunk
+	}
+	//if _, foundSealed := up.sealedChunks[logicChunkIndex]; foundSealed {
+	//	println("found already sealed chunk", logicChunkIndex)
+	//}
+	//if _, foundReading := up.activeReadChunks[logicChunkIndex]; foundReading {
+	//	println("found active read chunk", logicChunkIndex)
+	//}
+	n = pageChunk.WriteDataAt(p, off, tsNs)
+	up.maybeMoveToSealed(pageChunk, logicChunkIndex)
+
+	return
+}
+
+func (up *UploadPipeline) MaybeReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64) {
+	logicChunkIndex := LogicChunkIndex(off / up.ChunkSize)
+
+	up.chunksLock.Lock()
+	defer func() {
+		up.readerCountCond.Signal()
+		up.chunksLock.Unlock()
+	}()
+
+	// read from sealed chunks first
+	sealedChunk, found := up.sealedChunks[logicChunkIndex]
+	if found {
+		maxStop = sealedChunk.chunk.ReadDataAt(p, off, tsNs)
+		glog.V(4).Infof("%s read sealed memchunk [%d,%d)", up.filepath, off, maxStop)
+	}
+
+	// read from writable chunks last
+	writableChunk, found := up.writableChunks[logicChunkIndex]
+	if !found {
+		return
+	}
+	writableMaxStop := writableChunk.ReadDataAt(p, off, tsNs)
+	glog.V(4).Infof("%s read writable memchunk [%d,%d)", up.filepath, off, writableMaxStop)
+	maxStop = max(maxStop, writableMaxStop)
+
+	return
+}
+
+func (up *UploadPipeline) FlushAll() {
+	up.flushChunks()
+	up.waitForCurrentWritersToComplete()
+}
+
+func (up *UploadPipeline) flushChunks() {
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+
+	for logicChunkIndex, memChunk := range up.writableChunks {
+		up.moveToSealed(memChunk, logicChunkIndex)
+	}
+}
+
+func (up *UploadPipeline) maybeMoveToSealed(memChunk PageChunk, logicChunkIndex LogicChunkIndex) {
+	if memChunk.IsComplete() {
+		up.moveToSealed(memChunk, logicChunkIndex)
+	}
+}
+
+func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex LogicChunkIndex) {
+	newCount := atomic.AddInt32(&up.uploaderCount, 1)
+	glog.V(4).Infof("%s uploaderCount %d ++> %d", up.filepath, newCount-1, newCount)
+
+	if oldMemChunk, found := up.sealedChunks[logicChunkIndex]; found {
+		oldMemChunk.FreeReference(fmt.Sprintf("%s replace chunk %d", up.filepath, logicChunkIndex))
+	}
+	sealedChunk := &SealedChunk{
+		chunk: memChunk,
+		// One for the sealedChunks slot, one for the upload below, which
+		// drops its own. Execute() returns before SaveContent reads the
+		// chunk, so a later seal must not free the buffer under it.
+		referenceCounter: 2,
+		accountant:       up.accountant,
+		chunkSize:        up.ChunkSize,
+	}
+	up.sealedChunks[logicChunkIndex] = sealedChunk
+	delete(up.writableChunks, logicChunkIndex)
+
+	// unlock before submitting the uploading jobs
+	up.chunksLock.Unlock()
+	up.uploaders.Execute(func() {
+		// first add to the file chunks
+		sealedChunk.chunk.SaveContent(up.saveToStorageFn)
+
+		// notify waiting process
+		newCount := atomic.AddInt32(&up.uploaderCount, -1)
+		glog.V(4).Infof("%s uploaderCount %d --> %d", up.filepath, newCount+1, newCount)
+		// Lock and Unlock are not required,
+		// but it may signal multiple times during one wakeup,
+		// and the waiting goroutine may miss some of them!
+		up.uploaderCountCond.L.Lock()
+		up.uploaderCountCond.Broadcast()
+		up.uploaderCountCond.L.Unlock()
+
+		// wait for readers
+		up.chunksLock.Lock()
+		defer up.chunksLock.Unlock()
+		for up.IsLocked(logicChunkIndex) {
+			up.readerCountCond.Wait()
+		}
+
+		// then remove from sealed chunks, unless a later seal took over the
+		// index — deleting that one would hide its dirty pages from readers
+		if up.sealedChunks[logicChunkIndex] == sealedChunk {
+			delete(up.sealedChunks, logicChunkIndex)
+			sealedChunk.FreeReference(fmt.Sprintf("%s unindex chunk %d", up.filepath, logicChunkIndex))
+		}
+		sealedChunk.FreeReference(fmt.Sprintf("%s finished uploading chunk %d", up.filepath, logicChunkIndex))
+
+	})
+	up.chunksLock.Lock()
+}
+
+// EvictOneWritableChunk force-seals one writable chunk so the accountant's
+// Reserve loop can make progress. Strict pass picks the fullest gap-free
+// chunk (issue #9330). Fallback picks the oldest non-empty writer when
+// nothing is gap-free; without it Reserve deadlocks (cond.Wait only wakes
+// on Release, Release only fires on upload completion). Callers must not
+// hold up.chunksLock.
+func (up *UploadPipeline) EvictOneWritableChunk() bool {
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+	if len(up.writableChunks) == 0 {
+		return false
+	}
+
+	bestIndex := LogicChunkIndex(-1)
+	var bestBytes int64
+	for lci, wc := range up.writableChunks {
+		if !wc.IsContiguouslyWritten() {
+			continue
+		}
+		if b := wc.WrittenSize(); b > bestBytes {
+			bestIndex = lci
+			bestBytes = b
+		}
+	}
+
+	if bestIndex < 0 {
+		oldestTsNs := int64(math.MaxInt64)
+		var fallbackBytes int64
+		for lci, wc := range up.writableChunks {
+			b := wc.WrittenSize()
+			if b == 0 {
+				continue
+			}
+			ts := wc.LastWriteTsNs()
+			if ts == 0 {
+				continue
+			}
+			if ts < oldestTsNs || (ts == oldestTsNs && b > fallbackBytes) {
+				bestIndex = lci
+				oldestTsNs = ts
+				fallbackBytes = b
+			}
+		}
+	}
+
+	if bestIndex < 0 {
+		return false
+	}
+	up.moveToSealed(up.writableChunks[bestIndex], bestIndex)
+	return true
+}
+
+// ProactiveFlush seals at most one idle writable chunk that is unlikely to
+// receive further writes, submitting it for async upload. Returns true if a
+// chunk was sealed. The caller (ChunkFlusher) invokes this periodically so
+// that partially-written chunks drain continuously instead of piling up
+// until fsync.
+func (up *UploadPipeline) ProactiveFlush(nowNs int64, idleThresholdNs int64, maxHoldNs int64, fillRatio int64, frontierLag int, isSequential bool) bool {
+	if up.concurrentWriterMax <= 0 || atomic.LoadInt32(&up.uploaderCount)*2 >= up.concurrentWriterMax {
+		return false
+	}
+
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+
+	if len(up.writableChunks) == 0 {
+		return false
+	}
+
+	frontier := atomic.LoadInt64(&up.lastWriteChunkIndex)
+	isSeq := isSequential
+
+	var bestIdx LogicChunkIndex = -1
+	var bestBytes int64 = -1
+
+	for lci, chunk := range up.writableChunks {
+		lastWrite := chunk.LastWriteTsNs()
+		if lastWrite == 0 {
+			continue
+		}
+		age := nowNs - lastWrite
+		if age < idleThresholdNs {
+			continue
+		}
+		// Skip gappy chunks; FUSE writeback may still be in flight for the
+		// hole and SaveContent would bake it into split volume chunks
+		// (issue #9330). Failing here is just a missed flush — FlushAll
+		// at close still seals everything.
+		if !chunk.IsContiguouslyWritten() {
+			continue
+		}
+		written := chunk.WrittenSize()
+		nearlyFull := written >= fillRatio
+		behindFrontier := isSeq && int64(lci) <= frontier-int64(frontierLag)
+		stale := age >= maxHoldNs
+
+		if !nearlyFull && !behindFrontier && !stale {
+			continue
+		}
+		if written > bestBytes {
+			bestIdx = lci
+			bestBytes = written
+		}
+	}
+	if bestIdx < 0 {
+		return false
+	}
+	glog.V(3).Infof("%s proactive flush chunk %d (%d bytes written)", up.filepath, bestIdx, bestBytes)
+	up.moveToSealed(up.writableChunks[bestIdx], bestIdx)
+	return true
+}
+
+func (up *UploadPipeline) Shutdown() {
+	up.swapFile.FreeResource()
+
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+	// Free any writable chunks that were reserved but never sealed — on the
+	// Destroy() path (truncate / metadata invalidation) there is no
+	// preceding FlushData(), so dirty writable chunks would otherwise leak
+	// both their memory and their write-budget slots.
+	for logicChunkIndex, writableChunk := range up.writableChunks {
+		glog.V(4).Infof("%s uploadpipeline shutdown writable chunk %d", up.filepath, logicChunkIndex)
+		writableChunk.FreeResource()
+		up.accountant.Release(up.ChunkSize)
+		delete(up.writableChunks, logicChunkIndex)
+	}
+	for logicChunkIndex, sealedChunk := range up.sealedChunks {
+		// only the slot reference; an in-flight upload frees the chunk itself
+		sealedChunk.FreeReference(fmt.Sprintf("%s uploadpipeline shutdown chunk %d", up.filepath, logicChunkIndex))
+		delete(up.sealedChunks, logicChunkIndex)
+	}
+}

weed/mount/page_writer/upload_pipeline_lock.go

@@ -0,0 +1,61 @@
+package page_writer
+
+import (
+	"sync/atomic"
+)
+
+func (up *UploadPipeline) LockForRead(startOffset, stopOffset int64) {
+	startLogicChunkIndex := LogicChunkIndex(startOffset / up.ChunkSize)
+	stopLogicChunkIndex := LogicChunkIndex(stopOffset / up.ChunkSize)
+	if stopOffset%up.ChunkSize > 0 {
+		stopLogicChunkIndex += 1
+	}
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+	for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ {
+		if count, found := up.activeReadChunks[i]; found {
+			up.activeReadChunks[i] = count + 1
+		} else {
+			up.activeReadChunks[i] = 1
+		}
+	}
+}
+
+func (up *UploadPipeline) UnlockForRead(startOffset, stopOffset int64) {
+	startLogicChunkIndex := LogicChunkIndex(startOffset / up.ChunkSize)
+	stopLogicChunkIndex := LogicChunkIndex(stopOffset / up.ChunkSize)
+	if stopOffset%up.ChunkSize > 0 {
+		stopLogicChunkIndex += 1
+	}
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+	for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ {
+		if count, found := up.activeReadChunks[i]; found {
+			if count == 1 {
+				delete(up.activeReadChunks, i)
+			} else {
+				up.activeReadChunks[i] = count - 1
+			}
+		}
+	}
+}
+
+func (up *UploadPipeline) IsLocked(logicChunkIndex LogicChunkIndex) bool {
+	if count, found := up.activeReadChunks[logicChunkIndex]; found {
+		return count > 0
+	}
+	return false
+}
+
+func (up *UploadPipeline) waitForCurrentWritersToComplete() {
+	up.uploaderCountCond.L.Lock()
+	t := int32(100)
+	for {
+		t = atomic.LoadInt32(&up.uploaderCount)
+		if t <= 0 {
+			break
+		}
+		up.uploaderCountCond.Wait()
+	}
+	up.uploaderCountCond.L.Unlock()
+}

weed/mount/page_writer/upload_pipeline_reseal_test.go

@@ -0,0 +1,121 @@
+package page_writer
+
+import (
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// gatedChunk parks in SaveContent until the test opens its gate: the state
+// moveToSealed leaves an upload in, submitted but not yet reading its buffer.
+type gatedChunk struct {
+	gate        chan struct{}
+	saved       chan struct{}
+	freedBefore bool // FreeResource had already run when SaveContent began
+	freeCount   atomic.Int32
+}
+
+func newGatedChunk() *gatedChunk {
+	return &gatedChunk{
+		gate:  make(chan struct{}),
+		saved: make(chan struct{}),
+	}
+}
+
+func (c *gatedChunk) FreeResource()                         { c.freeCount.Add(1) }
+func (c *gatedChunk) WriteDataAt([]byte, int64, int64) int  { return 0 }
+func (c *gatedChunk) ReadDataAt([]byte, int64, int64) int64 { return 0 }
+func (c *gatedChunk) IsComplete() bool                      { return false }
+func (c *gatedChunk) IsContiguouslyWritten() bool           { return true }
+func (c *gatedChunk) ActivityScore() int64                  { return 0 }
+func (c *gatedChunk) WrittenSize() int64                    { return 0 }
+func (c *gatedChunk) LastWriteTsNs() int64                  { return 0 }
+func (c *gatedChunk) SaveContent(saveFn SaveToStorageFunc) {
+	<-c.gate
+	c.freedBefore = c.freeCount.Load() > 0
+	close(c.saved)
+}
+
+// Sealing the same index twice must not free the first chunk: freeing it
+// returns a live buffer to the slot pool and the in-flight upload then ships
+// whatever the next NewMemChunk writes there.
+func TestMoveToSealed_ReplaceKeepsUploadReference(t *testing.T) {
+	up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), 2*1024*1024, nil, 16, "", nil)
+
+	first, second := newGatedChunk(), newGatedChunk()
+
+	up.chunksLock.Lock()
+	up.moveToSealed(first, 0)
+	up.moveToSealed(second, 0) // takes over index 0 while first is uploading
+	up.chunksLock.Unlock()
+
+	// both seals are done, so any premature free has already happened
+	close(first.gate)
+	select {
+	case <-first.saved:
+	case <-time.After(5 * time.Second):
+		t.Fatal("first chunk's SaveContent never ran")
+	}
+	if first.freedBefore {
+		t.Error("first chunk was freed before its upload read it")
+	}
+
+	// a finished upload must leave the newer chunk indexed for readers
+	waitForFree(t, first)
+	up.chunksLock.Lock()
+	indexed := up.sealedChunks[0]
+	up.chunksLock.Unlock()
+	if indexed == nil || indexed.chunk != second {
+		t.Errorf("index 0 should still hold the second chunk, got %v", indexed)
+	}
+
+	close(second.gate)
+	waitForFree(t, second)
+	if got := first.freeCount.Load(); got != 1 {
+		t.Errorf("first chunk freed %d times, want 1", got)
+	}
+	if got := second.freeCount.Load(); got != 1 {
+		t.Errorf("second chunk freed %d times, want 1", got)
+	}
+}
+
+// Shutdown drops only the slot reference; the in-flight upload keeps the
+// chunk alive until it is done, then frees it exactly once.
+func TestShutdown_LeavesInFlightUploadItsReference(t *testing.T) {
+	up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), 2*1024*1024, nil, 16, t.TempDir(), nil)
+
+	chunk := newGatedChunk()
+	up.chunksLock.Lock()
+	up.moveToSealed(chunk, 0)
+	up.chunksLock.Unlock()
+
+	up.Shutdown()
+
+	close(chunk.gate)
+	select {
+	case <-chunk.saved:
+	case <-time.After(5 * time.Second):
+		t.Fatal("SaveContent never ran")
+	}
+	if chunk.freedBefore {
+		t.Error("Shutdown freed the chunk before its upload read it")
+	}
+	waitForFree(t, chunk)
+	if got := chunk.freeCount.Load(); got != 1 {
+		t.Errorf("chunk freed %d times, want 1", got)
+	}
+}
+
+func waitForFree(t *testing.T, c *gatedChunk) {
+	t.Helper()
+	<-c.saved
+	deadline := time.Now().Add(5 * time.Second)
+	for c.freeCount.Load() == 0 {
+		if time.Now().After(deadline) {
+			t.Fatal("chunk was never freed after its upload finished")
+		}
+		time.Sleep(time.Millisecond)
+	}
+}

weed/mount/page_writer/upload_pipeline_shutdown_test.go

@@ -0,0 +1,61 @@
+package page_writer
+
+import (
+	"io"
+	"testing"
+)
+
+// TestShutdown_ReleasesWritableReservation verifies that calling Shutdown
+// on a pipeline with dirty-but-unsealed writable chunks returns those
+// chunks' reservations to the accountant. Regression for the Destroy()
+// path (truncate / metadata invalidation) which never flushes before
+// dropping the pipeline, so writable chunks would otherwise leak their
+// write-budget slots permanently.
+func TestShutdown_ReleasesWritableReservation(t *testing.T) {
+	acc := NewWriteBufferAccountant(1024 * 1024)
+	saveFn := func(io.Reader, int64, int64, int64, func()) {}
+	up := NewUploadPipeline(nil, 64*1024, saveFn, 16, t.TempDir(), acc)
+
+	// A small partial write lands in writableChunks as a mem chunk and
+	// stays there (the chunk is not complete so maybeMoveToSealed is a
+	// no-op) — exactly the state Destroy() can hit.
+	buf := make([]byte, 128)
+	if _, err := up.SaveDataAt(buf, 0, true, 1); err != nil {
+		t.Fatalf("SaveDataAt: %v", err)
+	}
+	if got := acc.Used(); got != 64*1024 {
+		t.Fatalf("expected 64KiB reserved after one write, got %d", got)
+	}
+
+	up.Shutdown()
+
+	if got := acc.Used(); got != 0 {
+		t.Fatalf("expected 0 used after Shutdown, got %d (writable reservation leaked)", got)
+	}
+}
+
+// TestSealedChunk_FreeReferenceIsIdempotent verifies that FreeReference
+// releases the accountant slot exactly once even if it is invoked more
+// than once. This guards the path where Shutdown and the async uploader
+// goroutine both call FreeReference on the same sealed chunk: the second
+// call must be a no-op, not a double-release.
+func TestSealedChunk_FreeReferenceIsIdempotent(t *testing.T) {
+	acc := NewWriteBufferAccountant(1024 * 1024)
+	acc.Reserve(64 * 1024)
+
+	sc := &SealedChunk{
+		chunk:            NewMemChunk(0, 64*1024),
+		referenceCounter: 1,
+		accountant:       acc,
+		chunkSize:        64 * 1024,
+	}
+
+	sc.FreeReference("first")
+	if got := acc.Used(); got != 0 {
+		t.Fatalf("expected 0 used after first FreeReference, got %d", got)
+	}
+	sc.FreeReference("second")
+	if got := acc.Used(); got != 0 {
+		t.Fatalf("expected 0 used after second FreeReference, got %d (double release)", got)
+	}
+}

weed/mount/page_writer/upload_pipeline_test.go

@@ -0,0 +1,260 @@
+package page_writer
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestUploadPipeline(t *testing.T) {
+
+	uploadPipeline := NewUploadPipeline(nil, 2*1024*1024, nil, 16, "", nil)
+
+	writeRange(uploadPipeline, 0, 131072)
+	writeRange(uploadPipeline, 131072, 262144)
+	writeRange(uploadPipeline, 262144, 1025536)
+
+	confirmRange(t, uploadPipeline, 0, 1025536)
+
+	writeRange(uploadPipeline, 1025536, 1296896)
+
+	confirmRange(t, uploadPipeline, 1025536, 1296896)
+
+	writeRange(uploadPipeline, 1296896, 2162688)
+
+	confirmRange(t, uploadPipeline, 1296896, 2162688)
+
+	confirmRange(t, uploadPipeline, 1296896, 2162688)
+}
+
+// startOff and stopOff must be divided by 4
+func writeRange(uploadPipeline *UploadPipeline, startOff, stopOff int64) {
+	p := make([]byte, 4)
+	for i := startOff / 4; i < stopOff/4; i += 4 {
+		util.Uint32toBytes(p, uint32(i))
+		uploadPipeline.SaveDataAt(p, i, false, 0)
+	}
+}
+
+func confirmRange(t *testing.T, uploadPipeline *UploadPipeline, startOff, stopOff int64) {
+	p := make([]byte, 4)
+	for i := startOff; i < stopOff/4; i += 4 {
+		uploadPipeline.MaybeReadDataAt(p, i, 0)
+		x := util.BytesToUint32(p)
+		if x != uint32(i) {
+			t.Errorf("expecting %d found %d at offset [%d,%d)", i, x, i, i+4)
+		}
+	}
+}
+
+// Pressure-driven eviction must not seal a chunk with a leading or
+// internal gap (issue #9330).
+func TestEvictOneWritableChunk_SkipsGappyChunks(t *testing.T) {
+	const cs int64 = 2 * 1024 * 1024
+	// nil saveToStorage so the async upload SaveContent is a no-op.
+	up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), cs, nil, 16, "", nil)
+
+	block := make([]byte, cs/4)
+
+	// chunk 0: internal gap; chunk 1: leading gap; chunk 2: contiguous from 0.
+	if _, err := up.SaveDataAt(block, 0, true, 1); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 3*cs/4, true, 2); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+cs/4, true, 3); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+cs/2, true, 4); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 2*cs, true, 5); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 2*cs+cs/4, true, 6); err != nil {
+		t.Fatal(err)
+	}
+
+	if !up.EvictOneWritableChunk() {
+		t.Fatalf("EvictOneWritableChunk returned false; expected contiguous chunk 2 to be evictable")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(2)]; stillWritable {
+		t.Errorf("chunk 2 should have moved to sealed")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; !stillWritable {
+		t.Errorf("chunk 0 (internal gap) must remain writable")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; !stillWritable {
+		t.Errorf("chunk 1 (leading gap) must remain writable")
+	}
+
+	// Filling holes makes IsComplete true; maybeMoveToSealed auto-seals.
+	if _, err := up.SaveDataAt(block, cs/4, true, 7); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs/2, true, 8); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs, true, 9); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+3*cs/4, true, 10); err != nil {
+		t.Fatal(err)
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; stillWritable {
+		t.Errorf("chunk 0 should have auto-sealed after gap was filled")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; stillWritable {
+		t.Errorf("chunk 1 should have auto-sealed after leading range was filled")
+	}
+}
+
+// When every chunk is gappy, the fallback must still seal one so
+// accountant.Reserve can wake; oldest-LastWriteTsNs wins.
+func TestEvictOneWritableChunk_FallbackPicksOldestGappy(t *testing.T) {
+	const cs int64 = 2 * 1024 * 1024
+	up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), cs, nil, 16, "", nil)
+
+	block := make([]byte, cs/4)
+
+	// Strictly increasing tsNs so chunk 0 is oldest; equal WrittenSize.
+	if _, err := up.SaveDataAt(block, 0, true, 100); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 3*cs/4, true, 101); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+cs/4, true, 200); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+cs/2, true, 201); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 2*cs, true, 300); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 2*cs+3*cs/4, true, 301); err != nil {
+		t.Fatal(err)
+	}
+
+	if !up.EvictOneWritableChunk() {
+		t.Fatalf("fallback must seal a gappy chunk to free a Reserve slot")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; stillWritable {
+		t.Errorf("oldest gappy chunk (0) should have been picked by the fallback")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; !stillWritable {
+		t.Errorf("chunk 1 should remain writable")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(2)]; !stillWritable {
+		t.Errorf("chunk 2 should remain writable")
+	}
+}
+
+// Strict pass preempts the fallback even when a gappy chunk is older.
+func TestEvictOneWritableChunk_PrefersStrictOverFallback(t *testing.T) {
+	const cs int64 = 2 * 1024 * 1024
+	up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), cs, nil, 16, "", nil)
+
+	block := make([]byte, cs/4)
+
+	// chunk 0 (older, gappy), chunk 1 (newer, contiguous from 0).
+	if _, err := up.SaveDataAt(block, 0, true, 100); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 3*cs/4, true, 101); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs, true, 200); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+cs/4, true, 201); err != nil {
+		t.Fatal(err)
+	}
+
+	if !up.EvictOneWritableChunk() {
+		t.Fatalf("EvictOneWritableChunk returned false")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; stillWritable {
+		t.Errorf("contiguous chunk 1 must be picked over older gappy chunk 0")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; !stillWritable {
+		t.Errorf("gappy chunk 0 must remain writable")
+	}
+}
+
+// ProactiveFlush also skips gappy chunks (issue #9330). No liveness
+// fallback here — unlike EvictOneWritableChunk, returning false is just
+// a missed optimization.
+func TestProactiveFlush_SkipsGappyChunks(t *testing.T) {
+	const cs int64 = 2 * 1024 * 1024
+	up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), cs, nil, 16, "", nil)
+
+	block := make([]byte, cs/4)
+
+	// chunk 0: internal gap; chunk 1: leading gap; chunk 2: contiguous from 0.
+	if _, err := up.SaveDataAt(block, 0, true, 1); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 3*cs/4, true, 2); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+cs/4, true, 3); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+cs/2, true, 4); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 2*cs, true, 5); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 2*cs+cs/4, true, 6); err != nil {
+		t.Fatal(err)
+	}
+
+	// nowNs >> chunk timestamps so age clears the idle/maxHold gates;
+	// fillRatio < WrittenSize so nearlyFull is true. Leaves
+	// IsContiguouslyWritten as the only differentiator.
+	const (
+		nowNs           int64 = 1_000_000_000
+		idleThresholdNs int64 = 100
+		maxHoldNs       int64 = 200
+		fillRatio       int64 = cs / 8
+	)
+	if !up.ProactiveFlush(nowNs, idleThresholdNs, maxHoldNs, fillRatio, 0, false) {
+		t.Fatalf("ProactiveFlush returned false; expected contiguous chunk 2 to be sealed")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(2)]; stillWritable {
+		t.Errorf("chunk 2 should have moved to sealed")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; !stillWritable {
+		t.Errorf("chunk 0 (internal gap) must remain writable")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; !stillWritable {
+		t.Errorf("chunk 1 (leading gap) must remain writable")
+	}
+
+	if up.ProactiveFlush(nowNs, idleThresholdNs, maxHoldNs, fillRatio, 0, false) {
+		t.Errorf("ProactiveFlush returned true with only gappy chunks left")
+	}
+
+	if _, err := up.SaveDataAt(block, cs/4, true, 7); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs/2, true, 8); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs, true, 9); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+3*cs/4, true, 10); err != nil {
+		t.Fatal(err)
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; stillWritable {
+		t.Errorf("chunk 0 should have auto-sealed after gap was filled")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; stillWritable {
+		t.Errorf("chunk 1 should have auto-sealed after leading range was filled")
+	}
+}

weed/mount/page_writer/write_buffer_accountant.go

@@ -0,0 +1,180 @@
+package page_writer
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+const (
+	// softThresholdRatio is the fraction of cap at which soft throttling begins.
+	// At this level, writes sleep briefly to let uploaders drain.
+	softThresholdRatio = 0.8
+	// hardThresholdRatio is the fraction of cap at which hard throttling kicks in.
+	// Writes sleep longer to aggressively slow intake.
+	hardThresholdRatio = 0.95
+	// softThrottleDelay is the sleep duration when usage exceeds the soft threshold.
+	softThrottleDelay = 10 * time.Millisecond
+	// hardThrottleDelay is the sleep duration when usage exceeds the hard threshold.
+	hardThrottleDelay = 50 * time.Millisecond
+)
+
+// WriteBufferAccountant enforces a global byte budget across all
+// UploadPipeline instances. Callers Reserve chunk-sized slots before
+// allocating a page chunk (in memory or on swap) and Release them when
+// the chunk is freed. Reserve blocks when the cap would be exceeded,
+// providing natural backpressure to the FUSE write path when volume
+// uploads stall (e.g. all assigned volumes are full) instead of letting
+// the swap file grow without bound.
+//
+// A nil receiver is treated as "unlimited" for backward compatibility.
+type WriteBufferAccountant struct {
+	mu            sync.Mutex
+	cond          *sync.Cond
+	cap           int64 // 0 means unlimited
+	used          int64
+	softThreshold int64 // pre-computed: cap * softThresholdRatio
+	hardThreshold int64 // pre-computed: cap * hardThresholdRatio
+	evictor       func(needBytes int64) bool
+	evicting      bool
+
+	softThrottleCount atomic.Int64
+	hardThrottleCount atomic.Int64
+}
+
+func NewWriteBufferAccountant(capBytes int64) *WriteBufferAccountant {
+	a := &WriteBufferAccountant{
+		cap:           capBytes,
+		softThreshold: int64(float64(capBytes) * softThresholdRatio),
+		hardThreshold: int64(float64(capBytes) * hardThresholdRatio),
+	}
+	a.cond = sync.NewCond(&a.mu)
+	return a
+}
+
+// SetEvictor registers a callback that Reserve invokes when the cap would
+// otherwise block. The evictor is expected to force-seal at least one
+// writable chunk in some UploadPipeline, which turns a pinned-forever
+// writable chunk into a sealed chunk that the async uploader drains and
+// Releases. Without this hook, workloads that hold many files open for
+// write with less-than-chunkSize data in each (e.g. fio 4k randwrite with
+// nrfiles * chunkSize > cap) deadlock permanently, because writable chunks
+// only seal on close or when they fill.
+//
+// The evictor must not call Reserve on the same accountant or re-enter
+// Reserve transitively — it would deadlock on accountant.mu.
+func (a *WriteBufferAccountant) SetEvictor(fn func(needBytes int64) bool) {
+	if a == nil {
+		return
+	}
+	a.mu.Lock()
+	a.evictor = fn
+	a.mu.Unlock()
+}
+
+// Reserve blocks until n bytes can be accounted for under the cap.
+// It must not be called while holding any UploadPipeline lock, or the
+// uploader goroutines that eventually call Release will deadlock.
+func (a *WriteBufferAccountant) Reserve(n int64) {
+	if a == nil || a.cap <= 0 {
+		return
+	}
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	// Graduated backpressure: slow writers before hitting the hard cap.
+	// Use projected usage (used + n) so that a single reservation crossing
+	// a threshold is throttled immediately rather than on the next call.
+	//
+	// This runs once per Reserve, not inside the blocking loop below.
+	// Once usage reaches the cap, the evictor + cond.Wait is the correct
+	// mechanism: it blocks until a sealed chunk upload frees space, which
+	// is strictly more efficient than repeated time-based sleeps that
+	// cannot free capacity on their own.
+	projected := a.used + n
+	if projected >= a.hardThreshold && a.used > 0 {
+		a.hardThrottleCount.Add(1)
+		a.mu.Unlock()
+		time.Sleep(hardThrottleDelay)
+		a.mu.Lock()
+	} else if projected >= a.softThreshold && a.used > 0 {
+		a.softThrottleCount.Add(1)
+		a.mu.Unlock()
+		time.Sleep(softThrottleDelay)
+		a.mu.Lock()
+	}
+
+	for a.used+n > a.cap && a.used > 0 {
+		// Before blocking, try to force-seal a writable chunk somewhere so
+		// its async upload path will eventually Release a slot. Single-flight
+		// on `evicting` so a stampede of blocked reservers doesn't iterate
+		// the fhMap concurrently.
+		if a.evictor != nil && !a.evicting {
+			a.runEvictorLocked(n)
+			// A concurrent Release may have brought used back under the
+			// cap during the evict window — re-check before waiting so we
+			// do not block on a broadcast that has already fired.
+			if a.used+n <= a.cap || a.used == 0 {
+				break
+			}
+		}
+		a.cond.Wait()
+	}
+	a.used += n
+}
+
+// runEvictorLocked is called with a.mu held and !a.evicting. It drops the
+// lock around the evictor invocation so uploader goroutines (which call
+// Release under the same lock) can make progress, and uses defer to
+// guarantee the `evicting` flag and the lock are restored even if the
+// evictor panics.
+func (a *WriteBufferAccountant) runEvictorLocked(n int64) {
+	evictor := a.evictor
+	a.evicting = true
+	a.mu.Unlock()
+	defer func() {
+		a.mu.Lock()
+		a.evicting = false
+		a.cond.Broadcast()
+	}()
+	evictor(n)
+}
+
+func (a *WriteBufferAccountant) Release(n int64) {
+	if a == nil || a.cap <= 0 {
+		return
+	}
+	a.mu.Lock()
+	a.used -= n
+	if a.used < 0 {
+		a.used = 0
+	}
+	a.cond.Broadcast()
+	a.mu.Unlock()
+}
+
+// Used returns the currently reserved byte count (for tests/metrics).
+func (a *WriteBufferAccountant) Used() int64 {
+	if a == nil || a.cap <= 0 {
+		return 0
+	}
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	return a.used
+}
+
+// SoftThrottleCount returns the number of times soft throttling was triggered.
+func (a *WriteBufferAccountant) SoftThrottleCount() int64 {
+	if a == nil {
+		return 0
+	}
+	return a.softThrottleCount.Load()
+}
+
+// HardThrottleCount returns the number of times hard throttling was triggered.
+func (a *WriteBufferAccountant) HardThrottleCount() int64 {
+	if a == nil {
+		return 0
+	}
+	return a.hardThrottleCount.Load()
+}

weed/mount/page_writer/write_buffer_accountant_test.go

@@ -0,0 +1,199 @@
+package page_writer
+
+import (
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestWriteBufferAccountant_Unlimited(t *testing.T) {
+	a := NewWriteBufferAccountant(0)
+	a.Reserve(1 << 30)
+	a.Reserve(1 << 30)
+	if got := a.Used(); got != 0 {
+		t.Fatalf("unlimited accountant should not track usage, got %d", got)
+	}
+}
+
+func TestWriteBufferAccountant_Nil(t *testing.T) {
+	var a *WriteBufferAccountant
+	a.Reserve(100) // must not panic
+	a.Release(100)
+	if a.Used() != 0 {
+		t.Fatal("nil Used should return 0")
+	}
+}
+
+func TestWriteBufferAccountant_ReserveAndRelease(t *testing.T) {
+	a := NewWriteBufferAccountant(300)
+	a.Reserve(100)
+	a.Reserve(100)
+	if got := a.Used(); got != 200 {
+		t.Fatalf("expected 200 used, got %d", got)
+	}
+	a.Release(100)
+	if got := a.Used(); got != 100 {
+		t.Fatalf("expected 100 used after release, got %d", got)
+	}
+}
+
+func TestWriteBufferAccountant_BlocksWhenOverCap(t *testing.T) {
+	a := NewWriteBufferAccountant(100)
+	a.Reserve(100)
+
+	started := make(chan struct{})
+	var landed atomic.Bool
+	go func() {
+		close(started)
+		a.Reserve(50) // should block — used=100, cap=100
+		landed.Store(true)
+	}()
+
+	// Wait until the goroutine is about to enter Reserve. Once it does,
+	// Reserve cannot make progress until Release is called (cap is
+	// full), so landed is guaranteed to stay false until we release.
+	<-started
+	if landed.Load() {
+		t.Fatal("second reserve should be blocked while cap is full")
+	}
+
+	a.Release(100)
+
+	deadline := time.Now().Add(time.Second)
+	for !landed.Load() && time.Now().Before(deadline) {
+		time.Sleep(5 * time.Millisecond)
+	}
+	if !landed.Load() {
+		t.Fatal("second reserve should unblock after release")
+	}
+	if got := a.Used(); got != 50 {
+		t.Fatalf("expected 50 used after handoff, got %d", got)
+	}
+}
+
+func TestWriteBufferAccountant_SoftThrottle(t *testing.T) {
+	// Cap = 1000, soft threshold = 800. Reserve 850 (85%) then measure
+	// that the next Reserve is delayed by at least softThrottleDelay.
+	a := NewWriteBufferAccountant(1000)
+	a.Reserve(850)
+
+	start := time.Now()
+	a.Reserve(10) // should trigger soft throttle (used=850 > 800)
+	elapsed := time.Since(start)
+
+	if elapsed < softThrottleDelay {
+		t.Fatalf("expected Reserve to take >= %v (soft throttle), took %v", softThrottleDelay, elapsed)
+	}
+}
+
+func TestWriteBufferAccountant_HardThrottle(t *testing.T) {
+	// Cap = 1000, hard threshold = 950. Reserve 960 (96%) then measure
+	// that the next Reserve is delayed by at least hardThrottleDelay.
+	a := NewWriteBufferAccountant(1000)
+	a.Reserve(960)
+
+	start := time.Now()
+	a.Reserve(10) // should trigger hard throttle (used=960 > 950)
+	elapsed := time.Since(start)
+
+	if elapsed < hardThrottleDelay {
+		t.Fatalf("expected Reserve to take >= %v (hard throttle), took %v", hardThrottleDelay, elapsed)
+	}
+}
+
+func TestWriteBufferAccountant_NoThrottleBelowSoft(t *testing.T) {
+	// Cap = 1000, soft threshold = 800. Reserve 700 (70%) -- below soft
+	// threshold, so subsequent Reserve should return quickly.
+	a := NewWriteBufferAccountant(1000)
+	a.Reserve(700)
+
+	start := time.Now()
+	a.Reserve(10)
+	elapsed := time.Since(start)
+
+	if elapsed >= softThrottleDelay {
+		t.Fatalf("expected Reserve below soft threshold to avoid throttle delay (< %v), took %v", softThrottleDelay, elapsed)
+	}
+}
+
+func TestWriteBufferAccountant_ThrottleCounters(t *testing.T) {
+	a := NewWriteBufferAccountant(1000)
+
+	// Below soft -- no throttle
+	a.Reserve(700)
+	a.Reserve(10)
+	if a.SoftThrottleCount() != 0 || a.HardThrottleCount() != 0 {
+		t.Fatalf("expected no throttle counts below soft threshold, got soft=%d hard=%d",
+			a.SoftThrottleCount(), a.HardThrottleCount())
+	}
+
+	// Release back down, then fill to soft zone
+	a.Release(710)
+	a.Reserve(850) // now used=850, above soft (800)
+	a.Reserve(10)  // triggers soft throttle
+	if a.SoftThrottleCount() != 1 {
+		t.Fatalf("expected soft throttle count 1, got %d", a.SoftThrottleCount())
+	}
+	if a.HardThrottleCount() != 0 {
+		t.Fatalf("expected hard throttle count 0, got %d", a.HardThrottleCount())
+	}
+
+	// Release and fill to hard zone
+	a.Release(860)
+	a.Reserve(960) // now used=960, above hard (950)
+	a.Reserve(10)  // triggers hard throttle
+	if a.HardThrottleCount() != 1 {
+		t.Fatalf("expected hard throttle count 1, got %d", a.HardThrottleCount())
+	}
+}
+
+func TestWriteBufferAccountant_GraduatedRecovery(t *testing.T) {
+	// Fill to hard threshold, verify hard throttle. Release to soft zone,
+	// verify that throttle level decreases to soft.
+	a := NewWriteBufferAccountant(1000)
+
+	// Fill to hard zone (960 > 950)
+	a.Reserve(960)
+	start := time.Now()
+	a.Reserve(10) // hard throttle
+	elapsed := time.Since(start)
+	if elapsed < hardThrottleDelay {
+		t.Fatalf("expected hard throttle delay, got %v", elapsed)
+	}
+	if a.HardThrottleCount() != 1 {
+		t.Fatalf("expected hard throttle count 1, got %d", a.HardThrottleCount())
+	}
+
+	// Release down to soft zone: used = 970 - 130 = 840 (> 800 soft, < 950 hard)
+	a.Release(130)
+
+	start = time.Now()
+	a.Reserve(10) // should now be soft throttle
+	elapsed = time.Since(start)
+	if elapsed < softThrottleDelay {
+		t.Fatalf("expected soft throttle delay after recovery, got %v", elapsed)
+	}
+	if a.HardThrottleCount() != 1 {
+		t.Fatalf("expected hard throttle count to remain 1 after recovery, got %d", a.HardThrottleCount())
+	}
+	if a.SoftThrottleCount() != 1 {
+		t.Fatalf("expected soft throttle count 1 after recovery, got %d", a.SoftThrottleCount())
+	}
+}
+
+func TestWriteBufferAccountant_AllowsOversizeWhenEmpty(t *testing.T) {
+	// A single request larger than cap must succeed when nothing is in
+	// flight, otherwise a single file handle with a large chunk size would
+	// deadlock forever.
+	a := NewWriteBufferAccountant(100)
+	done := make(chan struct{})
+	go func() {
+		a.Reserve(500)
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("oversize reserve on empty accountant should not block")
+	}
+}

weed/mount/page_writer/write_buffer_cap_integration_test.go

@@ -0,0 +1,175 @@
+package page_writer
+
+import (
+	"io"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// TestWriteBufferCap_SharedAcrossPipelines exercises the full write-budget
+// plumbing end-to-end: one WriteBufferAccountant shared across several
+// UploadPipeline instances (modeling multiple open file handles in a
+// single mount), a deliberately stalled saveFn (modeling the reported
+// "volume server rejecting uploads" condition), and many concurrent
+// writers.  Without the cap this scenario is what filled /tmp to 1.8 TB;
+// with the cap, Used() must never exceed the configured budget and
+// writers must backpressure instead of allocating unboundedly.
+func TestWriteBufferCap_SharedAcrossPipelines(t *testing.T) {
+	const (
+		chunkSize    = 64 * 1024
+		capChunks    = 4 // global budget: 4 chunks across all pipelines
+		numPipelines = 3
+		numWriters   = 6
+		writesEach   = 8 // more than enough to exceed capChunks if unbounded
+	)
+	capBytes := int64(capChunks * chunkSize)
+	acc := NewWriteBufferAccountant(capBytes)
+
+	// Gate every upload so sealed chunks cannot drain until the test says
+	// so — this is what reproduces the "stalled volume server" scenario.
+	var inflight atomic.Int64
+	var maxInflight atomic.Int64
+	release := make(chan struct{})
+	saveFn := func(_ io.Reader, _, _, _ int64, cleanup func()) {
+		cur := inflight.Add(1)
+		for {
+			prev := maxInflight.Load()
+			if cur <= prev || maxInflight.CompareAndSwap(prev, cur) {
+				break
+			}
+		}
+		<-release
+		inflight.Add(-1)
+		if cleanup != nil {
+			cleanup()
+		}
+	}
+
+	// Sample Used() frequently on a background goroutine so we can catch
+	// any moment the accountant exceeds the cap.
+	var observedMax atomic.Int64
+	stopSampling := make(chan struct{})
+	var samplerWg sync.WaitGroup
+	samplerWg.Add(1)
+	go func() {
+		defer samplerWg.Done()
+		for {
+			select {
+			case <-stopSampling:
+				return
+			default:
+			}
+			u := acc.Used()
+			for {
+				prev := observedMax.Load()
+				if u <= prev || observedMax.CompareAndSwap(prev, u) {
+					break
+				}
+			}
+			time.Sleep(time.Millisecond)
+		}
+	}()
+
+	// Build N pipelines, each with its own uploader executor so chunks
+	// can actually enter the upload stage and hit the gated saveFn.
+	pipelines := make([]*UploadPipeline, numPipelines)
+	for i := range pipelines {
+		up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(4), chunkSize, saveFn, 2, t.TempDir(), acc)
+		pipelines[i] = up
+	}
+
+	// Launch numWriters goroutines, round-robin across pipelines. Each
+	// writer writes writesEach full chunks, on distinct logical chunk
+	// indices so every write triggers a new reservation.
+	var writerWg sync.WaitGroup
+	writerStart := make(chan struct{})
+	writerWg.Add(numWriters)
+	for w := 0; w < numWriters; w++ {
+		w := w
+		go func() {
+			defer writerWg.Done()
+			<-writerStart
+			up := pipelines[w%numPipelines]
+			data := make([]byte, chunkSize)
+			for k := 0; k < writesEach; k++ {
+				// Offset each writer into a distinct region of the file
+				// so chunks don't collide on the same logicChunkIndex.
+				off := int64(w)*int64(writesEach)*chunkSize + int64(k)*chunkSize
+				if _, err := up.SaveDataAt(data, off, true, int64(k+1)); err != nil {
+					t.Errorf("writer %d: SaveDataAt: %v", w, err)
+					return
+				}
+			}
+		}()
+	}
+	close(writerStart)
+
+	// Give the writers time to pile up against the cap. Any unbounded
+	// allocation would show up as Used() > capBytes; the sampler catches
+	// that. Writers should be blocked on Reserve well before writesEach
+	// chunks are created per pipeline.
+	deadline := time.Now().Add(500 * time.Millisecond)
+	for time.Now().Before(deadline) {
+		if acc.Used() >= capBytes {
+			break
+		}
+		time.Sleep(2 * time.Millisecond)
+	}
+	if got := acc.Used(); got < capBytes/2 {
+		t.Fatalf("expected writers to fill the budget, used=%d cap=%d", got, capBytes)
+	}
+
+	// While the gate is closed and writers are blocked, Used() must stay
+	// at or below the cap. Sample explicitly for a short window to catch
+	// any overshoot.
+	for i := 0; i < 50; i++ {
+		if u := acc.Used(); u > capBytes {
+			t.Fatalf("Used()=%d exceeded cap=%d while writers were stalled", u, capBytes)
+		}
+		time.Sleep(2 * time.Millisecond)
+	}
+
+	// Open the gate: every sealed chunk completes, writers drain.
+	close(release)
+
+	// Wait for all writers to finish.
+	writerDone := make(chan struct{})
+	go func() {
+		writerWg.Wait()
+		close(writerDone)
+	}()
+	select {
+	case <-writerDone:
+	case <-time.After(10 * time.Second):
+		close(stopSampling)
+		samplerWg.Wait()
+		t.Fatalf("writers did not drain after release; used=%d", acc.Used())
+	}
+
+	// Shut down all pipelines and stop sampling.
+	for _, up := range pipelines {
+		up.Shutdown()
+	}
+	close(stopSampling)
+	samplerWg.Wait()
+
+	if got := observedMax.Load(); got > capBytes {
+		t.Fatalf("observed Used()=%d exceeded cap=%d", got, capBytes)
+	}
+	// An uploader goroutine releases its budget slot only after reacquiring
+	// chunksLock post-Shutdown, so the last releases may land asynchronously.
+	drainDeadline := time.Now().Add(5 * time.Second)
+	for acc.Used() != 0 && time.Now().Before(drainDeadline) {
+		time.Sleep(2 * time.Millisecond)
+	}
+	if got := acc.Used(); got != 0 {
+		t.Fatalf("expected 0 used after shutdown, got %d", got)
+	}
+
+	t.Logf("peak inflight uploads=%d, peak Used()=%d bytes (cap=%d)",
+		maxInflight.Load(), observedMax.Load(), capBytes)
+}

weed/mount/page_writer_pattern.go

@@ -0,0 +1,68 @@
+package mount
+
+import "sync/atomic"
+
+type WriterPattern struct {
+	isSequentialCounter int64
+	writeFrontier       int64 // highest (offset+size) observed across writes
+	chunkSize           int64
+}
+
+const ModeChangeLimit = 3
+
+// SeqTolerance: a write whose start is within this many bytes of the current
+// write frontier still counts as sequential. Using a tolerance window rather
+// than strict contiguity absorbs reordered/concurrent FUSE writeback — multiple
+// Write upcalls for one handle interleave (MonitorWriteAt runs before the
+// per-handle write lock) and writeback flushes dirty pages slightly out of
+// offset order — while still rejecting far random seeks.
+const SeqTolerance = 8 << 20 // 8 MiB
+
+// For streaming write: keep dirty chunks in memory (NewMemChunk).
+// For random write: fall back to the on-disk swap file (NewSwapFileChunk).
+
+func NewWriterPattern(chunkSize int64) *WriterPattern {
+	return &WriterPattern{
+		isSequentialCounter: 0,
+		writeFrontier:       0,
+		chunkSize:           chunkSize,
+	}
+}
+
+func (rp *WriterPattern) MonitorWriteAt(offset int64, size int) {
+	// Advance the frontier to max(frontier, offset+size) and capture, in the same
+	// CAS loop, the pre-image this write is judged against. Reading the frontier
+	// inside the loop (rather than once up front) keeps `diff` below comparing
+	// against the freshest value even if a concurrent writeback upcall advances
+	// the frontier while we loop. Lock-free, consistent with the atomic counter.
+	end := offset + int64(size)
+	var frontier int64
+	for {
+		frontier = atomic.LoadInt64(&rp.writeFrontier)
+		if end <= frontier || atomic.CompareAndSwapInt64(&rp.writeFrontier, frontier, end) {
+			break
+		}
+	}
+
+	// near = this write starts within SeqTolerance of where writes had reached.
+	// Hysteresis (the ±ModeChangeLimit counter) keeps a single outlier write
+	// from flipping the mode and spilling a sequential stream to the swap file.
+	diff := offset - frontier
+	if diff < 0 {
+		diff = -diff
+	}
+	counter := atomic.LoadInt64(&rp.isSequentialCounter)
+	if diff <= SeqTolerance {
+		if counter < ModeChangeLimit {
+			atomic.AddInt64(&rp.isSequentialCounter, 1)
+		}
+	} else {
+		if counter > -ModeChangeLimit {
+			atomic.AddInt64(&rp.isSequentialCounter, -1)
+		}
+	}
+}
+
+func (rp *WriterPattern) IsSequentialMode() bool {
+	return atomic.LoadInt64(&rp.isSequentialCounter) >= 0
+}

weed/mount/page_writer_pattern_test.go

@@ -0,0 +1,104 @@
+package mount
+
+import (
+	"sync/atomic"
+	"testing"
+)
+
+const mb = 1 << 20
+
+func TestWriterPatternSequentialAndHysteresis(t *testing.T) {
+	wp := NewWriterPattern(2 * mb)
+	wp.MonitorWriteAt(0, mb)    // near(0 vs frontier 0) -> +1
+	wp.MonitorWriteAt(mb, mb)   // +2
+	wp.MonitorWriteAt(2*mb, mb) // +3 (capped)
+	if !wp.IsSequentialMode() {
+		t.Fatal("a stream from offset 0 must be sequential")
+	}
+	// a single far outlier does not flip to random (hysteresis): +3 -> +2
+	wp.MonitorWriteAt(900*mb, mb)
+	if !wp.IsSequentialMode() {
+		t.Fatal("a single outlier write must not flip out of sequential mode")
+	}
+}
+
+func TestWriterPatternFarFirstWriteIsRandom(t *testing.T) {
+	wp := NewWriterPattern(2 * mb)
+	wp.MonitorWriteAt(500*mb, mb) // far from frontier 0 -> -1
+	if wp.IsSequentialMode() {
+		t.Fatal("a far first write should be random")
+	}
+}
+
+func TestWriterPatternToleranceAbsorbsReorder(t *testing.T) {
+	wp := NewWriterPattern(2 * mb)
+	wp.MonitorWriteAt(0, mb)    // +1, frontier 1MB
+	wp.MonitorWriteAt(6*mb, mb) // |6-1|=5MB <= 8MB tolerance -> near, +2, frontier 7MB
+	wp.MonitorWriteAt(3*mb, mb) // |3-7|=4MB <= 8MB -> near, +3
+	if !wp.IsSequentialMode() {
+		t.Fatal("reordered writes within tolerance must stay sequential")
+	}
+}
+
+func TestWriterPatternRandomRecoveryNeedsHysteresis(t *testing.T) {
+	wp := NewWriterPattern(2 * mb)
+	wp.MonitorWriteAt(100*mb, mb) // far -> -1
+	wp.MonitorWriteAt(300*mb, mb) // far -> -2
+	wp.MonitorWriteAt(500*mb, mb) // far -> -3 (capped), frontier 501MB
+	// a single near write must not immediately flip back to sequential: -3 -> -2
+	wp.MonitorWriteAt(501*mb, mb)
+	if wp.IsSequentialMode() {
+		t.Fatal("one near write must not flip back from deep random mode")
+	}
+}
+
+// The max-frontier (vs the old atomic.Swap of the last write's stop offset) is the
+// load-bearing difference of this detector: the frontier must never move backward,
+// or a backward write would lower the baseline and make later far writes look near.
+func TestWriterPatternFrontierNeverRegresses(t *testing.T) {
+	wp := NewWriterPattern(2 * mb)
+	wp.MonitorWriteAt(500*mb, mb) // far first write -> -1, frontier 501MB
+	wp.MonitorWriteAt(0, mb)      // a backward write must not pull the frontier back
+	if got := atomic.LoadInt64(&wp.writeFrontier); got != 501*mb {
+		t.Fatalf("frontier regressed to %d; the max-frontier must never move backward", got)
+	}
+	// Behavioral consequence: writes far below the preserved frontier stay random.
+	// Had the frontier regressed to ~1MB, this would wrongly read as sequential.
+	if wp.IsSequentialMode() {
+		t.Fatal("writes far below the preserved frontier must remain random")
+	}
+}
+
+// SeqTolerance is the central new tuning knob; pin both sides of the inclusive
+// boundary so an off-by-one or a '<' vs '<=' change can't slip through silently.
+func TestWriterPatternToleranceBoundary(t *testing.T) {
+	atBoundary := NewWriterPattern(2 * mb)
+	atBoundary.MonitorWriteAt(SeqTolerance, 0) // diff == SeqTolerance -> near (inclusive), +1
+	if !atBoundary.IsSequentialMode() {
+		t.Fatal("a write exactly at frontier+SeqTolerance must count as sequential")
+	}
+	pastBoundary := NewWriterPattern(2 * mb)
+	pastBoundary.MonitorWriteAt(SeqTolerance+1, 0) // diff == SeqTolerance+1 -> far, -1
+	if pastBoundary.IsSequentialMode() {
+		t.Fatal("a write one byte past frontier+SeqTolerance must count as random")
+	}
+}
+
+// The perf-relevant escape from random (swap-file) mode: sustained near writes must
+// climb the counter back out of the negative floor and re-enter sequential (RAM) mode.
+func TestWriterPatternRecoversToSequential(t *testing.T) {
+	wp := NewWriterPattern(2 * mb)
+	wp.MonitorWriteAt(100*mb, mb) // far -> -1
+	wp.MonitorWriteAt(300*mb, mb) // far -> -2
+	wp.MonitorWriteAt(500*mb, mb) // far -> -3 (capped), frontier 501MB
+	if wp.IsSequentialMode() {
+		t.Fatal("three far writes must be random")
+	}
+	// contiguous near writes: -3 -> -2 -> -1 -> 0 -> +1, back into sequential mode
+	for i := int64(0); i < 4; i++ {
+		wp.MonitorWriteAt(501*mb+i*mb, mb)
+	}
+	if !wp.IsSequentialMode() {
+		t.Fatal("sustained near writes must recover sequential mode (escape the swap-file cliff)")
+	}
+}

weed/mount/peer_advertise.go

@@ -0,0 +1,52 @@
+package mount
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// ResolvePeerAdvertiseAddr returns the host:port this mount should
+// register with the filer and announce to peers. It handles three
+// common cases:
+//
+//  1. The operator set -peer.advertise=host:port explicitly — use it.
+//  2. -peer.listen contains a specific bind host — use that host with
+//     the bind port (e.g. "10.0.0.5:18080").
+//  3. -peer.listen is a wildcard bind (":18080", "0.0.0.0:18080", or
+//     "[::]:18080") — combine util.DetectedHostAddress() with the port.
+//
+// Returns an error if the listen string is unparseable or if case (3)
+// hits and auto-detection turns up nothing. We deliberately do NOT fall
+// back to "localhost": an advertised loopback gets registered with the
+// filer and hands other mounts a useless address to dial (their own
+// loopback). Better to fail loudly so the operator sets -peer.advertise.
+func ResolvePeerAdvertiseAddr(listen, advertise string) (string, error) {
+	if advertise != "" {
+		return advertise, nil
+	}
+	if listen == "" {
+		return "", fmt.Errorf("peer listen address is empty")
+	}
+	host, port, err := net.SplitHostPort(listen)
+	if err != nil {
+		return "", fmt.Errorf("parse -peer.listen %q: %w", listen, err)
+	}
+	if !isWildcardHost(host) {
+		return net.JoinHostPort(host, port), nil
+	}
+	detected := util.DetectedHostAddress()
+	if detected == "" {
+		return "", fmt.Errorf("cannot auto-detect host for wildcard -peer.listen %q; set -peer.advertise=host:port explicitly", listen)
+	}
+	return net.JoinHostPort(detected, port), nil
+}
+
+func isWildcardHost(h string) bool {
+	switch h {
+	case "", "0.0.0.0", "::", "[::]":
+		return true
+	}
+	return false
+}

weed/mount/peer_advertise_test.go

@@ -0,0 +1,55 @@
+package mount
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestResolvePeerAdvertiseAddr(t *testing.T) {
+	tests := []struct {
+		name      string
+		listen    string
+		advertise string
+		wantErr   bool
+		// addr substring check — DetectedHostAddress is system-dependent so
+		// we don't assert an exact value for the wildcard path.
+		wantSuffix string
+	}{
+		{"explicit advertise wins", ":18080", "10.1.1.9:20000", false, ":20000"},
+		{"bind host used verbatim", "10.0.0.5:18080", "", false, "10.0.0.5:18080"},
+		{"wildcard ipv4 bind", "0.0.0.0:18080", "", false, ":18080"},
+		{"empty host bind", ":18080", "", false, ":18080"},
+		{"ipv6 wildcard", "[::]:18080", "", false, ":18080"},
+		{"unparseable listen errors", "garbage", "", true, ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ResolvePeerAdvertiseAddr(tt.listen, tt.advertise)
+			if tt.wantErr {
+				if err == nil {
+					t.Fatalf("expected error, got addr=%q", got)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if !strings.HasSuffix(got, tt.wantSuffix) {
+				t.Errorf("addr=%q, expected suffix %q", got, tt.wantSuffix)
+			}
+		})
+	}
+}
+
+func TestIsWildcardHost(t *testing.T) {
+	for _, h := range []string{"", "0.0.0.0", "::", "[::]"} {
+		if !isWildcardHost(h) {
+			t.Errorf("%q should be wildcard", h)
+		}
+	}
+	for _, h := range []string{"10.0.0.5", "host.example", "localhost"} {
+		if isWildcardHost(h) {
+			t.Errorf("%q should NOT be wildcard", h)
+		}
+	}
+}

weed/mount/peer_announcer.go

@@ -0,0 +1,374 @@
+package mount
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mount_peer_pb"
+)
+
+// PeerAnnouncer batches and flushes ChunkAnnounce RPCs to the
+// HRW-assigned owner mounts for each cached fid this mount holds.
+//
+// Shape:
+//   - EnqueueAnnounce(fid) is a non-blocking push into an in-memory set.
+//   - A background ticker flushes every announceInterval. Each flush
+//     drains the set, adds fids due for TTL renewal, groups by owner
+//     mount via HRW, and sends one ChunkAnnounce RPC per distinct owner.
+//   - A successful send records announced_at[fid]. A rejected (not-owner)
+//     fid is requeued so it will be retried after the seed view refreshes.
+//
+// See design-weed-mount-peer-chunk-sharing.md §4.4.
+type PeerAnnouncer struct {
+	selfAddr         string
+	selfDataCenter   string
+	selfRack         string
+	ownerFor         func(fid string) string
+	dialPeer         MountPeerDialer
+	localDir         *PeerDirectory        // populated directly for fids whose HRW owner == self
+	isCached         func(fid string) bool // residency check; set nil to disable
+	announceInterval time.Duration
+	announceTTL      time.Duration
+
+	mu          sync.Mutex
+	pending     map[string]struct{}
+	announcedAt map[string]announceRecord
+
+	stopCh  chan struct{}
+	stopped atomic.Bool
+	runWg   sync.WaitGroup // tracks the run goroutine so Stop can wait
+
+	// flushCancelMu guards flushCancel, which is the cancel fn for the
+	// currently in-flight flush (nil when idle). Stop takes the lock,
+	// cancels the flush to unblock its RPCs fast, then waits on runWg.
+	flushCancelMu sync.Mutex
+	flushCancel   context.CancelFunc
+
+	flushes      atomic.Int64
+	sentFids     atomic.Int64
+	rejectedFids atomic.Int64
+	flushErrs    atomic.Int64
+
+	clock func() time.Time
+}
+
+// MountPeerDialer opens a MountPeer gRPC client to a given peer. Tests
+// inject a fake; production uses a real gRPC dial backed by a short
+// connection cache.
+type MountPeerDialer func(ctx context.Context, peerAddr string) (mount_peer_pb.MountPeerClient, func(), error)
+
+// announceRecord is what we remember about a successful announce: who
+// we told (the HRW owner at the time) and when. On each flush we
+// recompute the HRW owner and, if it moved — typically because a new
+// mount joined the seed list and the ring rebalanced — re-announce
+// against the new owner. Without this, seed-view divergence during
+// startup silently ages out fids on an owner that the reader never
+// looks up.
+type announceRecord struct {
+	owner string
+	at    time.Time
+}
+
+// NewPeerAnnouncer constructs an announcer. Caller must call Start.
+// selfDataCenter and selfRack are the locality labels attached to every
+// ChunkAnnounce so the receiving directory records holders with DC+rack
+// and the fetcher can later re-rank by locality.
+//
+// localDir is optional: when non-nil, any fid whose HRW owner resolves
+// to self is written directly into localDir instead of sending a
+// self→self RPC (which the receiver would reject anyway). This is the
+// only way a fid the writer itself owns becomes visible to peers'
+// ChunkLookup calls.
+func NewPeerAnnouncer(selfAddr, selfDataCenter, selfRack string, ownerFor func(fid string) string, dial MountPeerDialer, localDir *PeerDirectory) *PeerAnnouncer {
+	return &PeerAnnouncer{
+		selfAddr:         selfAddr,
+		selfDataCenter:   selfDataCenter,
+		selfRack:         selfRack,
+		ownerFor:         ownerFor,
+		dialPeer:         dial,
+		localDir:         localDir,
+		announceInterval: 15 * time.Second,
+		announceTTL:      300 * time.Second,
+		pending:          map[string]struct{}{},
+		announcedAt:      map[string]announceRecord{},
+		stopCh:           make(chan struct{}),
+		clock:            time.Now,
+	}
+}
+
+// EnqueueAnnounce marks a fid as needing announcement. Non-blocking;
+// dedupes within a flush window.
+func (a *PeerAnnouncer) EnqueueAnnounce(fid string) {
+	a.mu.Lock()
+	a.pending[fid] = struct{}{}
+	a.mu.Unlock()
+}
+
+// SetCachePresence wires an optional residency check. When set, the
+// announcer consults it right before dispatching a ChunkAnnounce RPC or
+// writing a self-owned entry: if the cache no longer has the chunk
+// (e.g. LRU-evicted under memory pressure between SetChunk and the
+// next flush tick) we drop the announce rather than advertise bytes we
+// can't serve back. nil disables the check.
+func (a *PeerAnnouncer) SetCachePresence(isCached func(fid string) bool) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	a.isCached = isCached
+}
+
+// Start launches the background flush loop.
+func (a *PeerAnnouncer) Start() {
+	a.runWg.Add(1)
+	go func() {
+		defer a.runWg.Done()
+		a.run()
+	}()
+}
+
+// Stop halts the flush loop and waits for the current flush (if any) to
+// finish. Safe to call multiple times. Callers that tear down shared
+// dependencies (conn pool, local directory) can rely on no more
+// sendTo/localDir.Announce goroutines being live after Stop returns.
+func (a *PeerAnnouncer) Stop() {
+	if a.stopped.Swap(true) {
+		return
+	}
+	close(a.stopCh)
+	// Cancel the in-flight flush's context so its RPCs return quickly
+	// instead of running out the 10 s timeout.
+	a.flushCancelMu.Lock()
+	if a.flushCancel != nil {
+		a.flushCancel()
+	}
+	a.flushCancelMu.Unlock()
+	a.runWg.Wait()
+}
+
+// Stats exposes counters for observability.
+func (a *PeerAnnouncer) Stats() (flushes, sentFids, rejectedFids, flushErrs int64) {
+	return a.flushes.Load(), a.sentFids.Load(), a.rejectedFids.Load(), a.flushErrs.Load()
+}
+
+// announceFlushTimeout caps how long a single flush cycle can run. A slow
+// or unresponsive owner peer would otherwise block a flush goroutine
+// indefinitely, and since flushes fan out per owner in parallel a
+// pileup would bleed into subsequent ticks and grow unbounded. The
+// bound is set a bit under one flush interval so the next tick's flush
+// doesn't chase the previous one's tail.
+const announceFlushTimeout = 10 * time.Second
+
+func (a *PeerAnnouncer) run() {
+	t := time.NewTicker(a.announceInterval)
+	defer t.Stop()
+	for {
+		select {
+		case <-a.stopCh:
+			return
+		case <-t.C:
+			ctx, cancel := context.WithTimeout(context.Background(), announceFlushTimeout)
+			a.flushCancelMu.Lock()
+			a.flushCancel = cancel
+			a.flushCancelMu.Unlock()
+
+			a.flushOnce(ctx)
+
+			a.flushCancelMu.Lock()
+			a.flushCancel = nil
+			a.flushCancelMu.Unlock()
+			cancel()
+		}
+	}
+}
+
+// flushOnce is exported via a wrapper for tests.
+func (a *PeerAnnouncer) FlushForTest(ctx context.Context) {
+	a.flushOnce(ctx)
+}
+
+func (a *PeerAnnouncer) flushOnce(ctx context.Context) {
+	a.flushes.Add(1)
+	now := a.clock()
+
+	a.mu.Lock()
+	// Swap pending in O(1) instead of copying.
+	toAnnounce := a.pending
+	a.pending = map[string]struct{}{}
+
+	// Add renewals: (1) fids we announced long enough ago that they'll
+	// expire on the owner side before our next flush tick; (2) fids
+	// whose HRW owner has changed since we last announced them —
+	// usually because a new mount joined the seed list and the ring
+	// rebalanced. Also prune announcedAt entries that are so old
+	// (> 2× TTL) that re-announcing them would be pointless.
+	renewThreshold := now.Add(-a.announceTTL + 2*a.announceInterval)
+	staleCutoff := now.Add(-2 * a.announceTTL)
+	for fid, rec := range a.announcedAt {
+		switch {
+		case rec.at.Before(staleCutoff):
+			delete(a.announcedAt, fid)
+		case rec.at.Before(renewThreshold):
+			toAnnounce[fid] = struct{}{}
+		default:
+			// Same fid, fresh timestamp — but owner may have moved.
+			// Force a re-announce if HRW says a different mount is
+			// the owner now.
+			if a.ownerFor != nil {
+				if cur := a.ownerFor(fid); cur != "" && cur != rec.owner {
+					toAnnounce[fid] = struct{}{}
+				}
+			}
+		}
+	}
+	a.mu.Unlock()
+
+	if len(toAnnounce) == 0 {
+		return
+	}
+
+	// Drop fids the cache no longer has. The write path hooks into
+	// SetChunk + EnqueueAnnounce, but the chunk can be LRU-evicted in
+	// the window between that and the next 15 s flush. Advertising a
+	// chunk we can't actually serve just sends remote fetchers to our
+	// FetchChunk, which then returns NOT_FOUND — a wasted round trip.
+	// Also drop the announcedAt record so we don't keep renewing
+	// evicted fids forever.
+	a.mu.Lock()
+	present := a.isCached
+	a.mu.Unlock()
+	if present != nil {
+		for fid := range toAnnounce {
+			if !present(fid) {
+				delete(toAnnounce, fid)
+				a.mu.Lock()
+				delete(a.announcedAt, fid)
+				a.mu.Unlock()
+			}
+		}
+		if len(toAnnounce) == 0 {
+			return
+		}
+	}
+
+	// Classify each fid. Three buckets:
+	//   byOwner — HRW owner is a remote mount; send a ChunkAnnounce RPC.
+	//   selfOwned — HRW owner is us; write directly into our local
+	//               directory (no RPC — we ARE the directory for this
+	//               fid). Without this, fids we wrote AND HRW-own stay
+	//               invisible to peer ChunkLookup forever.
+	//   deferred — HRW owner unknown (empty seed view). Re-queue for
+	//              the next flush once listOnce has populated seeds.
+	byOwner := map[string][]string{}
+	var selfOwned []string
+	var deferred []string
+	for fid := range toAnnounce {
+		owner := ""
+		if a.ownerFor != nil {
+			owner = a.ownerFor(fid)
+		}
+		switch owner {
+		case "":
+			deferred = append(deferred, fid)
+		case a.selfAddr:
+			selfOwned = append(selfOwned, fid)
+		default:
+			byOwner[owner] = append(byOwner[owner], fid)
+		}
+	}
+	if len(selfOwned) > 0 && a.localDir != nil {
+		a.localDir.Announce(a.selfAddr, a.selfDataCenter, a.selfRack, selfOwned, a.announceTTL, nil)
+		a.mu.Lock()
+		for _, fid := range selfOwned {
+			a.announcedAt[fid] = announceRecord{owner: a.selfAddr, at: now}
+			a.sentFids.Add(1)
+		}
+		a.mu.Unlock()
+	} else if len(selfOwned) > 0 {
+		// No local directory wired (e.g. unit tests with nil). Re-queue
+		// so a future flush with a possibly-different HRW owner can
+		// send them.
+		deferred = append(deferred, selfOwned...)
+	}
+	if len(deferred) > 0 {
+		a.mu.Lock()
+		for _, fid := range deferred {
+			a.pending[fid] = struct{}{}
+		}
+		a.mu.Unlock()
+	}
+
+	// Diagnostic V(2): shape of the current flush. Helps post-mortem
+	// debugging when an announce pipeline silently stops making progress
+	// (e.g. seed view stuck at self-only, owner unreachable, etc).
+	if len(byOwner) > 0 || len(selfOwned) > 0 || len(deferred) > 0 {
+		glog.V(2).Infof("peer-announce flush: toAnnounce=%d byOwner=%d selfOwned=%d deferred=%d",
+			len(toAnnounce), len(byOwner), len(selfOwned), len(deferred))
+	}
+
+	// Fan out one RPC per owner in parallel. A slow or unreachable
+	// owner blocks only its own goroutine; other owners' announces
+	// proceed immediately.
+	var wg sync.WaitGroup
+	for owner, fids := range byOwner {
+		wg.Add(1)
+		go func(owner string, fids []string) {
+			defer wg.Done()
+			a.sendTo(ctx, owner, fids, now)
+		}(owner, fids)
+	}
+	wg.Wait()
+}
+
+func (a *PeerAnnouncer) sendTo(ctx context.Context, owner string, fids []string, now time.Time) {
+	client, closeFn, err := a.dialPeer(ctx, owner)
+	if err != nil {
+		a.flushErrs.Add(1)
+		// Requeue these fids: a future flush will retry.
+		a.requeue(fids)
+		glog.V(2).Infof("peer-announce dial %s: %v", owner, err)
+		return
+	}
+	defer closeFn()
+
+	resp, err := client.ChunkAnnounce(ctx, &mount_peer_pb.ChunkAnnounceRequest{
+		FileIds:    fids,
+		PeerAddr:   a.selfAddr,
+		DataCenter: a.selfDataCenter,
+		Rack:       a.selfRack,
+		TtlSeconds: int32(a.announceTTL / time.Second),
+	})
+	if err != nil {
+		a.flushErrs.Add(1)
+		a.requeue(fids)
+		glog.V(2).Infof("peer-announce %s: %v", owner, err)
+		return
+	}
+
+	rejected := map[string]struct{}{}
+	for _, f := range resp.RejectedFileIds {
+		rejected[f] = struct{}{}
+	}
+
+	a.mu.Lock()
+	for _, fid := range fids {
+		if _, skip := rejected[fid]; skip {
+			// Owner has changed or doesn't yet agree; retry later.
+			a.pending[fid] = struct{}{}
+			a.rejectedFids.Add(1)
+			continue
+		}
+		a.announcedAt[fid] = announceRecord{owner: owner, at: now}
+		a.sentFids.Add(1)
+	}
+	a.mu.Unlock()
+}
+
+func (a *PeerAnnouncer) requeue(fids []string) {
+	a.mu.Lock()
+	for _, fid := range fids {
+		a.pending[fid] = struct{}{}
+	}
+	a.mu.Unlock()
+}

weed/mount/peer_announcer_test.go

@@ -0,0 +1,375 @@
+package mount
+
+import (
+	"context"
+	"sort"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/mount_peer_pb"
+	"google.golang.org/grpc"
+)
+
+// fakeMountPeerClient records each ChunkAnnounce call; returns rejected
+// fids from a configured set.
+type fakeMountPeerClient struct {
+	mu sync.Mutex
+	mount_peer_pb.MountPeerClient
+	announcedBy      map[string][]filerAnnouncement
+	rejectedByClient func(fid string) bool
+}
+
+type filerAnnouncement struct {
+	fids     []string
+	peerAddr string
+}
+
+func (f *fakeMountPeerClient) ChunkAnnounce(ctx context.Context, req *mount_peer_pb.ChunkAnnounceRequest, opts ...grpc.CallOption) (*mount_peer_pb.ChunkAnnounceResponse, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	// No owner key in the request; capture keyed by peer_addr (the announcer).
+	f.announcedBy[req.PeerAddr] = append(f.announcedBy[req.PeerAddr], filerAnnouncement{
+		fids:     append([]string(nil), req.FileIds...),
+		peerAddr: req.PeerAddr,
+	})
+	resp := &mount_peer_pb.ChunkAnnounceResponse{}
+	if f.rejectedByClient != nil {
+		for _, fid := range req.FileIds {
+			if f.rejectedByClient(fid) {
+				resp.RejectedFileIds = append(resp.RejectedFileIds, fid)
+			}
+		}
+	}
+	return resp, nil
+}
+
+// fakeDialer returns the same fakeMountPeerClient for every peer addr,
+// recording which owner the announcer dialed. Close is a no-op. The
+// dialed slice is protected by its own mutex because the announcer now
+// fans RPCs out across goroutines — concurrent appends would race.
+type dialRecorder struct {
+	mu      sync.Mutex
+	dialled []string
+}
+
+func (d *dialRecorder) record(addr string) {
+	d.mu.Lock()
+	d.dialled = append(d.dialled, addr)
+	d.mu.Unlock()
+}
+
+func (d *dialRecorder) snapshot() []string {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	out := make([]string, len(d.dialled))
+	copy(out, d.dialled)
+	return out
+}
+
+func fakeDialer(fc *fakeMountPeerClient, rec *dialRecorder) MountPeerDialer {
+	return func(ctx context.Context, peerAddr string) (mount_peer_pb.MountPeerClient, func(), error) {
+		rec.record(peerAddr)
+		return fc, func() {}, nil
+	}
+}
+
+func TestPeerAnnouncer_FlushGroupsByOwner(t *testing.T) {
+	fc := &fakeMountPeerClient{
+		announcedBy: map[string][]filerAnnouncement{},
+	}
+	rec := &dialRecorder{}
+
+	// Owner function: static assignment for test.
+	ownerFor := func(fid string) string {
+		if fid == "3,a" || fid == "3,c" {
+			return "owner-1:18081"
+		}
+		return "owner-2:18081"
+	}
+
+	a := NewPeerAnnouncer("self:18080", "", "", ownerFor, fakeDialer(fc, rec), nil)
+
+	a.EnqueueAnnounce("3,a")
+	a.EnqueueAnnounce("3,b")
+	a.EnqueueAnnounce("3,c")
+
+	a.FlushForTest(context.Background())
+
+	dialed := rec.snapshot()
+	sort.Strings(dialed)
+	if len(dialed) != 2 || dialed[0] != "owner-1:18081" || dialed[1] != "owner-2:18081" {
+		t.Errorf("expected both owners dialed once, got %v", dialed)
+	}
+
+	_, sent, _, _ := a.Stats()
+	if sent != 3 {
+		t.Errorf("expected 3 sent fids, got %d", sent)
+	}
+}
+
+func TestPeerAnnouncer_SkipOwnerEqualsSelf(t *testing.T) {
+	fc := &fakeMountPeerClient{announcedBy: map[string][]filerAnnouncement{}}
+	rec := &dialRecorder{}
+
+	a := NewPeerAnnouncer("self:18080", "", "", func(fid string) string {
+		return "self:18080"
+	}, fakeDialer(fc, rec), nil)
+
+	a.EnqueueAnnounce("3,a")
+	a.FlushForTest(context.Background())
+
+	if d := rec.snapshot(); len(d) != 0 {
+		t.Errorf("expected no dial when owner == self, dialed: %v", d)
+	}
+	// Self-owned fids stay in pending so a subsequent flush re-checks
+	// them against a possibly-refreshed seed view. Without this retry a
+	// mount whose initial MountList hadn't yet observed its peers would
+	// silently drop every write-path announce on the floor.
+	a.mu.Lock()
+	_, stillPending := a.pending["3,a"]
+	a.mu.Unlock()
+	if !stillPending {
+		t.Errorf("self-owned fid should stay pending for retry")
+	}
+}
+
+func TestPeerAnnouncer_RequeueOnRejection(t *testing.T) {
+	fc := &fakeMountPeerClient{
+		announcedBy:      map[string][]filerAnnouncement{},
+		rejectedByClient: func(fid string) bool { return fid == "3,x" },
+	}
+	rec := &dialRecorder{}
+
+	a := NewPeerAnnouncer("self:18080", "", "", func(fid string) string {
+		return "owner:18081"
+	}, fakeDialer(fc, rec), nil)
+	a.EnqueueAnnounce("3,x")
+	a.EnqueueAnnounce("3,y")
+
+	a.FlushForTest(context.Background())
+
+	_, sent, rejected, _ := a.Stats()
+	if sent != 1 {
+		t.Errorf("expected 1 sent, got %d", sent)
+	}
+	if rejected != 1 {
+		t.Errorf("expected 1 rejected, got %d", rejected)
+	}
+
+	// The rejected fid should be back in pending.
+	a.mu.Lock()
+	_, stillPending := a.pending["3,x"]
+	a.mu.Unlock()
+	if !stillPending {
+		t.Errorf("3,x should have been requeued after rejection")
+	}
+}
+
+func TestPeerAnnouncer_TTLRenewal(t *testing.T) {
+	fc := &fakeMountPeerClient{announcedBy: map[string][]filerAnnouncement{}}
+	rec := &dialRecorder{}
+
+	now := time.Unix(1000, 0)
+	a := NewPeerAnnouncer("self:18080", "", "", func(fid string) string {
+		return "owner:18081"
+	}, fakeDialer(fc, rec), nil)
+	a.clock = func() time.Time { return now }
+	a.announceInterval = 10 * time.Second
+	a.announceTTL = 60 * time.Second
+
+	// Announce once.
+	a.EnqueueAnnounce("3,a")
+	a.FlushForTest(context.Background())
+	if _, sent, _, _ := a.Stats(); sent != 1 {
+		t.Fatalf("want 1 sent on initial announce, got %d", sent)
+	}
+
+	// Time passes to within the renewal threshold.
+	now = now.Add(45 * time.Second)
+
+	// No new enqueue, but renewal should fire.
+	a.FlushForTest(context.Background())
+	if _, sent, _, _ := a.Stats(); sent != 2 {
+		t.Errorf("want 2 sent after renewal, got %d", sent)
+	}
+}
+
+// TestPeerAnnouncer_ReAnnouncesOnOwnerChange guards the fix for a
+// startup race where the writer's seed view reaches self-knowledge of
+// its peers in two stages: first just {self,A}, then {self,A,B}. HRW
+// may pick different owners across those views, so the announce from
+// stage 1 is sent to a mount that will NOT be the HRW owner by the
+// time a reader's seed view is the {self,A,B} version. Without
+// re-announcing on owner change, the fid silently ages out on the
+// wrong directory and readers get "no peer holder."
+func TestPeerAnnouncer_ReAnnouncesOnOwnerChange(t *testing.T) {
+	fc := &fakeMountPeerClient{announcedBy: map[string][]filerAnnouncement{}}
+	rec := &dialRecorder{}
+
+	ownerNow := "owner-1:18081"
+	a := NewPeerAnnouncer("self:18080", "", "", func(fid string) string {
+		return ownerNow
+	}, fakeDialer(fc, rec), nil)
+
+	a.EnqueueAnnounce("3,x")
+	a.FlushForTest(context.Background())
+	if _, sent, _, _ := a.Stats(); sent != 1 {
+		t.Fatalf("initial announce: want 1 sent, got %d", sent)
+	}
+
+	// Seed view shifts; HRW now picks a different owner for the same
+	// fid. A fresh-timestamped entry in announcedAt should be treated
+	// as stale in owner terms and re-announced even though its age is
+	// well within the TTL renewal window.
+	ownerNow = "owner-2:18081"
+	a.FlushForTest(context.Background())
+	if _, sent, _, _ := a.Stats(); sent != 2 {
+		t.Errorf("after owner change: want 2 sent, got %d", sent)
+	}
+	dialed := rec.snapshot()
+	if len(dialed) != 2 || dialed[0] == dialed[1] {
+		t.Errorf("want dial to both owners, got %v", dialed)
+	}
+}
+
+// TestPeerAnnouncer_DropsEvictedFids guards the write→announce race:
+// a chunk can be LRU-evicted between SetChunk and the next flush tick.
+// Advertising an evicted fid just hands remote fetchers a NOT_FOUND
+// from our FetchChunk server. With SetCachePresence wired, the
+// announcer must drop evicted fids before dispatching the RPC and
+// also clear them from announcedAt so they stop getting renewed.
+func TestPeerAnnouncer_DropsEvictedFids(t *testing.T) {
+	fc := &fakeMountPeerClient{announcedBy: map[string][]filerAnnouncement{}}
+	rec := &dialRecorder{}
+	present := map[string]bool{"3,here": true, "3,gone": false}
+
+	a := NewPeerAnnouncer("self:18080", "", "", func(fid string) string {
+		return "owner:18081"
+	}, fakeDialer(fc, rec), nil)
+	a.SetCachePresence(func(fid string) bool { return present[fid] })
+
+	a.EnqueueAnnounce("3,here")
+	a.EnqueueAnnounce("3,gone")
+	a.FlushForTest(context.Background())
+
+	if _, sent, _, _ := a.Stats(); sent != 1 {
+		t.Errorf("only the still-cached fid should be announced, got sent=%d", sent)
+	}
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+	for _, an := range fc.announcedBy["self:18080"] {
+		for _, fid := range an.fids {
+			if fid == "3,gone" {
+				t.Errorf("evicted fid should not have been announced, saw %q", fid)
+			}
+		}
+	}
+}
+
+// TestPeerAnnouncer_StopWaitsForFlush guards the shutdown race: Stop
+// must not return while a tick-triggered flushOnce is still dispatching
+// RPCs, otherwise the owning wfs can tear down the conn pool out from
+// under the in-flight sendTo goroutines.
+func TestPeerAnnouncer_StopWaitsForFlush(t *testing.T) {
+	started := make(chan struct{})
+	unblock := make(chan struct{})
+	slowDialer := func(ctx context.Context, peerAddr string) (mount_peer_pb.MountPeerClient, func(), error) {
+		close(started)
+		select {
+		case <-unblock:
+		case <-ctx.Done():
+		}
+		return nil, func() {}, context.DeadlineExceeded
+	}
+
+	a := NewPeerAnnouncer("self:18080", "", "", func(string) string {
+		return "owner:18081"
+	}, slowDialer, nil)
+
+	// Shortcut straight into a flush on a dedicated goroutine so we
+	// can assert Stop() blocks on it.
+	a.EnqueueAnnounce("3,slow")
+	flushDone := make(chan struct{})
+	go func() {
+		a.FlushForTest(context.Background())
+		close(flushDone)
+	}()
+	<-started
+
+	// Prove Stop() returns only after the flush finishes by racing
+	// them: Stop() normally can't cancel a FlushForTest because
+	// we're not going through run(); but unblocking after stop's
+	// goroutine starts lets us confirm flushDone happens before
+	// stopDone never does under the old code.
+	stopDone := make(chan struct{})
+	go func() {
+		a.Stop()
+		close(stopDone)
+	}()
+	close(unblock)
+
+	select {
+	case <-flushDone:
+	case <-time.After(3 * time.Second):
+		t.Fatalf("flush did not complete within deadline")
+	}
+	select {
+	case <-stopDone:
+	case <-time.After(3 * time.Second):
+		t.Fatalf("Stop() did not return after flush completed")
+	}
+}
+
+// TestPeerAnnouncer_SelfOwnedWritesToLocalDir guards the shortcut path
+// for fids whose HRW owner resolves to self: no RPC should go out, but
+// the local PeerDirectory must end up with a holder entry pointing at
+// self — otherwise remote ChunkLookup callers would get an empty
+// holder list for fids this mount owns directly.
+func TestPeerAnnouncer_SelfOwnedWritesToLocalDir(t *testing.T) {
+	fc := &fakeMountPeerClient{announcedBy: map[string][]filerAnnouncement{}}
+	rec := &dialRecorder{}
+	dir := NewPeerDirectory()
+	a := NewPeerAnnouncer("self:18080", "dc1", "rackA", func(fid string) string {
+		return "self:18080"
+	}, fakeDialer(fc, rec), dir)
+
+	a.EnqueueAnnounce("3,selfowned")
+	a.FlushForTest(context.Background())
+
+	if d := rec.snapshot(); len(d) != 0 {
+		t.Errorf("self-owned fid must not trigger any dial, got %v", d)
+	}
+	holders := dir.Lookup([]string{"3,selfowned"}, nil).PeersByFid["3,selfowned"]
+	if len(holders) != 1 {
+		t.Fatalf("local dir should have 1 holder after self-owned flush, got %d", len(holders))
+	}
+	if h := holders[0]; h.PeerAddr != "self:18080" || h.DataCenter != "dc1" || h.Rack != "rackA" {
+		t.Errorf("local dir entry wrong: %+v", h)
+	}
+	if _, sent, _, _ := a.Stats(); sent != 1 {
+		t.Errorf("sent counter should increment for self-owned too; got %d", sent)
+	}
+}
+
+func TestPeerAnnouncer_DialerErrorRequeues(t *testing.T) {
+	errDialer := func(ctx context.Context, peerAddr string) (mount_peer_pb.MountPeerClient, func(), error) {
+		return nil, func() {}, context.DeadlineExceeded
+	}
+	a := NewPeerAnnouncer("self:18080", "", "", func(fid string) string {
+		return "owner:18081"
+	}, errDialer, nil)
+	a.EnqueueAnnounce("3,a")
+	a.FlushForTest(context.Background())
+
+	_, _, _, errs := a.Stats()
+	if errs != 1 {
+		t.Errorf("expected 1 flush error, got %d", errs)
+	}
+	a.mu.Lock()
+	_, pending := a.pending["3,a"]
+	a.mu.Unlock()
+	if !pending {
+		t.Errorf("fid should be requeued after dial error")
+	}
+}

weed/mount/peer_dialer.go

@@ -0,0 +1,137 @@
+package mount
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/mount_peer_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// PeerConnPool caches one long-lived gRPC client connection per peer
+// address. Both the announcer flush loop and the read-path fetcher hit
+// the same handful of directory owners repeatedly; without a cache each
+// call would pay TCP handshake + HTTP/2 preface cost (and on TLS, an
+// additional handshake). The cache makes steady-state owner RPCs
+// effectively free after the first call.
+//
+// Connections in terminal failure states (Shutdown) are transparently
+// replaced on next access. Sizing: entries are ~1 KB + the conn itself;
+// bounded at maxPeerConnPoolEntries to contain runaway growth.
+type PeerConnPool struct {
+	dialOpts []grpc.DialOption
+
+	mu    sync.Mutex
+	conns map[string]*grpc.ClientConn
+}
+
+// maxPeerConnPoolEntries caps live peer conns per mount. A 10k-mount
+// fleet with HRW-sharded directory reaches only ~200 distinct owner
+// addresses per mount in the worst case, so this is far above any real
+// footprint while still bounding pathological growth.
+const maxPeerConnPoolEntries = 4096
+
+// NewPeerConnPool returns an empty pool. dialOpts should carry transport
+// credentials matching the server side (production wires
+// option.GrpcDialOption, which security.LoadClientTLS populates from
+// security.toml). When no options are supplied we fall back to insecure
+// cleartext — only safe for in-process tests.
+func NewPeerConnPool(dialOpts ...grpc.DialOption) *PeerConnPool {
+	if len(dialOpts) == 0 {
+		dialOpts = []grpc.DialOption{
+			grpc.WithTransportCredentials(insecure.NewCredentials()),
+		}
+	}
+	return &PeerConnPool{
+		dialOpts: dialOpts,
+		conns:    map[string]*grpc.ClientConn{},
+	}
+}
+
+// Dialer returns a MountPeerDialer bound to this pool. The returned
+// closeFn is a no-op — the pool owns the connection lifecycle. Tests
+// that want per-call dials can keep using the non-pooled variant.
+func (p *PeerConnPool) Dialer() MountPeerDialer {
+	return func(ctx context.Context, peerAddr string) (mount_peer_pb.MountPeerClient, func(), error) {
+		conn, err := p.get(peerAddr)
+		if err != nil {
+			return nil, func() {}, err
+		}
+		return mount_peer_pb.NewMountPeerClient(conn), func() {}, nil
+	}
+}
+
+func (p *PeerConnPool) get(peerAddr string) (*grpc.ClientConn, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if conn, ok := p.conns[peerAddr]; ok {
+		if conn.GetState() != connectivity.Shutdown {
+			return conn, nil
+		}
+		// Pooled conn is unusable — drop it and redial below.
+		_ = conn.Close()
+		delete(p.conns, peerAddr)
+	}
+	if len(p.conns) >= maxPeerConnPoolEntries {
+		// Evict one arbitrary entry. Simple over LRU: the pool is small
+		// in practice, and the victim will be re-dialed if needed.
+		for k, c := range p.conns {
+			_ = c.Close()
+			delete(p.conns, k)
+			break
+		}
+	}
+	conn, err := grpc.NewClient(peerAddr, p.dialOpts...)
+	if err != nil {
+		return nil, err
+	}
+	p.conns[peerAddr] = conn
+	return conn, nil
+}
+
+// Close tears down every cached connection. Safe to call multiple times.
+func (p *PeerConnPool) Close() {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	for addr, c := range p.conns {
+		_ = c.Close()
+		delete(p.conns, addr)
+	}
+}
+
+// Size returns the current number of cached connections — useful for
+// tests and metrics exports.
+func (p *PeerConnPool) Size() int {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return len(p.conns)
+}
+
+// DefaultMountPeerDialer returns a per-call dialer (no pooling). Kept for
+// tests and for any caller that genuinely wants a fresh connection per
+// invocation. Production code should prefer PeerConnPool.Dialer().
+//
+// Unused options silence dial-time lints when dialOpts is nil.
+func DefaultMountPeerDialer(dialOpts ...grpc.DialOption) MountPeerDialer {
+	opts := append([]grpc.DialOption{
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+	}, dialOpts...)
+	return func(ctx context.Context, peerAddr string) (mount_peer_pb.MountPeerClient, func(), error) {
+		conn, err := grpc.NewClient(peerAddr, opts...)
+		if err != nil {
+			return nil, func() {}, err
+		}
+		return mount_peer_pb.NewMountPeerClient(conn), func() { _ = conn.Close() }, nil
+	}
+}
+
+// peerConnMaxAge lets external callers (or future metrics) decide when a
+// pool entry is "stale" for monitoring purposes. The pool itself does not
+// expire on age — grpc handles reconnect internally.
+var peerConnMaxAge = 10 * time.Minute
+
+var _ = peerConnMaxAge // suppress unused-lint until a metric consumes it

weed/mount/peer_directory.go

@@ -0,0 +1,283 @@
+package mount
+
+import (
+	"sort"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// maxPeerDirectoryEntries caps the total (fid, holder) edges a single mount
+// will hold in its shard of the chunk directory. Each edge is a
+// (uint32, time.Time) map entry (~40 B with Go's map overhead), plus the
+// per-peer metadata is interned — see peerMetadata below — so 500k edges
+// come out to roughly 20 MB regardless of fleet size. A full directory
+// rejects new announces until TTL frees room.
+const maxPeerDirectoryEntries = 500000
+
+// peerID is the interned identifier for a peer. Peer addresses and locality
+// labels live once in the peers slice; the (fid -> holder) index references
+// peers by this fixed-size id so we avoid paying for the ~48 B worth of
+// string headers (addr, dc, rack) on every edge.
+type peerID uint32
+
+// peerMetadata is the per-peer data that used to be inlined on every
+// holderEntry. Fields are updated in place on each announce so a peer
+// reconfiguring its dc/rack is picked up without churn.
+type peerMetadata struct {
+	addr       string
+	dataCenter string
+	rack       string
+}
+
+// PeerDirectory is the tier-2 chunk directory: an in-memory
+// `fid -> set<peer>` map, holding entries only for fids for which this
+// mount is the HRW-assigned owner. Populated by inbound ChunkAnnounce RPCs,
+// queried by inbound ChunkLookup RPCs. Entries expire on TTL.
+//
+// Peer addresses are interned into peerID. That way the hot index stores
+// `fid -> (peerID -> expiry)`, a map of two fixed-size values, instead of
+// a full (addr, dc, rack, expiry) struct per edge. The peer slice itself
+// is bounded by fleet size (typically a few hundred), so the interning
+// table's memory cost is negligible next to the per-edge savings.
+//
+// Lookup takes only a read lock; TTL eviction is the sole responsibility of
+// the Sweep goroutine, which runs on a periodic ticker. This keeps the hot
+// read path concurrent even under heavy announce traffic.
+//
+// See design-weed-mount-peer-chunk-sharing.md §4.2.2.
+type PeerDirectory struct {
+	mu sync.RWMutex
+
+	// Interning table. peers[id] gives the peer's metadata; peerIDByAddr
+	// maps an address back to its id when a new announce comes in.
+	peers        []peerMetadata
+	peerIDByAddr map[string]peerID
+
+	// fid -> (peerID -> expiry). The inner map's value is a bare timestamp
+	// — metadata is looked up via peers[id] on demand.
+	index map[string]map[peerID]time.Time
+
+	// entryCount is maintained in lock-step with map mutations (under
+	// mu), so Stats() can report O(1) instead of scanning the index.
+	entryCount int
+
+	announces atomic.Int64
+	lookups   atomic.Int64
+	rejected  atomic.Int64
+	evictions atomic.Int64
+
+	clock func() time.Time
+}
+
+// NewPeerDirectory constructs a directory using the real wall clock.
+func NewPeerDirectory() *PeerDirectory {
+	return newPeerDirectoryWithClock(time.Now)
+}
+
+func newPeerDirectoryWithClock(clock func() time.Time) *PeerDirectory {
+	return &PeerDirectory{
+		index:        map[string]map[peerID]time.Time{},
+		peerIDByAddr: map[string]peerID{},
+		clock:        clock,
+	}
+}
+
+// AnnounceResult reports how an inbound announce was handled per-fid: fids
+// this receiver owns were recorded; others were rejected so the caller can
+// retry the correct owner after refreshing its seed view.
+type AnnounceResult struct {
+	Rejected []string
+}
+
+// OwnerCheck is a predicate provided by the caller (backed by HRW on the
+// current seed list) that says whether self is the owner for a given fid.
+type OwnerCheck func(fid string) bool
+
+// internPeer returns the interned id for peerAddr, allocating a new slot
+// if this is the first announce from that peer. Must be called under mu
+// (write). dc/rack are refreshed on each call so reconfig is picked up.
+func (d *PeerDirectory) internPeer(addr, dc, rack string) peerID {
+	if id, ok := d.peerIDByAddr[addr]; ok {
+		p := &d.peers[id]
+		p.dataCenter = dc
+		p.rack = rack
+		return id
+	}
+	id := peerID(len(d.peers))
+	d.peers = append(d.peers, peerMetadata{addr: addr, dataCenter: dc, rack: rack})
+	d.peerIDByAddr[addr] = id
+	return id
+}
+
+// Announce records the caller as a holder for every accepted fid. Any fid
+// for which ownerCheck returns false is rejected and surfaced back. A
+// directory at maxPeerDirectoryEntries silently drops *new* edges
+// (renewals of existing edges continue to succeed).
+func (d *PeerDirectory) Announce(peerAddr, dataCenter, rack string, fids []string, ttl time.Duration, ownerCheck OwnerCheck) AnnounceResult {
+	if ttl <= 0 {
+		ttl = 60 * time.Second
+	}
+	expiry := d.clock().Add(ttl)
+
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	pid := d.internPeer(peerAddr, dataCenter, rack)
+
+	result := AnnounceResult{}
+	for _, fid := range fids {
+		if ownerCheck != nil && !ownerCheck(fid) {
+			result.Rejected = append(result.Rejected, fid)
+			d.rejected.Add(1)
+			continue
+		}
+		holders, ok := d.index[fid]
+		if !ok {
+			if d.entryCount >= maxPeerDirectoryEntries {
+				continue
+			}
+			holders = map[peerID]time.Time{}
+			d.index[fid] = holders
+		}
+		if _, exists := holders[pid]; !exists {
+			if d.entryCount >= maxPeerDirectoryEntries {
+				continue
+			}
+			d.entryCount++
+		}
+		holders[pid] = expiry
+		d.announces.Add(1)
+	}
+	return result
+}
+
+// LookupHolder is the public form of a holder entry. DataCenter and Rack
+// are carried through so the fetcher can re-sort by its own locality
+// before picking a peer to dial.
+type LookupHolder struct {
+	PeerAddr   string
+	DataCenter string
+	Rack       string
+}
+
+// maxLookupHolders caps how many holders a single ChunkLookup response
+// returns per fid. The directory stores a holder per mount caching the
+// fid, which on a large hot file can reach fleet size (hundreds). The
+// fetcher typically only tries the first 1-3 before succeeding, so
+// returning every entry is pure wire-overhead. LRU ordering ensures the
+// freshest holders are the ones kept.
+const maxLookupHolders = 16
+
+// LookupResult holds the merged lookup response. Fids for which this
+// receiver is not the owner are split out so the caller can retry.
+type LookupResult struct {
+	PeersByFid   map[string][]LookupHolder
+	NotOwnerFids []string
+}
+
+// Lookup returns known holders for the requested fids. Expired edges are
+// filtered out of the response but NOT deleted here — Sweep handles that
+// under a write lock on its own schedule, so Lookup can stay read-only and
+// concurrent. A fid owned by this receiver with no known holders returns
+// an empty peer list.
+//
+// Holders are returned in LRU order (most recently announced first). Since
+// each Announce stamps a fresh `now + ttl` expiry, sorting by expiry
+// descending is equivalent to ordering by last-seen descending. Fetchers
+// should try these in the returned order — the most recent announcer is
+// the holder most likely to still have the chunk cached, given that each
+// mount's chunk_cache is itself LRU.
+func (d *PeerDirectory) Lookup(fids []string, ownerCheck OwnerCheck) LookupResult {
+	now := d.clock()
+	result := LookupResult{
+		PeersByFid: map[string][]LookupHolder{},
+	}
+
+	// Collect candidate holders under the read lock, including the peer
+	// metadata copies — Announce mutates peers[id] in place when an
+	// existing peer re-announces, so reading peers[pid] outside the lock
+	// would race with that write. Sorting happens after the unlock.
+	type candidate struct {
+		peerMetadata
+		expiry time.Time
+	}
+	perFid := make(map[string][]candidate, len(fids))
+
+	d.mu.RLock()
+	for _, fid := range fids {
+		d.lookups.Add(1)
+		if ownerCheck != nil && !ownerCheck(fid) {
+			result.NotOwnerFids = append(result.NotOwnerFids, fid)
+			continue
+		}
+		holders, ok := d.index[fid]
+		if !ok {
+			result.PeersByFid[fid] = nil
+			continue
+		}
+		cands := make([]candidate, 0, len(holders))
+		for pid, expiry := range holders {
+			if !expiry.After(now) {
+				continue // Sweep will clean this up
+			}
+			cands = append(cands, candidate{peerMetadata: d.peers[pid], expiry: expiry})
+		}
+		perFid[fid] = cands
+	}
+	d.mu.RUnlock()
+
+	for fid, cands := range perFid {
+		sort.Slice(cands, func(i, j int) bool {
+			if !cands[i].expiry.Equal(cands[j].expiry) {
+				return cands[i].expiry.After(cands[j].expiry) // newest first
+			}
+			return cands[i].addr < cands[j].addr // deterministic tiebreak
+		})
+		// Cap response at maxLookupHolders. The fetcher typically tries 1-3
+		// holders before succeeding; shipping every holder for hot fids is
+		// wire-overhead without hit-rate gain.
+		if len(cands) > maxLookupHolders {
+			cands = cands[:maxLookupHolders]
+		}
+		out := make([]LookupHolder, len(cands))
+		for i, c := range cands {
+			out[i] = LookupHolder{PeerAddr: c.addr, DataCenter: c.dataCenter, Rack: c.rack}
+		}
+		result.PeersByFid[fid] = out
+	}
+	return result
+}
+
+// Sweep purges expired edges. Safe to call periodically. Peer slots are
+// left in place — their count is bounded by fleet size (a few hundred at
+// most) and never grows faster than the fleet itself, so reclaiming them
+// isn't worth the bookkeeping.
+func (d *PeerDirectory) Sweep() int {
+	now := d.clock()
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	n := 0
+	for fid, holders := range d.index {
+		for pid, expiry := range holders {
+			if !expiry.After(now) {
+				delete(holders, pid)
+				d.entryCount--
+				n++
+			}
+		}
+		if len(holders) == 0 {
+			delete(d.index, fid)
+		}
+	}
+	d.evictions.Add(int64(n))
+	return n
+}
+
+// Stats exposes counters for observability. O(1) — entryCount is kept in
+// lock-step with map mutations.
+func (d *PeerDirectory) Stats() (announces, lookups, rejected, evictions int64, entries int) {
+	d.mu.RLock()
+	defer d.mu.RUnlock()
+	return d.announces.Load(), d.lookups.Load(), d.rejected.Load(), d.evictions.Load(), d.entryCount
+}

weed/mount/peer_directory_test.go

@@ -0,0 +1,217 @@
+package mount
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+	"time"
+)
+
+// alwaysOwner is a trivial OwnerCheck used by tests that aren't exercising
+// ownership rejection.
+func alwaysOwner(string) bool { return true }
+
+// ownerSet returns an OwnerCheck that accepts only the listed fids.
+func ownerSet(fids ...string) OwnerCheck {
+	m := map[string]struct{}{}
+	for _, f := range fids {
+		m[f] = struct{}{}
+	}
+	return func(fid string) bool {
+		_, ok := m[fid]
+		return ok
+	}
+}
+
+func TestPeerDirectory_AnnounceAndLookup(t *testing.T) {
+	now := time.Unix(1000, 0)
+	d := newPeerDirectoryWithClock(func() time.Time { return now })
+
+	res := d.Announce("mount-a:18080", "", "r1", []string{"3,a", "3,b"}, 60*time.Second, alwaysOwner)
+	if len(res.Rejected) != 0 {
+		t.Errorf("expected 0 rejected, got %v", res.Rejected)
+	}
+
+	lookup := d.Lookup([]string{"3,a", "3,b", "3,c"}, alwaysOwner)
+	if got := len(lookup.PeersByFid["3,a"]); got != 1 {
+		t.Errorf("3,a: expected 1 holder, got %d", got)
+	}
+	if got := len(lookup.PeersByFid["3,b"]); got != 1 {
+		t.Errorf("3,b: expected 1 holder, got %d", got)
+	}
+	if got := len(lookup.PeersByFid["3,c"]); got != 0 {
+		t.Errorf("3,c (unknown): expected 0 holders, got %d", got)
+	}
+}
+
+func TestPeerDirectory_OwnerRejection(t *testing.T) {
+	d := NewPeerDirectory()
+
+	res := d.Announce("mount-a:18080", "", "", []string{"3,mine", "3,yours"}, time.Minute, ownerSet("3,mine"))
+	if len(res.Rejected) != 1 || res.Rejected[0] != "3,yours" {
+		t.Errorf("expected 3,yours rejected, got %v", res.Rejected)
+	}
+
+	lookup := d.Lookup([]string{"3,mine", "3,yours"}, ownerSet("3,mine"))
+	if got := len(lookup.PeersByFid["3,mine"]); got != 1 {
+		t.Errorf("3,mine should have 1 holder, got %d", got)
+	}
+	if len(lookup.NotOwnerFids) != 1 || lookup.NotOwnerFids[0] != "3,yours" {
+		t.Errorf("expected 3,yours in not-owner list, got %v", lookup.NotOwnerFids)
+	}
+}
+
+func TestPeerDirectory_MultipleHolders(t *testing.T) {
+	d := NewPeerDirectory()
+	d.Announce("mount-a:18080", "", "r1", []string{"3,x"}, time.Minute, alwaysOwner)
+	d.Announce("mount-b:18080", "", "r2", []string{"3,x"}, time.Minute, alwaysOwner)
+
+	holders := d.Lookup([]string{"3,x"}, alwaysOwner).PeersByFid["3,x"]
+	if len(holders) != 2 {
+		t.Fatalf("expected 2 holders, got %d", len(holders))
+	}
+	sort.Slice(holders, func(i, j int) bool { return holders[i].PeerAddr < holders[j].PeerAddr })
+	if holders[0].PeerAddr != "mount-a:18080" || holders[1].PeerAddr != "mount-b:18080" {
+		t.Errorf("unexpected holder addrs: %+v", holders)
+	}
+}
+
+func TestPeerDirectory_RenewExtendsExpiry(t *testing.T) {
+	now := time.Unix(1000, 0)
+	d := newPeerDirectoryWithClock(func() time.Time { return now })
+
+	d.Announce("mount-a:18080", "", "", []string{"3,a"}, 30*time.Second, alwaysOwner)
+
+	now = now.Add(25 * time.Second)
+	d.Announce("mount-a:18080", "", "", []string{"3,a"}, 30*time.Second, alwaysOwner)
+
+	now = now.Add(10 * time.Second)
+	lookup := d.Lookup([]string{"3,a"}, alwaysOwner)
+	if len(lookup.PeersByFid["3,a"]) != 1 {
+		t.Errorf("renew should keep entry alive, got %d holders", len(lookup.PeersByFid["3,a"]))
+	}
+}
+
+func TestPeerDirectory_TTLExpiry(t *testing.T) {
+	now := time.Unix(1000, 0)
+	d := newPeerDirectoryWithClock(func() time.Time { return now })
+
+	d.Announce("mount-a:18080", "", "", []string{"3,a"}, 10*time.Second, alwaysOwner)
+	now = now.Add(15 * time.Second)
+	holders := d.Lookup([]string{"3,a"}, alwaysOwner).PeersByFid["3,a"]
+	if len(holders) != 0 {
+		t.Errorf("expected TTL-expired holder to be gone, got %+v", holders)
+	}
+}
+
+func TestPeerDirectory_Sweep(t *testing.T) {
+	now := time.Unix(1000, 0)
+	d := newPeerDirectoryWithClock(func() time.Time { return now })
+
+	d.Announce("mount-a:18080", "", "", []string{"3,a"}, 10*time.Second, alwaysOwner)
+	d.Announce("mount-b:18080", "", "", []string{"3,b"}, 60*time.Second, alwaysOwner)
+
+	now = now.Add(30 * time.Second)
+	got := d.Sweep()
+	if got != 1 {
+		t.Errorf("expected 1 swept entry, got %d", got)
+	}
+	_, _, _, _, entries := d.Stats()
+	if entries != 1 {
+		t.Errorf("expected 1 surviving entry, got %d", entries)
+	}
+}
+
+// TestPeerDirectory_LookupOrdersByRecency guards the LRU contract: the
+// most recently-announced holder comes first in the lookup response, so
+// fetchers that try holders in order hit the holder most likely to still
+// have the chunk cached locally.
+func TestPeerDirectory_LookupOrdersByRecency(t *testing.T) {
+	now := time.Unix(1000, 0)
+	d := newPeerDirectoryWithClock(func() time.Time { return now })
+
+	d.Announce("mount-a:18080", "", "", []string{"3,x"}, 60*time.Second, alwaysOwner)
+	now = now.Add(10 * time.Second)
+	d.Announce("mount-b:18080", "", "", []string{"3,x"}, 60*time.Second, alwaysOwner)
+	now = now.Add(10 * time.Second)
+	d.Announce("mount-c:18080", "", "", []string{"3,x"}, 60*time.Second, alwaysOwner)
+
+	got := d.Lookup([]string{"3,x"}, alwaysOwner).PeersByFid["3,x"]
+	if len(got) != 3 {
+		t.Fatalf("expected 3 holders, got %d", len(got))
+	}
+	want := []string{"mount-c:18080", "mount-b:18080", "mount-a:18080"}
+	for i, w := range want {
+		if got[i].PeerAddr != w {
+			t.Errorf("holder[%d] = %q, want %q (LRU order)", i, got[i].PeerAddr, w)
+		}
+	}
+}
+
+// TestPeerDirectory_LookupOrderAfterRenewal ensures renewing an older
+// holder promotes it to the head — matches LRU touch-on-access semantics
+// and keeps the freshest liveness signal in front.
+func TestPeerDirectory_LookupOrderAfterRenewal(t *testing.T) {
+	now := time.Unix(1000, 0)
+	d := newPeerDirectoryWithClock(func() time.Time { return now })
+
+	d.Announce("mount-a:18080", "", "", []string{"3,x"}, 60*time.Second, alwaysOwner)
+	now = now.Add(10 * time.Second)
+	d.Announce("mount-b:18080", "", "", []string{"3,x"}, 60*time.Second, alwaysOwner)
+	now = now.Add(10 * time.Second)
+	d.Announce("mount-a:18080", "", "", []string{"3,x"}, 60*time.Second, alwaysOwner)
+
+	got := d.Lookup([]string{"3,x"}, alwaysOwner).PeersByFid["3,x"]
+	if len(got) != 2 {
+		t.Fatalf("expected 2 holders, got %d", len(got))
+	}
+	if got[0].PeerAddr != "mount-a:18080" {
+		t.Errorf("after renewal, head should be mount-a; got %q", got[0].PeerAddr)
+	}
+}
+
+func TestPeerDirectory_StatsCountersIncrement(t *testing.T) {
+	d := NewPeerDirectory()
+	d.Announce("mount-a:18080", "", "", []string{"3,a", "3,b"}, time.Minute, alwaysOwner)
+	d.Lookup([]string{"3,a", "3,c"}, alwaysOwner)
+	d.Announce("mount-a:18080", "", "", []string{"3,nope"}, time.Minute, ownerSet("nothing"))
+
+	announces, lookups, rejected, _, _ := d.Stats()
+	if announces != 2 {
+		t.Errorf("announces counter: %d", announces)
+	}
+	if lookups != 2 {
+		t.Errorf("lookups counter: %d", lookups)
+	}
+	if rejected != 1 {
+		t.Errorf("rejected counter: %d", rejected)
+	}
+}
+
+// TestPeerDirectory_LookupCapsHolderList verifies that a hot fid with
+// more than maxLookupHolders holders returns only the top-N LRU entries.
+// The fetcher typically tries only the first 1-3 anyway, so caching
+// every holder on the wire is overhead.
+func TestPeerDirectory_LookupCapsHolderList(t *testing.T) {
+	now := time.Unix(1000, 0)
+	d := newPeerDirectoryWithClock(func() time.Time { return now })
+
+	// Announce 32 holders at strictly increasing timestamps so LRU order
+	// is well-defined and deterministic.
+	total := 2 * maxLookupHolders
+	for i := 0; i < total; i++ {
+		addr := fmt.Sprintf("mount-%02d:18080", i)
+		d.Announce(addr, "", "", []string{"3,hot"}, 60*time.Second, alwaysOwner)
+		now = now.Add(time.Millisecond)
+	}
+
+	got := d.Lookup([]string{"3,hot"}, alwaysOwner).PeersByFid["3,hot"]
+	if len(got) != maxLookupHolders {
+		t.Fatalf("expected %d holders in response, got %d", maxLookupHolders, len(got))
+	}
+	// Head should be the most recent (last announced).
+	want := fmt.Sprintf("mount-%02d:18080", total-1)
+	if got[0].PeerAddr != want {
+		t.Errorf("head of list: got %q want %q (LRU order)", got[0].PeerAddr, want)
+	}
+}

weed/mount/peer_fake_cache_test.go

@@ -0,0 +1,41 @@
+package mount
+
+// fakeChunkCache satisfies chunk_cache.ChunkCache with a simple in-memory
+// map keyed by fid. Just enough to exercise the peer-serve handler paths.
+type fakeChunkCache struct {
+	chunks map[string][]byte
+}
+
+func newFakeChunkCache() *fakeChunkCache {
+	return &fakeChunkCache{chunks: map[string][]byte{}}
+}
+
+func (f *fakeChunkCache) Put(fid string, data []byte) {
+	buf := make([]byte, len(data))
+	copy(buf, data)
+	f.chunks[fid] = buf
+}
+
+func (f *fakeChunkCache) ReadChunkAt(data []byte, fileId string, offset uint64) (int, error) {
+	b, ok := f.chunks[fileId]
+	if !ok {
+		return 0, nil
+	}
+	if int(offset) >= len(b) {
+		return 0, nil
+	}
+	return copy(data, b[offset:]), nil
+}
+
+func (f *fakeChunkCache) SetChunk(fileId string, data []byte) {
+	f.Put(fileId, data)
+}
+
+func (f *fakeChunkCache) IsInCache(fileId string, lockNeeded bool) bool {
+	_, ok := f.chunks[fileId]
+	return ok
+}
+
+func (f *fakeChunkCache) GetMaxFilePartSizeInCache() uint64 {
+	return 8 * 1024 * 1024
+}

weed/mount/peer_fetcher.go

@@ -0,0 +1,326 @@
+package mount
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"sort"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mount_peer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// errPeerReadSkipped signals that the read didn't go through the peer
+// path for a benign reason (local cache hit, no peer owner, etc.) rather
+// than a genuine failure. Callers fall through to the volume path just
+// as they would for a real error, but suppress the "peer read failed"
+// log that would otherwise mislead operators into thinking peer sharing
+// is broken.
+var errPeerReadSkipped = fmt.Errorf("peer read skipped (not an error)")
+
+// peerLookupTimeout bounds the ChunkLookup RPC. Short because the result
+// is consumed on the read critical path.
+const peerLookupTimeout = 500 * time.Millisecond
+
+// peerFetchTimeout bounds a single FetchChunk stream; on expiry we fall
+// through to the next holder or the volume server.
+const peerFetchTimeout = 5 * time.Second
+
+// maxPeerFetchChunkBytes caps how much we will accept from a single
+// FetchChunk stream. gRPC per-message size is already capped by the
+// server option; this is belt-and-suspenders against a runaway peer.
+const maxPeerFetchChunkBytes = 64 * 1024 * 1024
+
+// tryPeerRead attempts to satisfy a read from a peer mount's chunk cache.
+// Returns (bytesRead, modifiedTsNs, nil) on success. On any failure it
+// returns (0, 0, err) so the caller falls through to
+// entryChunkGroup.ReadDataAt (the volume-server path).
+//
+// Flow:
+//  1. Resolve the offset's leaf chunk (flattening manifests).
+//  2. Ask the HRW owner mount for current holders via ChunkLookup.
+//  3. For each holder (LRU order from PR #5), open a FetchChunk stream,
+//     assemble frames into a size-bounded buffer, and verify MD5
+//     end-to-end against FileChunk.ETag.
+//  4. On success, populate chunk_cache and enqueue an announce so
+//     other mounts can discover us as a new holder.
+//
+// chunks is a snapshot captured under the LockedEntry lock by the caller.
+func (fh *FileHandle) tryPeerRead(ctx context.Context, fileSize int64, buff []byte, offset int64, chunks []*filer_pb.FileChunk) (int64, int64, error) {
+	if fh.wfs.peerRegistrar == nil || fh.wfs.peerConnPool == nil {
+		return 0, 0, fmt.Errorf("peer sharing not configured")
+	}
+
+	// Resolve offset → leaf chunk, flattening any manifest indirection.
+	readStop := offset + int64(len(buff))
+	if readStop > fileSize {
+		readStop = fileSize
+	}
+	dataChunks, _, err := filer.ResolveChunkManifest(ctx, fh.wfs.LookupFn(), chunks, offset, readStop, fh.wfs.CacheInvalidator())
+	if err != nil {
+		return 0, 0, fmt.Errorf("resolve manifest: %w", err)
+	}
+	targetChunk, chunkOffset := findChunkContaining(dataChunks, offset)
+	if targetChunk == nil {
+		return 0, 0, fmt.Errorf("no leaf chunk for offset %d", offset)
+	}
+	// Reject reads that cross a chunk boundary. We only fetch one chunk
+	// here, and the caller (readFromChunks) maps a short non-error
+	// return to "success, zero-fill the rest" — which would silently
+	// corrupt reads that should actually span two chunks. Bail so the
+	// fallback ReadDataAt path can handle the multi-chunk case
+	// correctly. See weedfs_file_read.go short-read semantics.
+	chunkEnd := targetChunk.Offset + int64(targetChunk.Size)
+	if readStop > chunkEnd {
+		return 0, 0, errPeerReadSkipped
+	}
+
+	// Fail fast when the chunk is already cached locally: the fallback
+	// ReadDataAt path will satisfy the read from chunkCache with no RPCs,
+	// so dialing a peer (ChunkLookup + FetchChunk) would be pure overhead.
+	if fh.wfs.chunkCache != nil && fh.wfs.chunkCache.IsInCache(targetChunk.FileId, true) {
+		return 0, 0, errPeerReadSkipped
+	}
+
+	selfAddr := ""
+	if fh.wfs.peerGrpcServer != nil {
+		selfAddr = fh.wfs.peerGrpcServer.SelfAddr()
+	}
+
+	owner := fh.wfs.peerRegistrar.OwnerFor(targetChunk.FileId)
+	if owner == "" || owner == selfAddr {
+		return 0, 0, fmt.Errorf("no peer owner for fid %s", targetChunk.FileId)
+	}
+
+	holders, err := peerLookupHolders(ctx, fh.wfs.peerConnPool.Dialer(), owner, targetChunk.FileId)
+	if err != nil {
+		return 0, 0, fmt.Errorf("peer lookup: %w", err)
+	}
+	if len(holders) == 0 {
+		return 0, 0, fmt.Errorf("no peer holder for fid %s", targetChunk.FileId)
+	}
+
+	// Re-rank holders by locality (same rack > same DC > elsewhere), keeping
+	// the server's LRU order stable within each bucket. The server caps the
+	// list at maxLookupHolders, so this is always a small N.
+	sortHoldersByLocality(holders, fh.wfs.option.PeerDataCenter, fh.wfs.option.PeerRack)
+
+	dial := fh.wfs.peerConnPool.Dialer()
+	for _, h := range holders {
+		if h.addr == selfAddr {
+			continue
+		}
+		data, ferr := fetchChunkFromPeer(ctx, dial, h.addr, targetChunk.FileId, targetChunk.Size, targetChunk.ETag)
+		if ferr != nil {
+			glog.V(2).Infof("peer-fetch %s from %s: %v", targetChunk.FileId, h.addr, ferr)
+			continue
+		}
+		if fh.wfs.chunkCache != nil {
+			fh.wfs.chunkCache.SetChunk(targetChunk.FileId, data)
+		}
+		if fh.wfs.peerAnnouncer != nil {
+			fh.wfs.peerAnnouncer.EnqueueAnnounce(targetChunk.FileId)
+		}
+		if chunkOffset >= int64(len(data)) {
+			return 0, 0, fmt.Errorf("peer returned short chunk")
+		}
+		// Cap the copy to whichever is smaller: the caller's buffer, or
+		// the remaining bytes before logical EOF (readStop already
+		// clamped to fileSize). FileChunk.Size can legitimately exceed
+		// the logical file length when the last chunk is partially
+		// written and the filer stored the full padded buffer, so a
+		// naïve copy(buff, data[chunkOffset:]) would return bytes past
+		// EOF. The caller treats a short non-error return as success
+		// and zero-fills the tail, so returning too many bytes here is
+		// a correctness bug, not just wasted I/O.
+		remaining := readStop - offset
+		available := int64(len(data)) - chunkOffset
+		maxCopy := int64(len(buff))
+		if remaining < maxCopy {
+			maxCopy = remaining
+		}
+		if available < maxCopy {
+			maxCopy = available
+		}
+		copied := copy(buff[:maxCopy], data[chunkOffset:chunkOffset+maxCopy])
+		return int64(copied), targetChunk.ModifiedTsNs, nil
+	}
+	return 0, 0, fmt.Errorf("no peer served fid %s", targetChunk.FileId)
+}
+
+func findChunkContaining(chunks []*filer_pb.FileChunk, offset int64) (*filer_pb.FileChunk, int64) {
+	for _, c := range chunks {
+		start := c.Offset
+		stop := c.Offset + int64(c.Size)
+		if offset >= start && offset < stop {
+			return c, offset - start
+		}
+	}
+	return nil, 0
+}
+
+// peerHolder is a holder entry carried through the fetcher: addr for
+// dialing plus the DC/Rack labels the owner recorded at announce time, so
+// the fetcher can re-rank by its own locality before dialing.
+type peerHolder struct {
+	addr string
+	dc   string
+	rack string
+}
+
+// peerLookupHolders calls ChunkLookup on the given HRW owner and returns
+// the holders in the server-reported (LRU) order along with their locality
+// labels. The pooled dialer is passed in from the caller.
+func peerLookupHolders(ctx context.Context, dial MountPeerDialer, ownerAddr, fid string) ([]peerHolder, error) {
+	client, closeFn, err := dial(ctx, ownerAddr)
+	if err != nil {
+		return nil, err
+	}
+	defer closeFn()
+
+	callCtx, cancel := context.WithTimeout(ctx, peerLookupTimeout)
+	defer cancel()
+	resp, err := client.ChunkLookup(callCtx, &mount_peer_pb.ChunkLookupRequest{FileIds: []string{fid}})
+	if err != nil {
+		return nil, err
+	}
+	set, ok := resp.PeersByFid[fid]
+	if !ok || set == nil {
+		return nil, nil
+	}
+	out := make([]peerHolder, 0, len(set.Peers))
+	for _, p := range set.Peers {
+		out = append(out, peerHolder{addr: p.PeerAddr, dc: p.DataCenter, rack: p.Rack})
+	}
+	return out, nil
+}
+
+// localityBucket scores how "close" a peer is to self. Lower is better:
+//
+//	0 = same rack in same DC      (shortest hop)
+//	1 = same DC, different rack   (cross-rack but still intra-DC)
+//	2 = different DC, or unknown  (anything else)
+//
+// Missing labels on either side fall to bucket 2 — we don't claim locality
+// we can't prove.
+func localityBucket(selfDC, selfRack, peerDC, peerRack string) int {
+	if selfDC == "" || peerDC == "" || selfDC != peerDC {
+		return 2
+	}
+	if selfRack != "" && peerRack != "" && selfRack == peerRack {
+		return 0
+	}
+	return 1
+}
+
+// sortHoldersByLocality stable-sorts holders so the most local peers are
+// tried first. The server-returned order is LRU (freshest holder first),
+// and sort.SliceStable preserves that ordering within each locality
+// bucket, so among equally-local peers we still prefer the freshest.
+func sortHoldersByLocality(holders []peerHolder, selfDC, selfRack string) {
+	if len(holders) < 2 {
+		return
+	}
+	sort.SliceStable(holders, func(i, j int) bool {
+		return localityBucket(selfDC, selfRack, holders[i].dc, holders[i].rack) <
+			localityBucket(selfDC, selfRack, holders[j].dc, holders[j].rack)
+	})
+}
+
+// fetchChunkFromPeer server-streams a chunk from the given peer and
+// verifies the MD5 of the assembled bytes against expectedETag.
+//
+// expectedSize (when > 0) is the authoritative chunk length from the
+// filer entry; we pre-allocate exactly that to avoid slice growth and
+// reject streams that overshoot it. A zero expectedSize falls back to
+// maxPeerFetchChunkBytes as a safety ceiling.
+func fetchChunkFromPeer(ctx context.Context, dial MountPeerDialer, peerAddr, fid string, expectedSize uint64, expectedETag string) ([]byte, error) {
+	client, closeFn, err := dial(ctx, peerAddr)
+	if err != nil {
+		return nil, err
+	}
+	defer closeFn()
+
+	callCtx, cancel := context.WithTimeout(ctx, peerFetchTimeout)
+	defer cancel()
+
+	stream, err := client.FetchChunk(callCtx, &mount_peer_pb.FetchChunkRequest{
+		FileId:       fid,
+		ExpectedEtag: expectedETag,
+		ExpectedSize: expectedSize,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// capHint pre-sizes the assembly buffer from the filer-reported chunk
+	// size. When the caller didn't know the size (expectedSize == 0), we
+	// let append grow the buffer rather than reserve the 64 MiB ceiling —
+	// typical chunks are a few MiB, and the maxPeerFetchChunkBytes check
+	// during Recv is the real safety ceiling.
+	capHint := expectedSize
+	if capHint > maxPeerFetchChunkBytes {
+		capHint = maxPeerFetchChunkBytes
+	}
+	buf := make([]byte, 0, capHint)
+
+	for {
+		resp, rerr := stream.Recv()
+		if rerr == io.EOF {
+			break
+		}
+		if rerr != nil {
+			if status.Code(rerr) == codes.NotFound {
+				return nil, fmt.Errorf("peer not cached")
+			}
+			return nil, rerr
+		}
+		if len(buf)+len(resp.Data) > int(maxPeerFetchChunkBytes) {
+			return nil, fmt.Errorf("peer response exceeds max chunk size %d", maxPeerFetchChunkBytes)
+		}
+		buf = append(buf, resp.Data...)
+	}
+
+	if expectedSize > 0 && uint64(len(buf)) != expectedSize {
+		return nil, fmt.Errorf("peer returned %d bytes, expected %d", len(buf), expectedSize)
+	}
+
+	// Per-chunk integrity check — the peer is treated as untrusted.
+	// FileChunk.ETag is UploadResult.ContentMd5, which is base64 of the
+	// raw 16-byte MD5 (see weed/operation/upload_content.go:64 and
+	// filer's ETagChunks which decodes via util.Base64Md5ToBytes).
+	// Compare raw digests so we don't reject every valid peer response
+	// because of an encoding mismatch.
+	if expectedETag != "" {
+		got := md5.Sum(buf)
+		if !etagMatchesMD5(expectedETag, got[:]) {
+			return nil, fmt.Errorf("etag mismatch: peer=%s got=%x want=%s", peerAddr, got, expectedETag)
+		}
+	}
+	return buf, nil
+}
+
+// etagMatchesMD5 compares a stored FileChunk.ETag string to a raw 16-byte
+// MD5 digest. FileChunk.ETag is produced as base64(md5) on upload, but
+// some older chunks (and tests) use hex(md5). Accept either.
+func etagMatchesMD5(etag string, rawMD5 []byte) bool {
+	if len(rawMD5) != md5.Size {
+		return false
+	}
+	if dec := util.Base64Md5ToBytes(etag); len(dec) == md5.Size {
+		return bytes.Equal(dec, rawMD5)
+	}
+	if dec, err := hex.DecodeString(etag); err == nil && len(dec) == md5.Size {
+		return bytes.Equal(dec, rawMD5)
+	}
+	return false
+}

weed/mount/peer_fetcher_test.go

@@ -0,0 +1,180 @@
+package mount
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/base64"
+	"encoding/hex"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// newTestFetchServer stands up a minimal MountPeer gRPC server that only
+// serves FetchChunk out of a fakeChunkCache. Goes through the same
+// PeerGrpcServer.Start path production uses (via pb.NewGrpcServer with
+// the standard keepalive + msg-size options) so future default server
+// options are exercised by these tests too.
+func newTestFetchServer(t *testing.T, cache *fakeChunkCache) (addr string, stop func()) {
+	t.Helper()
+	dir := NewPeerDirectory()
+	srv := NewPeerGrpcServer(cache, dir, nil, "")
+	if err := srv.Start("127.0.0.1:0"); err != nil {
+		t.Fatalf("start: %v", err)
+	}
+	return srv.Addr(), func() { srv.Stop() }
+}
+
+func etagOf(b []byte) string {
+	sum := md5.Sum(b)
+	return hex.EncodeToString(sum[:])
+}
+
+// etagOfBase64 mirrors how the real upload path produces FileChunk.ETag:
+// base64 of the raw 16-byte MD5 digest. Kept alongside the hex form so
+// we can test that both encodings work end-to-end.
+func etagOfBase64(b []byte) string {
+	sum := md5.Sum(b)
+	return base64.StdEncoding.EncodeToString(sum[:])
+}
+
+func TestFetchChunkFromPeer_Hit(t *testing.T) {
+	cache := newFakeChunkCache()
+	payload := []byte("hello from peer stream")
+	cache.Put("3,abc", payload)
+	addr, stop := newTestFetchServer(t, cache)
+	defer stop()
+
+	dial := DefaultMountPeerDialer(grpc.WithTransportCredentials(insecure.NewCredentials()))
+	got, err := fetchChunkFromPeer(context.Background(), dial, addr, "3,abc", uint64(len(payload)), etagOf(payload))
+	if err != nil {
+		t.Fatalf("fetch: %v", err)
+	}
+	if string(got) != string(payload) {
+		t.Errorf("bytes mismatch: got %q want %q", got, payload)
+	}
+}
+
+// TestFetchChunkFromPeer_Base64Etag mirrors the real upload path where
+// FileChunk.ETag is base64(md5) rather than hex(md5). A regression
+// there would reject every valid peer response in production.
+func TestFetchChunkFromPeer_Base64Etag(t *testing.T) {
+	cache := newFakeChunkCache()
+	payload := []byte("base64-etag peer bytes")
+	cache.Put("3,b64", payload)
+	addr, stop := newTestFetchServer(t, cache)
+	defer stop()
+
+	dial := DefaultMountPeerDialer(grpc.WithTransportCredentials(insecure.NewCredentials()))
+	got, err := fetchChunkFromPeer(context.Background(), dial, addr, "3,b64", uint64(len(payload)), etagOfBase64(payload))
+	if err != nil {
+		t.Fatalf("fetch: %v", err)
+	}
+	if string(got) != string(payload) {
+		t.Errorf("bytes mismatch: got %q want %q", got, payload)
+	}
+}
+
+func TestFetchChunkFromPeer_EtagMismatch(t *testing.T) {
+	cache := newFakeChunkCache()
+	payload := []byte("unexpected bytes")
+	cache.Put("3,abc", payload)
+	addr, stop := newTestFetchServer(t, cache)
+	defer stop()
+
+	dial := DefaultMountPeerDialer(grpc.WithTransportCredentials(insecure.NewCredentials()))
+	_, err := fetchChunkFromPeer(context.Background(), dial, addr, "3,abc", 0, "not-the-real-etag")
+	if err == nil {
+		t.Fatalf("expected etag mismatch error, got nil")
+	}
+}
+
+func TestFetchChunkFromPeer_NotFound(t *testing.T) {
+	cache := newFakeChunkCache()
+	addr, stop := newTestFetchServer(t, cache)
+	defer stop()
+
+	dial := DefaultMountPeerDialer(grpc.WithTransportCredentials(insecure.NewCredentials()))
+	_, err := fetchChunkFromPeer(context.Background(), dial, addr, "3,missing", 0, "")
+	if err == nil {
+		t.Errorf("expected error for missing fid, got nil")
+	}
+}
+
+// TestSortHoldersByLocality verifies that same-rack peers sort ahead of
+// same-DC-different-rack peers, which in turn sort ahead of cross-DC
+// peers, and that LRU order is preserved within each bucket.
+func TestSortHoldersByLocality(t *testing.T) {
+	selfDC, selfRack := "dc1", "r1"
+	// Input order mimics a server-side LRU list (newest first).
+	holders := []peerHolder{
+		{addr: "far-newest", dc: "dc2", rack: "r9"},  // bucket 2 (diff DC)
+		{addr: "mid-newer", dc: "dc1", rack: "r2"},   // bucket 1 (same DC, diff rack)
+		{addr: "local-newer", dc: "dc1", rack: "r1"}, // bucket 0 (same rack)
+		{addr: "far-older", dc: "dc2", rack: "r9"},   // bucket 2
+		{addr: "mid-older", dc: "dc1", rack: "r2"},   // bucket 1
+		{addr: "local-older", dc: "dc1", rack: "r1"}, // bucket 0
+		{addr: "unlabeled-older", dc: "", rack: ""},  // bucket 2 (unknown)
+	}
+
+	sortHoldersByLocality(holders, selfDC, selfRack)
+
+	want := []string{
+		"local-newer", "local-older",
+		"mid-newer", "mid-older",
+		"far-newest", "far-older", "unlabeled-older",
+	}
+	if len(holders) != len(want) {
+		t.Fatalf("len got %d want %d", len(holders), len(want))
+	}
+	for i, w := range want {
+		if holders[i].addr != w {
+			t.Errorf("pos %d: got %q want %q (full: %+v)", i, holders[i].addr, w, holders)
+		}
+	}
+}
+
+// TestSortHoldersByLocality_NoSelfLabels — when the caller has no DC/rack
+// labels, every peer falls to bucket 2 and the server-returned LRU order
+// must pass through unchanged.
+func TestSortHoldersByLocality_NoSelfLabels(t *testing.T) {
+	holders := []peerHolder{
+		{addr: "a", dc: "dc1", rack: "r1"},
+		{addr: "b", dc: "dc2", rack: "r2"},
+		{addr: "c", dc: "", rack: ""},
+	}
+	sortHoldersByLocality(holders, "", "")
+	want := []string{"a", "b", "c"}
+	for i, w := range want {
+		if holders[i].addr != w {
+			t.Errorf("pos %d: got %q want %q", i, holders[i].addr, w)
+		}
+	}
+}
+
+func TestFetchChunkFromPeer_MultiFrameChunkAssembledCorrectly(t *testing.T) {
+	cache := newFakeChunkCache()
+	// Just over 2× the stream frame size so we get at least three frames.
+	payload := make([]byte, fetchChunkStreamSize*2+42)
+	for i := range payload {
+		payload[i] = byte((i * 7) & 0xff)
+	}
+	cache.Put("3,large", payload)
+	addr, stop := newTestFetchServer(t, cache)
+	defer stop()
+
+	dial := DefaultMountPeerDialer(grpc.WithTransportCredentials(insecure.NewCredentials()))
+	got, err := fetchChunkFromPeer(context.Background(), dial, addr, "3,large", uint64(len(payload)), etagOf(payload))
+	if err != nil {
+		t.Fatalf("fetch: %v", err)
+	}
+	if len(got) != len(payload) {
+		t.Fatalf("len got %d want %d", len(got), len(payload))
+	}
+	for i := range payload {
+		if got[i] != payload[i] {
+			t.Fatalf("mismatch at offset %d", i)
+		}
+	}
+}

weed/mount/peer_grpc.go

@@ -0,0 +1,258 @@
+package mount
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mount_peer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
+	"github.com/seaweedfs/seaweedfs/weed/util/mem"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// fetchChunkStreamSize is the frame size used when server-streaming a chunk's
+// bytes back to a peer. 1 MiB is well above gRPC's 32 KiB preferred frame but
+// comfortably under the default 4 MiB message cap, so each Recv on the client
+// returns quickly and the chunk is assembled with ~16 Recv calls for typical
+// 16 MiB chunks.
+const fetchChunkStreamSize = 1 * 1024 * 1024
+
+// maxFetchChunkBytes caps the size of a single FetchChunk read buffer.
+// The caller's expected_size is untrusted — a misbehaving peer could
+// claim 1 TiB and OOM the server. 64 MiB is well above any realistic
+// chunk (SeaweedFS defaults to 16 MiB, filer manifests cap at 32 MiB);
+// anything larger is treated as invalid input.
+const maxFetchChunkBytes = 64 * 1024 * 1024
+
+// PeerGrpcServer is the single mount-to-mount gRPC endpoint. It serves:
+//   - ChunkAnnounce / ChunkLookup — the tier-2 directory RPCs, populated
+//     by inbound announces and queried by inbound lookups. Each handler
+//     is HRW-gated on the caller-side seed view.
+//   - FetchChunk (server stream) — serves bytes from the local
+//     chunk_cache to peers. Replaces the earlier HTTP-only peer-serve
+//     endpoint: one port, one authentication path, one connection pool.
+//
+// Transport credentials (TLS/mTLS) are injected via serverOpts. When the
+// caller passes nothing the server runs plaintext — only appropriate on
+// a trusted single-host test cluster. Production wiring passes
+// security.LoadServerTLS from security.toml so cross-host traffic is
+// authenticated + encrypted.
+type PeerGrpcServer struct {
+	mount_peer_pb.UnimplementedMountPeerServer
+
+	dir        *PeerDirectory
+	cache      chunk_cache.ChunkCache
+	ownerFor   func(fid string) string // HRW owner predicate on current seeds
+	selfAddr   string
+	serverOpts []grpc.ServerOption
+	grpcS      *grpc.Server
+	listener   net.Listener
+	stopped    bool
+}
+
+// NewPeerGrpcServer constructs the server. cache is the local chunk_cache
+// (used to serve FetchChunk); dir is the local directory shard (used to
+// answer ChunkAnnounce / ChunkLookup); ownerFor returns the HRW owner of a
+// fid on the current seed view; serverOpts are forwarded to the underlying
+// gRPC server (use grpc.Creds(...) for TLS/mTLS).
+func NewPeerGrpcServer(cache chunk_cache.ChunkCache, dir *PeerDirectory, ownerFor func(fid string) string, selfAddr string, serverOpts ...grpc.ServerOption) *PeerGrpcServer {
+	return &PeerGrpcServer{
+		cache:      cache,
+		dir:        dir,
+		ownerFor:   ownerFor,
+		selfAddr:   selfAddr,
+		serverOpts: serverOpts,
+	}
+}
+
+// Start binds a TCP listener at addr and registers the MountPeer service.
+func (s *PeerGrpcServer) Start(addr string) error {
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		return fmt.Errorf("peer grpc listen %s: %w", addr, err)
+	}
+	s.listener = ln
+	s.grpcS = pb.NewGrpcServer(s.serverOpts...)
+	mount_peer_pb.RegisterMountPeerServer(s.grpcS, s)
+	go func() {
+		if err := s.grpcS.Serve(ln); err != nil && err != grpc.ErrServerStopped {
+			glog.Warningf("peer-grpc terminated: %v", err)
+		}
+	}()
+	glog.V(0).Infof("peer-grpc listening on %s", ln.Addr())
+	return nil
+}
+
+// Stop halts the gRPC server without waiting for in-flight streams.
+func (s *PeerGrpcServer) Stop() {
+	if s.stopped {
+		return
+	}
+	s.stopped = true
+	if s.grpcS != nil {
+		s.grpcS.Stop()
+	}
+}
+
+// Addr returns the bound address (useful when the caller used ":0").
+func (s *PeerGrpcServer) Addr() string {
+	if s.listener == nil {
+		return ""
+	}
+	return s.listener.Addr().String()
+}
+
+// SelfAddr returns the advertise address this server was constructed with —
+// the identity the fetcher compares against to avoid dialing itself.
+func (s *PeerGrpcServer) SelfAddr() string { return s.selfAddr }
+
+// ChunkAnnounce accepts holder entries for fids this mount owns; rejects
+// others so the caller can retry against the correct owner.
+func (s *PeerGrpcServer) ChunkAnnounce(ctx context.Context, req *mount_peer_pb.ChunkAnnounceRequest) (*mount_peer_pb.ChunkAnnounceResponse, error) {
+	ttl := time.Duration(req.TtlSeconds) * time.Second
+	res := s.dir.Announce(req.PeerAddr, req.DataCenter, req.Rack, req.FileIds, ttl, s.ownerPredicate)
+	return &mount_peer_pb.ChunkAnnounceResponse{
+		RejectedFileIds: res.Rejected,
+	}, nil
+}
+
+// ChunkLookup returns known holders for each requested fid in LRU order.
+func (s *PeerGrpcServer) ChunkLookup(ctx context.Context, req *mount_peer_pb.ChunkLookupRequest) (*mount_peer_pb.ChunkLookupResponse, error) {
+	res := s.dir.Lookup(req.FileIds, s.ownerPredicate)
+	resp := &mount_peer_pb.ChunkLookupResponse{
+		PeersByFid:      make(map[string]*mount_peer_pb.PeerSet, len(res.PeersByFid)),
+		NotOwnerFileIds: res.NotOwnerFids,
+	}
+	for fid, holders := range res.PeersByFid {
+		peers := &mount_peer_pb.PeerSet{}
+		for _, h := range holders {
+			peers.Peers = append(peers.Peers, &mount_peer_pb.PeerInfo{
+				PeerAddr:   h.PeerAddr,
+				DataCenter: h.DataCenter,
+				Rack:       h.Rack,
+			})
+		}
+		resp.PeersByFid[fid] = peers
+	}
+	return resp, nil
+}
+
+// FetchChunk streams bytes of a cached chunk back to the caller. Missing
+// fid → gRPC NOT_FOUND. Bytes are framed at fetchChunkStreamSize so gRPC's
+// default 4 MiB message cap does not constrain chunk size.
+func (s *PeerGrpcServer) FetchChunk(req *mount_peer_pb.FetchChunkRequest, stream mount_peer_pb.MountPeer_FetchChunkServer) error {
+	if s.cache == nil {
+		return status.Error(codes.Unavailable, "chunk cache not configured")
+	}
+	fid := req.FileId
+	if fid == "" {
+		return status.Error(codes.InvalidArgument, "missing file_id")
+	}
+
+	// Size the read buffer to the caller-reported chunk length. The
+	// TieredChunkCache.ReadChunkAt wrapper only returns success when n
+	// equals len(data), so a buffer larger than the actual stored chunk
+	// (e.g. an 8 MiB buffer for a 2 MiB chunk) makes every read look
+	// like a miss. Client sends expected_size from FileChunk.Size so we
+	// allocate exactly the right amount. Fall back to the max-part-size
+	// when the caller left it zero (older clients) and hope the chunk
+	// is that big.
+	//
+	// expected_size is untrusted — cap at maxFetchChunkBytes to prevent
+	// a misbehaving peer from requesting an OOM-sized allocation.
+	if req.ExpectedSize > maxFetchChunkBytes {
+		return status.Errorf(codes.InvalidArgument,
+			"expected_size %d exceeds max %d", req.ExpectedSize, maxFetchChunkBytes)
+	}
+	if req.Length > maxFetchChunkBytes {
+		return status.Errorf(codes.InvalidArgument,
+			"length %d exceeds max %d", req.Length, maxFetchChunkBytes)
+	}
+	readSize := int(req.ExpectedSize)
+	if readSize <= 0 {
+		max := s.cache.GetMaxFilePartSizeInCache()
+		if max == 0 {
+			max = 8 * 1024 * 1024
+		}
+		readSize = int(max)
+	}
+	// mem.Allocate rounds up to the nearest power-of-2 slot backed by a
+	// shared sync.Pool; avoids an allocation per FetchChunk call.
+	buf := mem.Allocate(readSize)
+	defer mem.Free(buf)
+
+	n, err := s.cache.ReadChunkAt(buf, fid, 0)
+	if err != nil || n <= 0 {
+		return status.Errorf(codes.NotFound, "fid %s not cached", fid)
+	}
+
+	// Apply optional offset / length range to the whole-chunk buffer.
+	// Default (both 0) is a full-chunk transfer starting at byte 0,
+	// which is what the fetcher uses today. Non-default is reserved
+	// for future range-read callers.
+	lo := int(req.Offset)
+	if lo < 0 || lo > n {
+		return status.Errorf(codes.OutOfRange, "offset %d outside chunk length %d", lo, n)
+	}
+	hi := n
+	if req.Length > 0 {
+		hi = lo + int(req.Length)
+		if hi > n {
+			hi = n
+		}
+	}
+
+	for off := lo; off < hi; off += fetchChunkStreamSize {
+		end := off + fetchChunkStreamSize
+		if end > hi {
+			end = hi
+		}
+		if sendErr := stream.Send(&mount_peer_pb.FetchChunkResponse{
+			Data: buf[off:end],
+		}); sendErr != nil {
+			return sendErr
+		}
+	}
+	return nil
+}
+
+func (s *PeerGrpcServer) ownerPredicate(fid string) bool {
+	if s.ownerFor == nil {
+		return true // no HRW configured → accept all (single-mount mode)
+	}
+	return s.ownerFor(fid) == s.selfAddr
+}
+
+// peerDirectorySweepInterval is how often the mount evicts expired
+// directory entries. Lookup no longer deletes inline (it takes only an
+// RLock), so this sweeper is the sole memory reclamation path.
+const peerDirectorySweepInterval = 60 * time.Second
+
+// runPeerDirectorySweeper runs until stopCh closes. Stopping it on
+// unmount prevents the goroutine from outliving the WFS, which matters
+// for tests and for any embedded use of NewSeaweedFileSystem where the
+// filesystem object is recreated without a process exit.
+func (wfs *WFS) runPeerDirectorySweeper(stopCh <-chan struct{}) {
+	ticker := time.NewTicker(peerDirectorySweepInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-stopCh:
+			return
+		case <-ticker.C:
+			dir := wfs.peerDirectory
+			if dir == nil {
+				return
+			}
+			if evicted := dir.Sweep(); evicted > 0 {
+				glog.V(2).Infof("peer directory: evicted %d expired entries", evicted)
+			}
+		}
+	}
+}

weed/mount/peer_grpc_test.go

@@ -0,0 +1,150 @@
+package mount
+
+import (
+	"context"
+	"io"
+	"net"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/mount_peer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/status"
+)
+
+func newTestPeerGrpc(t *testing.T, cache chunk_cache.ChunkCache, ownerFor func(fid string) string, selfAddr string) (mount_peer_pb.MountPeerClient, func()) {
+	t.Helper()
+	dir := NewPeerDirectory()
+	srv := NewPeerGrpcServer(cache, dir, ownerFor, selfAddr)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	srv.listener = ln
+	srv.grpcS = grpc.NewServer()
+	mount_peer_pb.RegisterMountPeerServer(srv.grpcS, srv)
+	go srv.grpcS.Serve(ln)
+
+	conn, err := grpc.NewClient(ln.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("dial: %v", err)
+	}
+	client := mount_peer_pb.NewMountPeerClient(conn)
+	cleanup := func() {
+		conn.Close()
+		srv.Stop()
+	}
+	return client, cleanup
+}
+
+func TestPeerGrpcServer_AnnounceAndLookup(t *testing.T) {
+	self := "self:18080"
+	client, cleanup := newTestPeerGrpc(t, nil, func(fid string) string {
+		return self
+	}, self)
+	defer cleanup()
+
+	_, err := client.ChunkAnnounce(context.Background(), &mount_peer_pb.ChunkAnnounceRequest{
+		FileIds:    []string{"3,a", "3,b"},
+		PeerAddr:   "holder:18080",
+		TtlSeconds: 60,
+	})
+	if err != nil {
+		t.Fatalf("announce: %v", err)
+	}
+
+	resp, err := client.ChunkLookup(context.Background(), &mount_peer_pb.ChunkLookupRequest{
+		FileIds: []string{"3,a", "3,missing"},
+	})
+	if err != nil {
+		t.Fatalf("lookup: %v", err)
+	}
+	if got := len(resp.PeersByFid["3,a"].Peers); got != 1 {
+		t.Errorf("3,a: expected 1 holder, got %d", got)
+	}
+	if _, ok := resp.PeersByFid["3,missing"]; !ok {
+		t.Errorf("3,missing should have a (nil/empty) peer set entry")
+	}
+}
+
+func TestPeerGrpcServer_OwnerMismatch(t *testing.T) {
+	client, cleanup := newTestPeerGrpc(t, nil, func(fid string) string {
+		return "some-other:18080"
+	}, "self:18080")
+	defer cleanup()
+
+	ann, err := client.ChunkAnnounce(context.Background(), &mount_peer_pb.ChunkAnnounceRequest{
+		FileIds:    []string{"3,x"},
+		PeerAddr:   "holder:18080",
+		TtlSeconds: 60,
+	})
+	if err != nil {
+		t.Fatalf("announce: %v", err)
+	}
+	if len(ann.RejectedFileIds) != 1 || ann.RejectedFileIds[0] != "3,x" {
+		t.Errorf("expected 3,x rejected, got %v", ann.RejectedFileIds)
+	}
+}
+
+// TestPeerGrpcServer_FetchChunk_StreamsHit exercises the new byte-transfer
+// path: a cached chunk returned as multiple gRPC frames, concatenated by
+// the caller into the original payload.
+func TestPeerGrpcServer_FetchChunk_StreamsHit(t *testing.T) {
+	cache := newFakeChunkCache()
+	// A payload deliberately larger than fetchChunkStreamSize to force
+	// multiple Send() calls.
+	payload := make([]byte, fetchChunkStreamSize*2+123)
+	for i := range payload {
+		payload[i] = byte(i & 0xff)
+	}
+	cache.Put("3,stream", payload)
+
+	client, cleanup := newTestPeerGrpc(t, cache, nil, "self:18080")
+	defer cleanup()
+
+	stream, err := client.FetchChunk(context.Background(), &mount_peer_pb.FetchChunkRequest{FileId: "3,stream"})
+	if err != nil {
+		t.Fatalf("FetchChunk: %v", err)
+	}
+	var got []byte
+	for {
+		resp, err := stream.Recv()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatalf("Recv: %v", err)
+		}
+		got = append(got, resp.Data...)
+	}
+	if len(got) != len(payload) {
+		t.Fatalf("got %d bytes, want %d", len(got), len(payload))
+	}
+	for i, b := range got {
+		if b != payload[i] {
+			t.Fatalf("byte mismatch at offset %d: got %02x want %02x", i, b, payload[i])
+		}
+	}
+}
+
+// TestPeerGrpcServer_FetchChunk_NotFound verifies that a miss returns
+// gRPC NOT_FOUND so the client can distinguish miss from transport error.
+func TestPeerGrpcServer_FetchChunk_NotFound(t *testing.T) {
+	cache := newFakeChunkCache()
+	client, cleanup := newTestPeerGrpc(t, cache, nil, "self:18080")
+	defer cleanup()
+
+	stream, err := client.FetchChunk(context.Background(), &mount_peer_pb.FetchChunkRequest{FileId: "3,missing"})
+	if err != nil {
+		t.Fatalf("FetchChunk dial: %v", err)
+	}
+	_, err = stream.Recv()
+	if err == nil {
+		t.Fatalf("expected error on missing fid, got nil")
+	}
+	if status.Code(err) != codes.NotFound {
+		t.Errorf("expected NotFound code, got %v", status.Code(err))
+	}
+}

weed/mount/peer_hrw.go

@@ -0,0 +1,51 @@
+package mount
+
+import (
+	"hash/fnv"
+)
+
+// SeedPeer is a mount server present in the filer's mount registry.
+// DataCenter and Rack ride along so downstream locality-aware decisions
+// (peer re-ranking in the fetcher) don't have to re-query the filer.
+type SeedPeer struct {
+	PeerAddr   string
+	DataCenter string
+	Rack       string
+}
+
+// OwnerFor returns the peer_addr of the mount that is the HRW-assigned
+// directory owner for fid on the given seed list.
+//
+// Rendezvous (highest-random-weight) hashing is deterministic for a given
+// (fid, seed set): every mount computes the same owner without any
+// consensus. When a mount joins or leaves the seed set, only 1/N of fids
+// change owner — the rest remain on their existing mount.
+//
+// If seeds is empty, returns "".
+//
+// See design-weed-mount-peer-chunk-sharing.md §4.2.2.
+func OwnerFor(fid string, seeds []SeedPeer) string {
+	if len(seeds) == 0 {
+		return ""
+	}
+	var bestScore uint64
+	var bestAddr string
+	for _, s := range seeds {
+		score := hrwScore(s.PeerAddr, fid)
+		if bestAddr == "" || score > bestScore || (score == bestScore && s.PeerAddr < bestAddr) {
+			bestScore = score
+			bestAddr = s.PeerAddr
+		}
+	}
+	return bestAddr
+}
+
+// hrwScore combines peer_addr and fid into a 64-bit score. Uses FNV-1a
+// (cheap and fine for balancing; not a security primitive).
+func hrwScore(peerAddr, fid string) uint64 {
+	h := fnv.New64a()
+	_, _ = h.Write([]byte(peerAddr))
+	_, _ = h.Write([]byte{0}) // separator so "ab" + "c" != "a" + "bc"
+	_, _ = h.Write([]byte(fid))
+	return h.Sum64()
+}

weed/mount/peer_hrw_test.go

@@ -0,0 +1,90 @@
+package mount
+
+import (
+	"fmt"
+	"testing"
+)
+
+func mkSeeds(addrs ...string) []SeedPeer {
+	out := make([]SeedPeer, len(addrs))
+	for i, a := range addrs {
+		out[i] = SeedPeer{PeerAddr: a}
+	}
+	return out
+}
+
+func TestOwnerFor_EmptySeeds(t *testing.T) {
+	if got := OwnerFor("3,01637037d6", nil); got != "" {
+		t.Errorf("expected empty owner for empty seed list, got %q", got)
+	}
+}
+
+func TestOwnerFor_Deterministic(t *testing.T) {
+	seeds := mkSeeds("a:1", "b:1", "c:1", "d:1")
+	fid := "3,01637037d6"
+	first := OwnerFor(fid, seeds)
+	for i := 0; i < 100; i++ {
+		if got := OwnerFor(fid, seeds); got != first {
+			t.Fatalf("non-deterministic: iter %d got %q first %q", i, got, first)
+		}
+	}
+}
+
+func TestOwnerFor_DistributesEvenly(t *testing.T) {
+	const N = 4
+	seeds := mkSeeds("a:1", "b:1", "c:1", "d:1")
+	counts := map[string]int{}
+	for i := 0; i < 10000; i++ {
+		fid := fmt.Sprintf("%d,%x", i, i*17)
+		counts[OwnerFor(fid, seeds)]++
+	}
+	// Each seed should get roughly 25% (±10% slack for a 10k sample).
+	for addr, c := range counts {
+		ratio := float64(c) / 10000.0
+		if ratio < 0.225 || ratio > 0.275 {
+			t.Errorf("HRW distribution skewed: %s got %.3f", addr, ratio)
+		}
+	}
+	if len(counts) != N {
+		t.Errorf("expected %d distinct owners, got %d", N, len(counts))
+	}
+}
+
+func TestOwnerFor_MinimalShuffleOnSeedChange(t *testing.T) {
+	// Adding one seed should move ~1/(N+1) fids to the new seed and leave
+	// the rest on their prior owners. Tolerance generous for a 10k sample.
+	const trials = 10000
+	before := mkSeeds("a:1", "b:1", "c:1")
+	after := mkSeeds("a:1", "b:1", "c:1", "d:1")
+
+	moved := 0
+	toNewSeed := 0
+	for i := 0; i < trials; i++ {
+		fid := fmt.Sprintf("%d,%x", i, i*31)
+		pre := OwnerFor(fid, before)
+		post := OwnerFor(fid, after)
+		if pre != post {
+			moved++
+			if post == "d:1" {
+				toNewSeed++
+			}
+		}
+	}
+	// Expected: ~1/4 = 25% of fids move, all to the new seed.
+	ratio := float64(moved) / float64(trials)
+	if ratio < 0.20 || ratio > 0.30 {
+		t.Errorf("expected ~25%% fids to move on seed-add, got %.3f", ratio)
+	}
+	if moved != toNewSeed {
+		t.Errorf("expected every moved fid to land on the new seed; moved=%d toNewSeed=%d", moved, toNewSeed)
+	}
+}
+
+func TestOwnerFor_TieBreakerDeterministic(t *testing.T) {
+	// Two seeds with equal hash score (engineered collision is unlikely in
+	// practice, but we guarantee determinism by lex-comparing addresses).
+	// This test just confirms OwnerFor runs without panicking on duplicates.
+	seeds := mkSeeds("a:1", "a:1", "b:1")
+	fid := "3,01637037d6"
+	_ = OwnerFor(fid, seeds)
+}

weed/mount/peer_registrar.go

@@ -0,0 +1,251 @@
+package mount
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+// PeerRegistrar maintains this mount's presence in every configured
+// filer's mount registry and its local snapshot of the merged seed set.
+// It runs two background tickers:
+//
+//   - MountRegister heartbeats, fanned out to every filer in parallel so
+//     each filer keeps a fresh TTL entry for us. Mounts pointing at
+//     different filers therefore still see each other once all filers
+//     have been heartbeated.
+//   - MountList polling, fanned out identically, merged by peer_addr
+//     (newest LastSeenNs wins) so OwnerFor() has a view of the whole
+//     fleet regardless of which filer each peer happens to heartbeat
+//     through.
+//
+// An unreachable filer is tolerated: we log and continue as long as at
+// least one filer succeeds. Entries cached on a permanently-gone filer
+// fall out of the merged view on their own TTL.
+type PeerRegistrar struct {
+	filerAddrs       []pb.ServerAddress
+	dialFiler        filerDialFn
+	selfPeerAddr     string
+	selfDc           string
+	selfRack         string
+	registerInterval time.Duration
+	registerTTL      time.Duration
+	listInterval     time.Duration
+
+	mu    sync.RWMutex
+	seeds []SeedPeer
+
+	// stopCtx cancels when Stop() is called. Background RPCs scope their
+	// deadline to this so unmount does not block on pending filer calls.
+	stopCtx    context.Context
+	stopCancel context.CancelFunc
+	stopped    atomic.Bool
+}
+
+// filerDialFn is how the registrar reaches one configured filer. The
+// production wiring is pb.WithGrpcFilerClient; tests inject a fake.
+type filerDialFn func(ctx context.Context, addr pb.ServerAddress, fn func(client filer_pb.SeaweedFilerClient) error) error
+
+// NewPeerRegistrar constructs the registrar; Start launches the background
+// loops. Callers must supply the full filer set so heartbeats and list
+// polls reach every filer — otherwise mounts talking to different filers
+// never observe each other.
+func NewPeerRegistrar(filers []pb.ServerAddress, dial filerDialFn, selfAddr, dc, rack string) *PeerRegistrar {
+	ctx, cancel := context.WithCancel(context.Background())
+	return &PeerRegistrar{
+		filerAddrs:       filers,
+		dialFiler:        dial,
+		selfPeerAddr:     selfAddr,
+		selfRack:         rack,
+		selfDc:           dc,
+		registerInterval: 30 * time.Second,
+		registerTTL:      90 * time.Second,
+		listInterval:     30 * time.Second,
+		stopCtx:          ctx,
+		stopCancel:       cancel,
+	}
+}
+
+// Start does an initial register+list synchronously (so OwnerFor has a
+// usable view immediately) and then kicks off the background loops.
+func (r *PeerRegistrar) Start(ctx context.Context) error {
+	if err := r.registerOnce(ctx); err != nil {
+		glog.V(1).Infof("initial MountRegister: %v", err)
+		// Do not fail startup — the mount must still serve reads even if
+		// no filer yet knows about us.
+	}
+	if err := r.listOnce(ctx); err != nil {
+		glog.V(1).Infof("initial MountList: %v", err)
+	}
+	go r.loopRegister()
+	go r.loopList()
+	return nil
+}
+
+// Stop halts the background loops and cancels any in-flight RPCs they
+// may have launched. Safe to call multiple times.
+func (r *PeerRegistrar) Stop() {
+	if r.stopped.Swap(true) {
+		return
+	}
+	r.stopCancel()
+}
+
+// Seeds returns the currently-known seed set. Callers MUST NOT mutate the
+// returned slice — it is shared with concurrent listOnce readers. Because
+// listOnce atomically swaps to a brand-new slice on every refresh rather
+// than mutating in place, returning the slice header directly is safe and
+// avoids a per-call allocation on the read-hot OwnerFor path.
+func (r *PeerRegistrar) Seeds() []SeedPeer {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	return r.seeds
+}
+
+// OwnerFor is a convenience wrapper that runs HRW against the current
+// seed snapshot.
+func (r *PeerRegistrar) OwnerFor(fid string) string {
+	return OwnerFor(fid, r.Seeds())
+}
+
+// filerRPCTimeout bounds a single MountRegister / MountList call so a slow
+// or partitioned filer can't wedge the background loops (which run on a
+// 30 s cadence). 20 s gives room for TLS handshakes on cold connections
+// while leaving headroom before the next tick.
+const filerRPCTimeout = 20 * time.Second
+
+// registerOnce fans a MountRegister out to every configured filer in
+// parallel. Returns an error only if every filer failed; otherwise the
+// best-effort semantics let the mount proceed when some filer is down.
+func (r *PeerRegistrar) registerOnce(ctx context.Context) error {
+	if len(r.filerAddrs) == 0 {
+		return fmt.Errorf("no filers configured")
+	}
+	ctx, cancel := context.WithTimeout(ctx, filerRPCTimeout)
+	defer cancel()
+	req := &filer_pb.MountRegisterRequest{
+		PeerAddr:   r.selfPeerAddr,
+		Rack:       r.selfRack,
+		DataCenter: r.selfDc,
+		TtlSeconds: int32(r.registerTTL / time.Second),
+	}
+	var wg sync.WaitGroup
+	var successes atomic.Int32
+	for _, addr := range r.filerAddrs {
+		wg.Add(1)
+		go func(addr pb.ServerAddress) {
+			defer wg.Done()
+			err := r.dialFiler(ctx, addr, func(c filer_pb.SeaweedFilerClient) error {
+				_, err := c.MountRegister(ctx, req)
+				return err
+			})
+			if err != nil {
+				glog.V(2).Infof("MountRegister %s: %v", addr, err)
+				return
+			}
+			successes.Add(1)
+		}(addr)
+	}
+	wg.Wait()
+	if successes.Load() == 0 {
+		return fmt.Errorf("MountRegister failed on all %d filer(s)", len(r.filerAddrs))
+	}
+	return nil
+}
+
+// listOnce polls MountList from every filer in parallel and merges the
+// responses by peer_addr (newest LastSeenNs wins). This way two mounts
+// heartbeating through different filers still end up in each other's
+// seed view as soon as at least one filer has been listed on each side.
+func (r *PeerRegistrar) listOnce(ctx context.Context) error {
+	if len(r.filerAddrs) == 0 {
+		r.mu.Lock()
+		r.seeds = nil
+		r.mu.Unlock()
+		return nil
+	}
+	ctx, cancel := context.WithTimeout(ctx, filerRPCTimeout)
+	defer cancel()
+	var (
+		mu     sync.Mutex
+		merged = map[string]*filer_pb.MountInfo{}
+		fails  int
+	)
+	var wg sync.WaitGroup
+	for _, addr := range r.filerAddrs {
+		wg.Add(1)
+		go func(addr pb.ServerAddress) {
+			defer wg.Done()
+			err := r.dialFiler(ctx, addr, func(c filer_pb.SeaweedFilerClient) error {
+				resp, err := c.MountList(ctx, &filer_pb.MountListRequest{})
+				if err != nil {
+					return err
+				}
+				mu.Lock()
+				for _, m := range resp.Mounts {
+					if prev, ok := merged[m.PeerAddr]; !ok || m.LastSeenNs > prev.LastSeenNs {
+						merged[m.PeerAddr] = m
+					}
+				}
+				mu.Unlock()
+				return nil
+			})
+			if err != nil {
+				mu.Lock()
+				fails++
+				mu.Unlock()
+				glog.V(2).Infof("MountList %s: %v", addr, err)
+			}
+		}(addr)
+	}
+	wg.Wait()
+
+	if fails == len(r.filerAddrs) {
+		return fmt.Errorf("MountList failed on all %d filer(s)", len(r.filerAddrs))
+	}
+
+	next := make([]SeedPeer, 0, len(merged))
+	for _, m := range merged {
+		next = append(next, SeedPeer{PeerAddr: m.PeerAddr, DataCenter: m.DataCenter, Rack: m.Rack})
+	}
+	r.mu.Lock()
+	r.seeds = next
+	r.mu.Unlock()
+	return nil
+}
+
+func (r *PeerRegistrar) loopRegister() {
+	t := time.NewTicker(r.registerInterval)
+	defer t.Stop()
+	for {
+		select {
+		case <-r.stopCtx.Done():
+			return
+		case <-t.C:
+			if err := r.registerOnce(r.stopCtx); err != nil {
+				glog.V(2).Infof("MountRegister heartbeat: %v", err)
+			}
+		}
+	}
+}
+
+func (r *PeerRegistrar) loopList() {
+	t := time.NewTicker(r.listInterval)
+	defer t.Stop()
+	for {
+		select {
+		case <-r.stopCtx.Done():
+			return
+		case <-t.C:
+			if err := r.listOnce(r.stopCtx); err != nil {
+				glog.V(2).Infof("MountList refresh: %v", err)
+			}
+		}
+	}
+}

weed/mount/peer_registrar_test.go

@@ -0,0 +1,210 @@
+package mount
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"google.golang.org/grpc"
+)
+
+// fakeFilerClient captures MountRegister/MountList calls and lets the test
+// pre-seed MountList responses. Implements just enough of
+// filer_pb.SeaweedFilerClient to drive the registrar.
+type fakeFilerClient struct {
+	filer_pb.SeaweedFilerClient // embed for methods we don't need
+
+	mu            sync.Mutex
+	registerCalls []filer_pb.MountRegisterRequest
+	listResponse  filer_pb.MountListResponse
+}
+
+func (f *fakeFilerClient) MountRegister(ctx context.Context, req *filer_pb.MountRegisterRequest, opts ...grpc.CallOption) (*filer_pb.MountRegisterResponse, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.registerCalls = append(f.registerCalls, *req)
+	return &filer_pb.MountRegisterResponse{}, nil
+}
+
+func (f *fakeFilerClient) MountList(ctx context.Context, req *filer_pb.MountListRequest, opts ...grpc.CallOption) (*filer_pb.MountListResponse, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	resp := f.listResponse // value copy
+	return &resp, nil
+}
+
+// fakeFilerFleet maps each addr to its own fakeFilerClient so a test can
+// simulate several filers with different registered/listed state.
+type fakeFilerFleet struct {
+	clients map[pb.ServerAddress]*fakeFilerClient
+}
+
+func (f *fakeFilerFleet) dial(ctx context.Context, addr pb.ServerAddress, fn func(client filer_pb.SeaweedFilerClient) error) error {
+	c, ok := f.clients[addr]
+	if !ok {
+		// Treat unknown filer as "reachable but empty" — lets tests omit
+		// pre-populating a client when they don't care.
+		c = &fakeFilerClient{}
+		f.clients[addr] = c
+	}
+	return fn(c)
+}
+
+func singleFilerFleet(c *fakeFilerClient) ([]pb.ServerAddress, filerDialFn) {
+	addr := pb.ServerAddress("filer-1:18888")
+	fleet := &fakeFilerFleet{clients: map[pb.ServerAddress]*fakeFilerClient{addr: c}}
+	return []pb.ServerAddress{addr}, fleet.dial
+}
+
+func TestPeerRegistrar_StartPopulatesSeedsFromFiler(t *testing.T) {
+	fc := &fakeFilerClient{
+		listResponse: filer_pb.MountListResponse{
+			Mounts: []*filer_pb.MountInfo{
+				{PeerAddr: "mount-a:18080", Rack: "r1"},
+				{PeerAddr: "mount-b:18080", Rack: "r2"},
+			},
+		},
+	}
+	filers, dial := singleFilerFleet(fc)
+
+	r := NewPeerRegistrar(filers, dial, "self:18080", "dc1", "r1")
+	if err := r.registerOnce(context.Background()); err != nil {
+		t.Fatalf("registerOnce: %v", err)
+	}
+	if err := r.listOnce(context.Background()); err != nil {
+		t.Fatalf("listOnce: %v", err)
+	}
+
+	fc.mu.Lock()
+	if len(fc.registerCalls) != 1 {
+		t.Errorf("expected 1 register call, got %d", len(fc.registerCalls))
+	} else if fc.registerCalls[0].PeerAddr != "self:18080" {
+		t.Errorf("register sent wrong peer addr: %q", fc.registerCalls[0].PeerAddr)
+	}
+	fc.mu.Unlock()
+
+	seeds := r.Seeds()
+	if len(seeds) != 2 {
+		t.Errorf("expected 2 seeds, got %d", len(seeds))
+	}
+
+	owner := r.OwnerFor("3,01637037d6")
+	if owner != "mount-a:18080" && owner != "mount-b:18080" {
+		t.Errorf("OwnerFor returned unexpected addr: %q", owner)
+	}
+}
+
+func TestPeerRegistrar_HeartbeatTTLMatchesConfig(t *testing.T) {
+	fc := &fakeFilerClient{}
+	filers, dial := singleFilerFleet(fc)
+	r := NewPeerRegistrar(filers, dial, "self:18080", "", "")
+
+	if err := r.registerOnce(context.Background()); err != nil {
+		t.Fatalf("registerOnce: %v", err)
+	}
+
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+	if len(fc.registerCalls) != 1 {
+		t.Fatalf("expected 1 register call, got %d", len(fc.registerCalls))
+	}
+	want := int32(r.registerTTL.Seconds())
+	if got := fc.registerCalls[0].TtlSeconds; got != want {
+		t.Errorf("TtlSeconds: got %d want %d", got, want)
+	}
+}
+
+func TestPeerRegistrar_StopIsIdempotent(t *testing.T) {
+	filers, dial := singleFilerFleet(&fakeFilerClient{})
+	r := NewPeerRegistrar(filers, dial, "self:18080", "", "")
+	r.Stop()
+	r.Stop() // second call must be a no-op (no panic)
+}
+
+// TestPeerRegistrar_RegisterBroadcastsToAllFilers guards the core
+// multi-filer property: a single registerOnce must hit every configured
+// filer so mounts pointing at different filers still converge.
+func TestPeerRegistrar_RegisterBroadcastsToAllFilers(t *testing.T) {
+	fc1 := &fakeFilerClient{}
+	fc2 := &fakeFilerClient{}
+	fc3 := &fakeFilerClient{}
+	a1, a2, a3 := pb.ServerAddress("f1:18888"), pb.ServerAddress("f2:18888"), pb.ServerAddress("f3:18888")
+	fleet := &fakeFilerFleet{clients: map[pb.ServerAddress]*fakeFilerClient{a1: fc1, a2: fc2, a3: fc3}}
+
+	r := NewPeerRegistrar([]pb.ServerAddress{a1, a2, a3}, fleet.dial, "self:18080", "", "")
+	if err := r.registerOnce(context.Background()); err != nil {
+		t.Fatalf("registerOnce: %v", err)
+	}
+
+	for addr, fc := range fleet.clients {
+		fc.mu.Lock()
+		if len(fc.registerCalls) != 1 {
+			t.Errorf("filer %s: got %d register calls, want 1", addr, len(fc.registerCalls))
+		}
+		fc.mu.Unlock()
+	}
+}
+
+// TestPeerRegistrar_ListMergesAcrossFilers guards cross-filer convergence:
+// mount A registered on filer-1, mount B on filer-2; a registrar that
+// lists both filers must see both mounts.
+func TestPeerRegistrar_ListMergesAcrossFilers(t *testing.T) {
+	fc1 := &fakeFilerClient{
+		listResponse: filer_pb.MountListResponse{
+			Mounts: []*filer_pb.MountInfo{{PeerAddr: "mount-a:18080", Rack: "r1", LastSeenNs: 200}},
+		},
+	}
+	fc2 := &fakeFilerClient{
+		listResponse: filer_pb.MountListResponse{
+			Mounts: []*filer_pb.MountInfo{{PeerAddr: "mount-b:18080", Rack: "r2", LastSeenNs: 200}},
+		},
+	}
+	a1, a2 := pb.ServerAddress("f1:18888"), pb.ServerAddress("f2:18888")
+	fleet := &fakeFilerFleet{clients: map[pb.ServerAddress]*fakeFilerClient{a1: fc1, a2: fc2}}
+
+	r := NewPeerRegistrar([]pb.ServerAddress{a1, a2}, fleet.dial, "self:18080", "", "")
+	if err := r.listOnce(context.Background()); err != nil {
+		t.Fatalf("listOnce: %v", err)
+	}
+
+	seeds := r.Seeds()
+	if len(seeds) != 2 {
+		t.Fatalf("want 2 merged seeds, got %d: %+v", len(seeds), seeds)
+	}
+	addrs := map[string]bool{}
+	for _, s := range seeds {
+		addrs[s.PeerAddr] = true
+	}
+	if !addrs["mount-a:18080"] || !addrs["mount-b:18080"] {
+		t.Errorf("merged seeds missing an entry: %+v", addrs)
+	}
+}
+
+// TestPeerRegistrar_ListMergeKeepsNewestLastSeen guards the dedupe rule:
+// the same mount reported by two filers collapses to one entry, keeping
+// the freshest LastSeenNs for liveness-ordering decisions.
+func TestPeerRegistrar_ListMergeKeepsNewestLastSeen(t *testing.T) {
+	fc1 := &fakeFilerClient{
+		listResponse: filer_pb.MountListResponse{
+			Mounts: []*filer_pb.MountInfo{{PeerAddr: "mount-a:18080", Rack: "r1", LastSeenNs: 100}},
+		},
+	}
+	fc2 := &fakeFilerClient{
+		listResponse: filer_pb.MountListResponse{
+			Mounts: []*filer_pb.MountInfo{{PeerAddr: "mount-a:18080", Rack: "r1", LastSeenNs: 500}},
+		},
+	}
+	a1, a2 := pb.ServerAddress("f1:18888"), pb.ServerAddress("f2:18888")
+	fleet := &fakeFilerFleet{clients: map[pb.ServerAddress]*fakeFilerClient{a1: fc1, a2: fc2}}
+
+	r := NewPeerRegistrar([]pb.ServerAddress{a1, a2}, fleet.dial, "self:18080", "", "")
+	if err := r.listOnce(context.Background()); err != nil {
+		t.Fatalf("listOnce: %v", err)
+	}
+	seeds := r.Seeds()
+	if len(seeds) != 1 {
+		t.Fatalf("want 1 deduped seed, got %d", len(seeds))
+	}
+}

weed/mount/posix_file_lock.go

@@ -0,0 +1,446 @@
+package mount
+
+import (
+	"math"
+	"sort"
+	"sync"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+)
+
+// lockRange represents a single held POSIX byte-range lock.
+type lockRange struct {
+	Start uint64 // inclusive byte offset
+	End   uint64 // inclusive; math.MaxUint64 means "to EOF"
+	Typ   uint32 // f_RDLCK or f_WRLCK
+	Owner uint64 // FUSE lock owner (from LkIn.Owner)
+	Pid   uint32 // PID of lock holder (for GetLk reporting)
+	// flock and fcntl locks have different ownership and close semantics.
+	// Keep them in separate namespaces inside the table.
+	IsFlock bool
+}
+
+// inodeLocks holds all locks for one inode plus a waiter queue for SetLkw.
+type inodeLocks struct {
+	mu       sync.Mutex
+	locks    []lockRange   // currently held locks, sorted by Start
+	waiters  []*lockWaiter // blocked SetLkw callers
+	wakeRefs int           // woken waiters still retrying on this inodeLocks
+	// dead marks an inodeLocks that has been removed from the table's map.
+	// A caller holding a pointer returned by getInodeLocks/getOrCreateInodeLocks
+	// must recheck this flag after acquiring il.mu: if true, the pointer is
+	// orphaned and must be refreshed from the table so findConflict runs
+	// against the live state.
+	dead bool
+}
+
+// lockWaiter represents a blocked SetLkw caller.
+type lockWaiter struct {
+	requested   lockRange     // the lock this waiter is trying to acquire
+	ch          chan struct{} // closed when the waiter should re-check
+	wakeRefHeld bool
+}
+
+// PosixLockTable is the per-mount POSIX lock manager.
+type PosixLockTable struct {
+	mu     sync.Mutex
+	inodes map[uint64]*inodeLocks
+}
+
+func NewPosixLockTable() *PosixLockTable {
+	return &PosixLockTable{
+		inodes: make(map[uint64]*inodeLocks),
+	}
+}
+
+// getOrCreateInodeLocks returns the lock state for an inode, creating it if
+// needed. A dead entry (which maybeCleanupInode normally evicts together
+// with setting dead) is replaced transparently so callers never observe one
+// via this path and the SetLk retry loop cannot spin on a stale map entry.
+// The il.dead read is safe without il.mu: dead transitions only under both
+// plt.mu and il.mu, so holding plt.mu here is sufficient happens-before.
+func (plt *PosixLockTable) getOrCreateInodeLocks(inode uint64) *inodeLocks {
+	plt.mu.Lock()
+	defer plt.mu.Unlock()
+	il, ok := plt.inodes[inode]
+	if !ok || il.dead {
+		il = &inodeLocks{}
+		plt.inodes[inode] = il
+	}
+	return il
+}
+
+// getInodeLocks returns the lock state for an inode, or nil if none exists.
+// A dead entry (which maybeCleanupInode normally evicts together with setting
+// dead) is dropped so callers never observe an orphaned inodeLocks via this
+// path. The il.dead read is safe without il.mu for the same reason as in
+// getOrCreateInodeLocks: transitions happen under both plt.mu and il.mu.
+func (plt *PosixLockTable) getInodeLocks(inode uint64) *inodeLocks {
+	plt.mu.Lock()
+	defer plt.mu.Unlock()
+	il, ok := plt.inodes[inode]
+	if !ok {
+		return nil
+	}
+	if il.dead {
+		delete(plt.inodes, inode)
+		return nil
+	}
+	return il
+}
+
+// maybeCleanupInode removes the inodeLocks entry if it has no locks, no waiters,
+// and no woken waiters still retrying against this inodeLocks. The removed
+// inodeLocks is marked dead so any caller that is mid-way through acquiring
+// il.mu with a stale pointer will notice and refetch from the map instead of
+// mutating an orphaned instance (which would be invisible to future callers).
+func (plt *PosixLockTable) maybeCleanupInode(inode uint64, il *inodeLocks) {
+	// Caller must NOT hold il.mu. We acquire both locks in the correct order.
+	plt.mu.Lock()
+	defer plt.mu.Unlock()
+	il.mu.Lock()
+	defer il.mu.Unlock()
+	if len(il.locks) == 0 && len(il.waiters) == 0 && il.wakeRefs == 0 {
+		if plt.inodes[inode] == il {
+			delete(plt.inodes, inode)
+		}
+		il.dead = true
+	}
+}
+
+// rangesOverlap returns true if two inclusive ranges overlap.
+func rangesOverlap(aStart, aEnd, bStart, bEnd uint64) bool {
+	return aStart <= bEnd && bStart <= aEnd
+}
+
+// findConflict returns the first lock that conflicts with the proposed lock.
+// A conflict exists when ranges overlap, at least one is a write lock, and the owners differ.
+func findConflict(locks []lockRange, proposed lockRange) (lockRange, bool) {
+	for _, h := range locks {
+		if h.IsFlock != proposed.IsFlock {
+			continue
+		}
+		if h.Owner == proposed.Owner {
+			continue
+		}
+		if !rangesOverlap(h.Start, h.End, proposed.Start, proposed.End) {
+			continue
+		}
+		if h.Typ == f_RDLCK && proposed.Typ == f_RDLCK {
+			continue
+		}
+		return h, true
+	}
+	return lockRange{}, false
+}
+
+// insertAndCoalesce inserts a lock for the given owner, replacing/splitting any
+// existing same-owner locks that overlap. Adjacent same-type locks are merged.
+// Caller must hold il.mu.
+func insertAndCoalesce(il *inodeLocks, lk lockRange) {
+	owner := lk.Owner
+
+	var kept []lockRange
+	for _, h := range il.locks {
+		if h.Owner != owner || h.IsFlock != lk.IsFlock {
+			kept = append(kept, h)
+			continue
+		}
+		if !rangesOverlap(h.Start, h.End, lk.Start, lk.End) {
+			// Check for adjacency with same type for merging.
+			if h.Typ == lk.Typ && ((h.End < ^uint64(0) && h.End+1 == lk.Start) || (lk.End < ^uint64(0) && lk.End+1 == h.Start)) {
+				// Merge adjacent same-type lock into lk.
+				if h.Start < lk.Start {
+					lk.Start = h.Start
+				}
+				if h.End > lk.End {
+					lk.End = h.End
+				}
+				continue
+			}
+			kept = append(kept, h)
+			continue
+		}
+		// Overlapping same-owner lock.
+		if h.Typ == lk.Typ {
+			// Same type: absorb into lk (expand range).
+			if h.Start < lk.Start {
+				lk.Start = h.Start
+			}
+			if h.End > lk.End {
+				lk.End = h.End
+			}
+			continue
+		}
+		// Different type: truncate or split the existing lock.
+		if h.Start < lk.Start {
+			// Left portion survives.
+			left := h
+			left.End = lk.Start - 1
+			kept = append(kept, left)
+		}
+		if h.End > lk.End {
+			// Right portion survives.
+			right := h
+			right.Start = lk.End + 1
+			kept = append(kept, right)
+		}
+	}
+
+	kept = append(kept, lk)
+	sort.Slice(kept, func(i, j int) bool {
+		return kept[i].Start < kept[j].Start
+	})
+	il.locks = kept
+}
+
+// removeLocks removes or splits matching locks in the given range.
+// Caller must hold il.mu.
+func removeLocks(il *inodeLocks, matches func(lockRange) bool, start, end uint64) {
+	var kept []lockRange
+	for _, h := range il.locks {
+		if !matches(h) || !rangesOverlap(h.Start, h.End, start, end) {
+			kept = append(kept, h)
+			continue
+		}
+		// h overlaps the unlock range.
+		if h.Start < start {
+			// Left portion survives.
+			left := h
+			left.End = start - 1
+			kept = append(kept, left)
+		}
+		if h.End > end {
+			// Right portion survives.
+			right := h
+			right.Start = end + 1
+			kept = append(kept, right)
+		}
+		// If fully contained, it's simply dropped.
+	}
+	il.locks = kept
+}
+
+func (plt *PosixLockTable) releaseMatching(inode uint64, matches func(lockRange) bool) {
+	il := plt.getInodeLocks(inode)
+	if il == nil {
+		return
+	}
+	il.mu.Lock()
+	removeLocks(il, matches, 0, math.MaxUint64)
+	wakeEligibleWaiters(il)
+	il.mu.Unlock()
+	plt.maybeCleanupInode(inode, il)
+}
+
+// HasPosixOwner reports whether owner currently holds any POSIX byte-range
+// locks on inode. FUSE may provide a non-zero FlushIn.LockOwner even when no
+// locks were taken, so callers should consult the lock table before treating a
+// flush as lock-sensitive.
+func (plt *PosixLockTable) HasPosixOwner(inode uint64, owner uint64) bool {
+	if owner == 0 {
+		return false
+	}
+	il := plt.getInodeLocks(inode)
+	if il == nil {
+		return false
+	}
+	il.mu.Lock()
+	defer il.mu.Unlock()
+	if il.dead {
+		return false
+	}
+	for _, lk := range il.locks {
+		if !lk.IsFlock && lk.Owner == owner {
+			return true
+		}
+	}
+	return false
+}
+
+// releaseWakeRef drops the temporary reference that keeps inodeLocks live while
+// a woken waiter retries its SetLkw acquisition.
+func releaseWakeRef(il *inodeLocks, waiter *lockWaiter) {
+	if waiter == nil || !waiter.wakeRefHeld {
+		return
+	}
+	waiter.wakeRefHeld = false
+	il.wakeRefs--
+}
+
+// wakeEligibleWaiters selectively wakes blocked SetLkw callers that can now
+// succeed given the current lock state. Waiters whose requests still conflict
+// with held locks remain in the queue, avoiding a thundering herd.
+// Caller must hold il.mu.
+func wakeEligibleWaiters(il *inodeLocks) {
+	remaining := il.waiters[:0]
+	for _, w := range il.waiters {
+		if _, conflicted := findConflict(il.locks, w.requested); !conflicted {
+			w.wakeRefHeld = true
+			il.wakeRefs++
+			close(w.ch)
+		} else {
+			remaining = append(remaining, w)
+		}
+	}
+	il.waiters = remaining
+}
+
+// removeWaiter removes a specific waiter from the list.
+// Caller must hold il.mu.
+func removeWaiter(il *inodeLocks, w *lockWaiter) {
+	for i, existing := range il.waiters {
+		if existing == w {
+			il.waiters = append(il.waiters[:i], il.waiters[i+1:]...)
+			return
+		}
+	}
+}
+
+// GetLk checks for a conflicting lock. If found, it populates out with the
+// conflict details. If no conflict, out.Typ is set to F_UNLCK.
+func (plt *PosixLockTable) GetLk(inode uint64, proposed lockRange, out *fuse.LkOut) {
+	for {
+		il := plt.getInodeLocks(inode)
+		if il == nil {
+			out.Lk.Typ = f_UNLCK
+			return
+		}
+		il.mu.Lock()
+		if il.dead {
+			// Orphaned by a concurrent cleanup: the map may already hold a
+			// fresh inodeLocks with different state, so answering from this
+			// stale pointer would miss real conflicts. Refetch.
+			il.mu.Unlock()
+			continue
+		}
+		conflict, found := findConflict(il.locks, proposed)
+		il.mu.Unlock()
+
+		if found {
+			out.Lk.Start = conflict.Start
+			out.Lk.End = conflict.End
+			out.Lk.Typ = conflict.Typ
+			out.Lk.Pid = conflict.Pid
+		} else {
+			out.Lk.Typ = f_UNLCK
+		}
+		return
+	}
+}
+
+// SetLk attempts a non-blocking lock or unlock.
+// For unlock (F_UNLCK): removes locks in the given range for the owner.
+// For lock: returns fuse.EAGAIN if a conflict exists, fuse.OK on success.
+func (plt *PosixLockTable) SetLk(inode uint64, lk lockRange) fuse.Status {
+	if lk.Typ == f_UNLCK {
+		il := plt.getInodeLocks(inode)
+		if il == nil {
+			return fuse.OK
+		}
+		il.mu.Lock()
+		if il.dead {
+			// Orphaned by a concurrent cleanup: any lock we may have held
+			// is gone with it, and there is no point waking waiters on a
+			// detached inodeLocks.
+			il.mu.Unlock()
+			return fuse.OK
+		}
+		removeLocks(il, func(existing lockRange) bool {
+			return existing.Owner == lk.Owner && existing.IsFlock == lk.IsFlock
+		}, lk.Start, lk.End)
+		wakeEligibleWaiters(il)
+		il.mu.Unlock()
+		plt.maybeCleanupInode(inode, il)
+		return fuse.OK
+	}
+
+	for {
+		il := plt.getOrCreateInodeLocks(inode)
+		il.mu.Lock()
+		if il.dead {
+			il.mu.Unlock()
+			continue
+		}
+		if _, found := findConflict(il.locks, lk); found {
+			il.mu.Unlock()
+			return fuse.EAGAIN
+		}
+		insertAndCoalesce(il, lk)
+		il.mu.Unlock()
+		return fuse.OK
+	}
+}
+
+// SetLkw attempts a blocking lock. It waits until the lock can be acquired
+// or the cancel channel is closed.
+func (plt *PosixLockTable) SetLkw(inode uint64, lk lockRange, cancel <-chan struct{}) fuse.Status {
+	if lk.Typ == f_UNLCK {
+		return plt.SetLk(inode, lk)
+	}
+
+	il := plt.getOrCreateInodeLocks(inode)
+	var waiter *lockWaiter
+	for {
+		il.mu.Lock()
+		if il.dead {
+			// The inodeLocks we were holding was orphaned by a concurrent
+			// cleanup. Refetch the live instance from the table and retry.
+			// A dead il never has waiters or wakeRefs (the cleanup condition
+			// requires both to be zero), so waiter is guaranteed nil here.
+			il.mu.Unlock()
+			il = plt.getOrCreateInodeLocks(inode)
+			continue
+		}
+		releaseWakeRef(il, waiter)
+		if _, found := findConflict(il.locks, lk); !found {
+			insertAndCoalesce(il, lk)
+			il.mu.Unlock()
+			return fuse.OK
+		}
+		// Register waiter with the requested lock details for selective waking.
+		waiter = &lockWaiter{requested: lk, ch: make(chan struct{})}
+		il.waiters = append(il.waiters, waiter)
+		il.mu.Unlock()
+
+		// Block until woken or cancelled.
+		select {
+		case <-waiter.ch:
+			// Woken — retry.
+			continue
+		case <-cancel:
+			// Request cancelled.
+			il.mu.Lock()
+			releaseWakeRef(il, waiter)
+			removeWaiter(il, waiter)
+			il.mu.Unlock()
+			plt.maybeCleanupInode(inode, il)
+			return fuse.EINTR
+		}
+	}
+}
+
+// ReleaseOwner removes all locks held by the given owner on the given inode.
+// Used for same-owner cleanup in tests and lock-table operations.
+func (plt *PosixLockTable) ReleaseOwner(inode uint64, owner uint64) {
+	plt.releaseMatching(inode, func(lk lockRange) bool {
+		return lk.Owner == owner
+	})
+}
+
+// ReleaseFlockOwner removes flock locks for a released file description.
+// FUSE only provides LockOwner on RELEASE when FUSE_RELEASE_FLOCK_UNLOCK is set.
+func (plt *PosixLockTable) ReleaseFlockOwner(inode uint64, owner uint64) {
+	plt.releaseMatching(inode, func(lk lockRange) bool {
+		return lk.IsFlock && lk.Owner == owner
+	})
+}
+
+// ReleasePosixOwner removes POSIX fcntl locks for a closing lock owner.
+// FUSE passes the closing fi->owner on FLUSH, which is the correct close-time
+// identity for POSIX byte-range lock cleanup.
+func (plt *PosixLockTable) ReleasePosixOwner(inode uint64, owner uint64) {
+	plt.releaseMatching(inode, func(lk lockRange) bool {
+		return !lk.IsFlock && lk.Owner == owner
+	})
+}

weed/mount/posix_file_lock_test.go

@@ -0,0 +1,852 @@
+//go:build !windows
+
+package mount
+
+import (
+	"math"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+)
+
+func TestNonOverlappingLocksFromDifferentOwners(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	s1 := plt.SetLk(inode, lockRange{Start: 0, End: 49, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	if s1 != fuse.OK {
+		t.Fatalf("expected OK, got %v", s1)
+	}
+	s2 := plt.SetLk(inode, lockRange{Start: 50, End: 99, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20})
+	if s2 != fuse.OK {
+		t.Fatalf("expected OK, got %v", s2)
+	}
+}
+
+func TestOverlappingReadLocksFromDifferentOwners(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	s1 := plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_RDLCK, Owner: 1, Pid: 10})
+	if s1 != fuse.OK {
+		t.Fatalf("expected OK, got %v", s1)
+	}
+	s2 := plt.SetLk(inode, lockRange{Start: 50, End: 149, Typ: syscall.F_RDLCK, Owner: 2, Pid: 20})
+	if s2 != fuse.OK {
+		t.Fatalf("expected OK, got %v", s2)
+	}
+}
+
+func TestOverlappingWriteReadConflict(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	s := plt.SetLk(inode, lockRange{Start: 50, End: 149, Typ: syscall.F_RDLCK, Owner: 2, Pid: 20})
+	if s != fuse.EAGAIN {
+		t.Fatalf("expected EAGAIN, got %v", s)
+	}
+}
+
+func TestOverlappingWriteWriteConflict(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	s := plt.SetLk(inode, lockRange{Start: 50, End: 149, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20})
+	if s != fuse.EAGAIN {
+		t.Fatalf("expected EAGAIN, got %v", s)
+	}
+}
+
+func TestSameOwnerUpgradeReadToWrite(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_RDLCK, Owner: 1, Pid: 10})
+	s := plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	if s != fuse.OK {
+		t.Fatalf("expected OK for same-owner upgrade, got %v", s)
+	}
+
+	// Verify the lock is now a write lock.
+	var out fuse.LkOut
+	plt.GetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20}, &out)
+	if out.Lk.Typ != syscall.F_WRLCK {
+		t.Fatalf("expected conflicting write lock, got type %d", out.Lk.Typ)
+	}
+}
+
+func TestSameOwnerDowngradeWriteToRead(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	s := plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_RDLCK, Owner: 1, Pid: 10})
+	if s != fuse.OK {
+		t.Fatalf("expected OK for same-owner downgrade, got %v", s)
+	}
+
+	// Another owner should now be able to get a read lock.
+	s2 := plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_RDLCK, Owner: 2, Pid: 20})
+	if s2 != fuse.OK {
+		t.Fatalf("expected OK for shared read lock, got %v", s2)
+	}
+}
+
+func TestLockCoalescing(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 0, End: 9, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	plt.SetLk(inode, lockRange{Start: 10, End: 19, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+
+	il := plt.getInodeLocks(inode)
+	il.mu.Lock()
+	ownerLocks := 0
+	for _, lk := range il.locks {
+		if lk.Owner == 1 {
+			ownerLocks++
+			if lk.Start != 0 || lk.End != 19 {
+				t.Errorf("expected coalesced lock [0,19], got [%d,%d]", lk.Start, lk.End)
+			}
+		}
+	}
+	il.mu.Unlock()
+	if ownerLocks != 1 {
+		t.Fatalf("expected 1 coalesced lock, got %d", ownerLocks)
+	}
+}
+
+func TestLockSplitting(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	// Unlock the middle portion.
+	plt.SetLk(inode, lockRange{Start: 40, End: 59, Typ: syscall.F_UNLCK, Owner: 1, Pid: 10})
+
+	il := plt.getInodeLocks(inode)
+	il.mu.Lock()
+	ownerLocks := 0
+	for _, lk := range il.locks {
+		if lk.Owner == 1 {
+			ownerLocks++
+		}
+	}
+	if ownerLocks != 2 {
+		il.mu.Unlock()
+		t.Fatalf("expected 2 locks after split, got %d", ownerLocks)
+	}
+	// Check the ranges.
+	if il.locks[0].Start != 0 || il.locks[0].End != 39 {
+		t.Errorf("expected left lock [0,39], got [%d,%d]", il.locks[0].Start, il.locks[0].End)
+	}
+	if il.locks[1].Start != 60 || il.locks[1].End != 99 {
+		t.Errorf("expected right lock [60,99], got [%d,%d]", il.locks[1].Start, il.locks[1].End)
+	}
+	il.mu.Unlock()
+}
+
+func TestGetLkConflict(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 10, End: 50, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+
+	var out fuse.LkOut
+	plt.GetLk(inode, lockRange{Start: 30, End: 70, Typ: syscall.F_RDLCK, Owner: 2, Pid: 20}, &out)
+	if out.Lk.Typ != syscall.F_WRLCK {
+		t.Fatalf("expected conflicting write lock, got type %d", out.Lk.Typ)
+	}
+	if out.Lk.Pid != 10 {
+		t.Fatalf("expected holder PID 10, got %d", out.Lk.Pid)
+	}
+	if out.Lk.Start != 10 || out.Lk.End != 50 {
+		t.Fatalf("expected conflict [10,50], got [%d,%d]", out.Lk.Start, out.Lk.End)
+	}
+}
+
+func TestGetLkNoConflict(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 10, End: 50, Typ: syscall.F_RDLCK, Owner: 1, Pid: 10})
+
+	var out fuse.LkOut
+	plt.GetLk(inode, lockRange{Start: 30, End: 70, Typ: syscall.F_RDLCK, Owner: 2, Pid: 20}, &out)
+	if out.Lk.Typ != syscall.F_UNLCK {
+		t.Fatalf("expected F_UNLCK (no conflict), got type %d", out.Lk.Typ)
+	}
+}
+
+func TestGetLkSameOwnerNoConflict(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+
+	var out fuse.LkOut
+	plt.GetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10}, &out)
+	if out.Lk.Typ != syscall.F_UNLCK {
+		t.Fatalf("same owner should not conflict with itself, got type %d", out.Lk.Typ)
+	}
+}
+
+func TestReleaseOwner(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 0, End: 49, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	plt.SetLk(inode, lockRange{Start: 50, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	plt.SetLk(inode, lockRange{Start: 200, End: 299, Typ: syscall.F_RDLCK, Owner: 2, Pid: 20})
+
+	plt.ReleaseOwner(inode, 1)
+
+	// Owner 1's locks should be gone.
+	var out fuse.LkOut
+	plt.GetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 3, Pid: 30}, &out)
+	if out.Lk.Typ != syscall.F_UNLCK {
+		t.Fatalf("expected no conflict after ReleaseOwner, got type %d", out.Lk.Typ)
+	}
+
+	// Owner 2's lock should still exist.
+	plt.GetLk(inode, lockRange{Start: 200, End: 299, Typ: syscall.F_WRLCK, Owner: 3, Pid: 30}, &out)
+	if out.Lk.Typ != syscall.F_RDLCK {
+		t.Fatalf("expected owner 2's read lock to remain, got type %d", out.Lk.Typ)
+	}
+}
+
+func TestDifferentLockKindsDoNotConflict(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	s1 := plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	if s1 != fuse.OK {
+		t.Fatalf("expected POSIX lock OK, got %v", s1)
+	}
+
+	s2 := plt.SetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20, IsFlock: true})
+	if s2 != fuse.OK {
+		t.Fatalf("expected flock lock OK in separate namespace, got %v", s2)
+	}
+}
+
+func TestReleasePosixOwnerReleasesPosixLocksAndWakesWaiters(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+
+	done := make(chan fuse.Status, 1)
+	go func() {
+		cancel := make(chan struct{})
+		done <- plt.SetLkw(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20}, cancel)
+	}()
+
+	time.Sleep(50 * time.Millisecond)
+	plt.ReleasePosixOwner(inode, 1)
+
+	select {
+	case s := <-done:
+		if s != fuse.OK {
+			t.Fatalf("expected OK after ReleasePosixOwner, got %v", s)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("SetLkw did not unblock after ReleasePosixOwner")
+	}
+}
+
+func TestReleasePosixOwnerDoesNotReleaseFlockLocks(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10, IsFlock: true})
+	plt.ReleasePosixOwner(inode, 1)
+
+	var out fuse.LkOut
+	plt.GetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20, IsFlock: true}, &out)
+	if out.Lk.Typ != syscall.F_WRLCK {
+		t.Fatalf("expected flock lock to remain after ReleasePosixOwner, got type %d", out.Lk.Typ)
+	}
+}
+
+func TestHasPosixOwnerIgnoresMissingOwnerAndFlock(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	if plt.HasPosixOwner(inode, 1) {
+		t.Fatal("missing owner should not be reported as holding POSIX locks")
+	}
+
+	if s := plt.SetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10, IsFlock: true}); s != fuse.OK {
+		t.Fatalf("set flock: %v", s)
+	}
+	if plt.HasPosixOwner(inode, 1) {
+		t.Fatal("flock owner should not be reported as a POSIX lock owner")
+	}
+
+	if s := plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20}); s != fuse.OK {
+		t.Fatalf("set POSIX lock: %v", s)
+	}
+	if !plt.HasPosixOwner(inode, 2) {
+		t.Fatal("POSIX lock owner was not reported")
+	}
+	if plt.HasPosixOwner(inode, 0) {
+		t.Fatal("zero owner should not be reported")
+	}
+}
+
+func TestWakeEligibleWaitersKeepsInodeUntilWakeRefReleased(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+	il := plt.getOrCreateInodeLocks(inode)
+	waiter := &lockWaiter{
+		requested: lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20},
+		ch:        make(chan struct{}),
+	}
+
+	il.mu.Lock()
+	il.waiters = append(il.waiters, waiter)
+	il.mu.Unlock()
+
+	plt.releaseMatching(inode, func(lockRange) bool { return false })
+
+	select {
+	case <-waiter.ch:
+		// Expected.
+	default:
+		t.Fatal("expected waiter to be woken")
+	}
+
+	plt.mu.Lock()
+	_, exists := plt.inodes[inode]
+	plt.mu.Unlock()
+	if !exists {
+		t.Fatal("inodeLocks should remain while a woken waiter still holds a wake ref")
+	}
+
+	il.mu.Lock()
+	releaseWakeRef(il, waiter)
+	il.mu.Unlock()
+	plt.maybeCleanupInode(inode, il)
+
+	plt.mu.Lock()
+	_, exists = plt.inodes[inode]
+	plt.mu.Unlock()
+	if exists {
+		t.Fatal("inodeLocks should be cleaned up after the final wake ref is released")
+	}
+}
+
+func TestReleaseFlockOwnerDoesNotReleasePosixLocks(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	plt.SetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 2, Pid: 10, IsFlock: true})
+
+	plt.ReleaseFlockOwner(inode, 2)
+
+	var out fuse.LkOut
+	plt.GetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 3, Pid: 30}, &out)
+	if out.Lk.Typ != syscall.F_WRLCK {
+		t.Fatalf("expected POSIX lock to remain after ReleaseFlockOwner, got type %d", out.Lk.Typ)
+	}
+
+	plt.GetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 4, Pid: 40, IsFlock: true}, &out)
+	if out.Lk.Typ != syscall.F_UNLCK {
+		t.Fatalf("expected flock lock to be removed after ReleaseFlockOwner, got type %d", out.Lk.Typ)
+	}
+}
+
+func TestReleaseOwnerWakesWaiters(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+
+	done := make(chan fuse.Status, 1)
+	go func() {
+		cancel := make(chan struct{})
+		s := plt.SetLkw(inode, lockRange{Start: 50, End: 60, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20}, cancel)
+		done <- s
+	}()
+
+	// Give the goroutine time to block.
+	time.Sleep(50 * time.Millisecond)
+
+	plt.ReleaseOwner(inode, 1)
+
+	select {
+	case s := <-done:
+		if s != fuse.OK {
+			t.Fatalf("expected OK after ReleaseOwner woke waiter, got %v", s)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("SetLkw did not unblock after ReleaseOwner")
+	}
+}
+
+func TestSetLkwBlocksAndSucceeds(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+
+	done := make(chan fuse.Status, 1)
+	go func() {
+		cancel := make(chan struct{})
+		s := plt.SetLkw(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20}, cancel)
+		done <- s
+	}()
+
+	// Give the goroutine time to block.
+	time.Sleep(50 * time.Millisecond)
+
+	// Release the conflicting lock.
+	plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_UNLCK, Owner: 1, Pid: 10})
+
+	select {
+	case s := <-done:
+		if s != fuse.OK {
+			t.Fatalf("expected OK, got %v", s)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("SetLkw did not unblock after conflicting lock was released")
+	}
+}
+
+func TestSetLkwCancellation(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+
+	cancel := make(chan struct{})
+	done := make(chan fuse.Status, 1)
+	go func() {
+		s := plt.SetLkw(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20}, cancel)
+		done <- s
+	}()
+
+	// Give the goroutine time to block.
+	time.Sleep(50 * time.Millisecond)
+
+	close(cancel)
+
+	select {
+	case s := <-done:
+		if s != fuse.EINTR {
+			t.Fatalf("expected EINTR on cancel, got %v", s)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("SetLkw did not unblock after cancel")
+	}
+}
+
+func TestWholeFileLock(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	// Simulate flock() — whole-file exclusive lock.
+	s1 := plt.SetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	if s1 != fuse.OK {
+		t.Fatalf("expected OK, got %v", s1)
+	}
+
+	// Second owner should be blocked.
+	s2 := plt.SetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20})
+	if s2 != fuse.EAGAIN {
+		t.Fatalf("expected EAGAIN, got %v", s2)
+	}
+
+	// Even a partial overlap should fail.
+	s3 := plt.SetLk(inode, lockRange{Start: 100, End: 200, Typ: syscall.F_RDLCK, Owner: 2, Pid: 20})
+	if s3 != fuse.EAGAIN {
+		t.Fatalf("expected EAGAIN for partial overlap with whole-file lock, got %v", s3)
+	}
+}
+
+func TestUnlockNoExistingLocks(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	// Unlock on an inode with no locks should succeed silently.
+	s := plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_UNLCK, Owner: 1, Pid: 10})
+	if s != fuse.OK {
+		t.Fatalf("expected OK for unlock with no existing locks, got %v", s)
+	}
+}
+
+func TestMultipleInodesIndependent(t *testing.T) {
+	plt := NewPosixLockTable()
+
+	// Write lock on inode 1 should not affect inode 2.
+	plt.SetLk(1, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	s := plt.SetLk(2, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20})
+	if s != fuse.OK {
+		t.Fatalf("locks on different inodes should be independent, got %v", s)
+	}
+}
+
+func TestMemoryCleanup(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	plt.ReleaseOwner(inode, 1)
+
+	plt.mu.Lock()
+	_, exists := plt.inodes[inode]
+	plt.mu.Unlock()
+	if exists {
+		t.Fatal("expected inode entry to be cleaned up after all locks released")
+	}
+}
+
+func TestSelectiveWaking(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	// Owner 1 holds write lock on [0, 99], owner 2 holds write lock on [200, 299].
+	plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	plt.SetLk(inode, lockRange{Start: 200, End: 299, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20})
+
+	// Owner 3 waits for [50, 60] (blocked by owner 1).
+	done3 := make(chan fuse.Status, 1)
+	go func() {
+		cancel := make(chan struct{})
+		s := plt.SetLkw(inode, lockRange{Start: 50, End: 60, Typ: syscall.F_WRLCK, Owner: 3, Pid: 30}, cancel)
+		done3 <- s
+	}()
+	// Owner 4 waits for [250, 260] (blocked by owner 2).
+	done4 := make(chan fuse.Status, 1)
+	go func() {
+		cancel := make(chan struct{})
+		s := plt.SetLkw(inode, lockRange{Start: 250, End: 260, Typ: syscall.F_WRLCK, Owner: 4, Pid: 40}, cancel)
+		done4 <- s
+	}()
+
+	time.Sleep(50 * time.Millisecond)
+
+	// Release owner 1's lock. Only owner 3 should be woken; owner 4 is still blocked.
+	plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_UNLCK, Owner: 1, Pid: 10})
+
+	select {
+	case s := <-done3:
+		if s != fuse.OK {
+			t.Fatalf("expected OK for owner 3, got %v", s)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("owner 3 was not woken after owner 1 released")
+	}
+
+	// Owner 4 should still be blocked.
+	select {
+	case s := <-done4:
+		t.Fatalf("owner 4 should still be blocked, but got %v", s)
+	case <-time.After(100 * time.Millisecond):
+		// Expected — still blocked.
+	}
+
+	// Now release owner 2's lock. Owner 4 should wake.
+	plt.SetLk(inode, lockRange{Start: 200, End: 299, Typ: syscall.F_UNLCK, Owner: 2, Pid: 20})
+
+	select {
+	case s := <-done4:
+		if s != fuse.OK {
+			t.Fatalf("expected OK for owner 4, got %v", s)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("owner 4 was not woken after owner 2 released")
+	}
+}
+
+func TestSameOwnerReplaceDifferentType(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	// Lock [0, 99] as write.
+	plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	// Replace middle portion [30, 60] with read lock.
+	plt.SetLk(inode, lockRange{Start: 30, End: 60, Typ: syscall.F_RDLCK, Owner: 1, Pid: 10})
+
+	il := plt.getInodeLocks(inode)
+	il.mu.Lock()
+	defer il.mu.Unlock()
+
+	// Should have 3 locks: write [0,29], read [30,60], write [61,99].
+	if len(il.locks) != 3 {
+		t.Fatalf("expected 3 locks after partial type change, got %d", len(il.locks))
+	}
+	if il.locks[0].Typ != syscall.F_WRLCK || il.locks[0].Start != 0 || il.locks[0].End != 29 {
+		t.Errorf("expected write [0,29], got type=%d [%d,%d]", il.locks[0].Typ, il.locks[0].Start, il.locks[0].End)
+	}
+	if il.locks[1].Typ != syscall.F_RDLCK || il.locks[1].Start != 30 || il.locks[1].End != 60 {
+		t.Errorf("expected read [30,60], got type=%d [%d,%d]", il.locks[1].Typ, il.locks[1].Start, il.locks[1].End)
+	}
+	if il.locks[2].Typ != syscall.F_WRLCK || il.locks[2].Start != 61 || il.locks[2].End != 99 {
+		t.Errorf("expected write [61,99], got type=%d [%d,%d]", il.locks[2].Typ, il.locks[2].Start, il.locks[2].End)
+	}
+}
+
+func TestNonAdjacentRangesNotCoalesced(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	// Lock [5, MaxUint64] then [0, 2] — gap at [3,4] must prevent coalescing.
+	plt.SetLk(inode, lockRange{Start: 5, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	s := plt.SetLk(inode, lockRange{Start: 0, End: 2, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	if s != fuse.OK {
+		t.Fatalf("expected OK, got %v", s)
+	}
+
+	il := plt.getInodeLocks(inode)
+	il.mu.Lock()
+	defer il.mu.Unlock()
+
+	if len(il.locks) != 2 {
+		t.Fatalf("expected 2 separate locks (gap [3,4] prevents coalescing), got %d", len(il.locks))
+	}
+	if il.locks[0].Start != 0 || il.locks[0].End != 2 {
+		t.Errorf("expected first lock [0,2], got [%d,%d]", il.locks[0].Start, il.locks[0].End)
+	}
+	if il.locks[1].Start != 5 || il.locks[1].End != math.MaxUint64 {
+		t.Errorf("expected second lock [5,MaxUint64], got [%d,%d]", il.locks[1].Start, il.locks[1].End)
+	}
+}
+
+func TestAdjacencyNoOverflowAtMaxUint64(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(1)
+
+	// Lock to EOF (End = MaxUint64), then lock [0, 0] same type.
+	// Without the overflow guard, MaxUint64+1 wraps to 0, falsely merging.
+	plt.SetLk(inode, lockRange{Start: 100, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+	plt.SetLk(inode, lockRange{Start: 0, End: 0, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
+
+	il := plt.getInodeLocks(inode)
+	il.mu.Lock()
+	defer il.mu.Unlock()
+
+	// Should remain 2 separate locks, not merged.
+	ownerLocks := 0
+	for _, lk := range il.locks {
+		if lk.Owner == 1 {
+			ownerLocks++
+		}
+	}
+	if ownerLocks != 2 {
+		t.Fatalf("expected 2 separate locks (no overflow merge), got %d", ownerLocks)
+	}
+}
+
+// TestSetLkRetriesPastDeadInodeLocks deterministically exercises the
+// getOrCreateInodeLocks vs maybeCleanupInode race: a caller holding a
+// pointer to an inodeLocks that is concurrently marked dead must refetch
+// from the map instead of mutating the orphaned instance (which would be
+// invisible to subsequent callers and let two exclusive flock holders
+// coexist). The test bypasses scheduling by hand-installing a dead il into
+// the table and asserting that the next SetLk routes around it.
+func TestSetLkRetriesPastDeadInodeLocks(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(42)
+
+	// Acquire and release a lock so maybeCleanupInode marks the il dead and
+	// removes it from the map.
+	lock := lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 1, IsFlock: true}
+	if s := plt.SetLk(inode, lock); s != fuse.OK {
+		t.Fatalf("prime SetLk: got %v", s)
+	}
+	dead := plt.getInodeLocks(inode)
+	unlock := lock
+	unlock.Typ = syscall.F_UNLCK
+	if s := plt.SetLk(inode, unlock); s != fuse.OK {
+		t.Fatalf("prime unlock: got %v", s)
+	}
+	if !dead.dead {
+		t.Fatal("expected il to be marked dead after unlock+cleanup")
+	}
+	plt.mu.Lock()
+	_, stillMapped := plt.inodes[inode]
+	plt.mu.Unlock()
+	if stillMapped {
+		t.Fatal("expected the dead il to be removed from the map")
+	}
+
+	// Simulate the race: the next caller's getOrCreateInodeLocks races with
+	// the cleanup and ends up holding a pointer to the dead il. We force that
+	// state by re-publishing `dead` into the map.
+	plt.mu.Lock()
+	plt.inodes[inode] = dead
+	plt.mu.Unlock()
+
+	// SetLk must notice dead, refetch, and install the new lock in a fresh il.
+	if s := plt.SetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 2, IsFlock: true}); s != fuse.OK {
+		t.Fatalf("SetLk after dead: got %v", s)
+	}
+
+	dead.mu.Lock()
+	if n := len(dead.locks); n != 0 {
+		t.Fatalf("dead il should not have accepted the insert, found %d locks", n)
+	}
+	dead.mu.Unlock()
+
+	plt.mu.Lock()
+	live := plt.inodes[inode]
+	plt.mu.Unlock()
+	if live == nil || live == dead {
+		t.Fatalf("expected a fresh live il, got %v", live)
+	}
+
+	// A conflicting owner must see the new lock and be rejected.
+	if s := plt.SetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 3, IsFlock: true}); s != fuse.EAGAIN {
+		t.Fatalf("second owner should conflict with owner 2, got %v", s)
+	}
+
+	// GetLk must report the conflict as well: without the dead-recheck the
+	// GetLk path would answer F_UNLCK off the orphaned il.
+	var out fuse.LkOut
+	plt.GetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 4, IsFlock: true}, &out)
+	if out.Lk.Typ != syscall.F_WRLCK {
+		t.Fatalf("GetLk should report the live conflict, got Typ=%d", out.Lk.Typ)
+	}
+}
+
+// TestGetInodeLocksEvictsDeadEntry verifies that a dead inodeLocks which
+// somehow ends up in the map (e.g. through a future refactor that reorders
+// delete and dead=true) is dropped on read so callers never observe one.
+// This is the backstop that lets GetLk's and SetLk's retry loops terminate.
+func TestGetInodeLocksEvictsDeadEntry(t *testing.T) {
+	plt := NewPosixLockTable()
+	inode := uint64(42)
+
+	lock := lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 1, IsFlock: true}
+	if s := plt.SetLk(inode, lock); s != fuse.OK {
+		t.Fatalf("prime SetLk: got %v", s)
+	}
+	dead := plt.getInodeLocks(inode)
+	unlock := lock
+	unlock.Typ = syscall.F_UNLCK
+	if s := plt.SetLk(inode, unlock); s != fuse.OK {
+		t.Fatalf("prime unlock: got %v", s)
+	}
+	if !dead.dead {
+		t.Fatal("expected dead after cleanup")
+	}
+
+	// Force the broken state that production cannot reach but tests and
+	// future refactors might: dead entry still in the map.
+	plt.mu.Lock()
+	plt.inodes[inode] = dead
+	plt.mu.Unlock()
+
+	if il := plt.getInodeLocks(inode); il != nil {
+		t.Fatalf("getInodeLocks should drop a dead map entry, got %p", il)
+	}
+	plt.mu.Lock()
+	_, stillMapped := plt.inodes[inode]
+	plt.mu.Unlock()
+	if stillMapped {
+		t.Fatal("expected dead entry to be removed from the map")
+	}
+
+	// getOrCreateInodeLocks must also self-heal (replace the dead entry with
+	// a fresh live one) so SetLk's retry path cannot spin.
+	plt.mu.Lock()
+	plt.inodes[inode] = dead
+	plt.mu.Unlock()
+	fresh := plt.getOrCreateInodeLocks(inode)
+	if fresh == dead {
+		t.Fatal("getOrCreateInodeLocks should not return a dead entry")
+	}
+	if fresh.dead {
+		t.Fatal("fresh entry should not be dead")
+	}
+}
+
+// TestConcurrentFlockChurnPreservesMutualExclusion is a stress companion to
+// the deterministic tests above. It uses a Swap+CAS detector that flags
+// overlap at two points (on acquire and on release), so a second granted
+// holder is caught even if it sneaks in after the first goroutine's claim
+// but before its release. 16 goroutines churn whole-file exclusive flock on
+// one inode; with the race the detector fires hundreds of times per run,
+// with the fix it stays at zero.
+func TestConcurrentFlockChurnPreservesMutualExclusion(t *testing.T) {
+	plt := NewPosixLockTable()
+	const (
+		inode      = uint64(42)
+		numWorkers = 16
+		iterations = 500
+	)
+	var (
+		wg          sync.WaitGroup
+		holder      atomic.Int64 // 0 = nobody; otherwise = holder's claim token
+		overlapSeen atomic.Int32
+	)
+
+	for w := 0; w < numWorkers; w++ {
+		wg.Add(1)
+		go func(id int) {
+			defer wg.Done()
+			owner := uint64(100 + id)
+			lock := lockRange{
+				Start:   0,
+				End:     math.MaxUint64,
+				Typ:     syscall.F_WRLCK,
+				Owner:   owner,
+				Pid:     uint32(id + 1),
+				IsFlock: true,
+			}
+			unlock := lock
+			unlock.Typ = syscall.F_UNLCK
+			token := int64(id + 1)
+			for i := 0; i < iterations; i++ {
+				// SetLk(WRLCK) may only return OK (granted) or EAGAIN
+				// (conflict); anything else indicates a bug and the test
+				// must fail rather than spin. Use Errorf + return because
+				// Fatalf is not safe from a non-test goroutine.
+				for {
+					s := plt.SetLk(inode, lock)
+					if s == fuse.OK {
+						break
+					}
+					if s != fuse.EAGAIN {
+						t.Errorf("worker %d iter %d: unexpected SetLk(WRLCK) status %v", id, i, s)
+						return
+					}
+					runtime.Gosched()
+				}
+				// Claim the slot. If Swap observes a non-zero predecessor,
+				// another goroutine already believes it holds the lock.
+				if prev := holder.Swap(token); prev != 0 {
+					overlapSeen.Add(1)
+				}
+				// Widen the window so a concurrently-granted peer has a
+				// chance to race into its own Swap before we release.
+				runtime.Gosched()
+				runtime.Gosched()
+				// Release the slot. If CAS fails someone else overwrote our
+				// claim, which only happens when two holders raced.
+				if !holder.CompareAndSwap(token, 0) {
+					overlapSeen.Add(1)
+				}
+				if s := plt.SetLk(inode, unlock); s != fuse.OK {
+					t.Errorf("worker %d iter %d: unexpected SetLk(UNLCK) status %v", id, i, s)
+					return
+				}
+			}
+		}(w)
+	}
+	wg.Wait()
+
+	if n := overlapSeen.Load(); n != 0 {
+		t.Fatalf("flock overlap detected %d times: two owners simultaneously granted the same exclusive lock", n)
+	}
+}

weed/mount/syscall_shim_unix.go

@@ -0,0 +1,12 @@
+//go:build !windows
+
+package mount
+
+import "syscall"
+
+const (
+	f_RDLCK   = syscall.F_RDLCK
+	f_WRLCK   = syscall.F_WRLCK
+	f_UNLCK   = syscall.F_UNLCK
+	o_ACCMODE = syscall.O_ACCMODE
+)

weed/mount/syscall_shim_windows.go

@@ -0,0 +1,14 @@
+package mount
+
+// Windows has no fcntl lock types and no access-mode mask. The Linux values
+// stand in: nothing on Windows feeds these, and the lock table only ever
+// compares them against each other.
+const (
+	f_RDLCK   = 0
+	f_WRLCK   = 1
+	f_UNLCK   = 2
+	o_ACCMODE = 0x3
+
+	xattr_CREATE  = 1
+	xattr_REPLACE = 2
+)

weed/mount/syscall_shim_windows_test.go

@@ -0,0 +1,16 @@
+package mount
+
+import "testing"
+
+// The Windows adapter forwards cgofuse's xattr flags to SetXAttr untouched,
+// which only holds while both sides number them the same. cgofuse's side is
+// pinned in weed/mount/winfsp; this pins ours, so editing either alone fails
+// rather than silently turning a create into a replace.
+func TestXattrFlagValues(t *testing.T) {
+	if xattr_CREATE != 1 {
+		t.Errorf("xattr_CREATE = %d, want 1", xattr_CREATE)
+	}
+	if xattr_REPLACE != 2 {
+		t.Errorf("xattr_REPLACE = %d, want 2", xattr_REPLACE)
+	}
+}

weed/mount/unmount/unmount.go

@@ -0,0 +1,6 @@
+package unmount
+
+// Unmount tries to unmount the filesystem mounted at dir.
+func Unmount(dir string) error {
+	return unmount(dir)
+}

weed/mount/unmount/unmount_linux.go

@@ -0,0 +1,21 @@
+package unmount
+
+import (
+	"bytes"
+	"errors"
+	"os/exec"
+)
+
+func unmount(dir string) error {
+	cmd := exec.Command("fusermount", "-u", dir)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		if len(output) > 0 {
+			output = bytes.TrimRight(output, "\n")
+			msg := err.Error() + ": " + string(output)
+			err = errors.New(msg)
+		}
+		return err
+	}
+	return nil
+}

weed/mount/unmount/unmount_std.go

@@ -0,0 +1,17 @@
+//go:build !linux && !windows
+
+package unmount
+
+import (
+	"os"
+	"syscall"
+)
+
+func unmount(dir string) error {
+	err := syscall.Unmount(dir, 0)
+	if err != nil {
+		err = &os.PathError{Op: "unmount", Path: dir, Err: err}
+		return err
+	}
+	return nil
+}

weed/mount/unmount/unmount_unsupported.go

@@ -0,0 +1,7 @@
+//go:build windows
+
+package unmount
+
+func unmount(dir string) error {
+	return nil
+}

weed/mount/weedfs_access.go

@@ -0,0 +1,166 @@
+package mount
+
+import (
+	"os/user"
+	"strconv"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+type cachedGroupIDs struct {
+	groups    []string
+	expiresAt time.Time
+}
+
+var (
+	supplementaryGroupCache    = make(map[uint32]*cachedGroupIDs)
+	supplementaryGroupCacheMu  sync.RWMutex
+	supplementaryGroupCacheTTL = 5 * time.Minute
+
+	lookupSupplementaryGroupIDs = func(callerUid uint32) ([]string, error) {
+		u, err := user.LookupId(strconv.Itoa(int(callerUid)))
+		if err != nil {
+			glog.Warningf("hasAccess: user.LookupId for uid %d failed: %v", callerUid, err)
+			return nil, err
+		}
+		groupIDs, err := u.GroupIds()
+		if err != nil {
+			glog.Warningf("hasAccess: u.GroupIds for uid %d failed: %v", callerUid, err)
+			return nil, err
+		}
+		return groupIDs, nil
+	}
+)
+
+// cachedLookupSupplementaryGroupIDs returns supplementary group IDs for a UID,
+// caching results for 5 minutes to avoid repeated expensive system calls.
+func cachedLookupSupplementaryGroupIDs(callerUid uint32) ([]string, error) {
+	now := time.Now()
+
+	supplementaryGroupCacheMu.RLock()
+	cached, ok := supplementaryGroupCache[callerUid]
+	supplementaryGroupCacheMu.RUnlock()
+	if ok && now.Before(cached.expiresAt) {
+		return cached.groups, nil
+	}
+
+	groupIDs, err := lookupSupplementaryGroupIDs(callerUid)
+	if err != nil {
+		return nil, err
+	}
+
+	supplementaryGroupCacheMu.Lock()
+	supplementaryGroupCache[callerUid] = &cachedGroupIDs{
+		groups:    groupIDs,
+		expiresAt: now.Add(supplementaryGroupCacheTTL),
+	}
+	supplementaryGroupCacheMu.Unlock()
+
+	return groupIDs, nil
+}
+
+// clearSupplementaryGroupCache wipes the UID->groups cache for test isolation.
+func clearSupplementaryGroupCache() {
+	supplementaryGroupCacheMu.Lock()
+	defer supplementaryGroupCacheMu.Unlock()
+	for k := range supplementaryGroupCache {
+		delete(supplementaryGroupCache, k)
+	}
+}
+
+/**
+ * Check file access permissions
+ *
+ * This will be called for the access() system call.  If the
+ * 'default_permissions' mount option is given, this method is not
+ * called.
+ *
+ * This method is not called under Linux kernel versions 2.4.x
+ */
+func (wfs *WFS) Access(cancel <-chan struct{}, input *fuse.AccessIn) (code fuse.Status) {
+	_, _, entry, code := wfs.maybeReadEntry(input.NodeId)
+	if code != fuse.OK {
+		return code
+	}
+	if entry == nil || entry.Attributes == nil {
+		return fuse.EIO
+	}
+	// Map entry uid/gid from filer-space to local-space so the permission
+	// check compares like with like (caller uid/gid from FUSE are local).
+	fileUid, fileGid := entry.Attributes.Uid, entry.Attributes.Gid
+	if wfs.option.UidGidMapper != nil {
+		fileUid, fileGid = wfs.option.UidGidMapper.FilerToLocal(fileUid, fileGid)
+	}
+	if hasAccess(input.Uid, input.Gid, fileUid, fileGid, entry.Attributes.FileMode, input.Mask) {
+		return fuse.OK
+	}
+	return fuse.EACCES
+}
+
+func hasAccess(callerUid, callerGid, fileUid, fileGid uint32, perm uint32, mask uint32) bool {
+	mask &= fuse.R_OK | fuse.W_OK | fuse.X_OK
+	if mask == 0 {
+		return true
+	}
+	if callerUid == 0 {
+		return mask&fuse.X_OK == 0 || perm&0o111 != 0
+	}
+
+	if callerUid == fileUid {
+		return (perm>>6)&mask == mask
+	}
+
+	isMember := callerGid == fileGid
+	if !isMember {
+		groupIDs, err := cachedLookupSupplementaryGroupIDs(callerUid)
+		if err != nil {
+			// Cannot determine supplementary group membership.
+			// Fall through to "other" permission check since we already
+			// know the caller is not the owner (checked above) and not
+			// in the primary group.
+			return (perm & mask) == mask
+		}
+		fileGidStr := strconv.Itoa(int(fileGid))
+		for _, gidStr := range groupIDs {
+			if gidStr == fileGidStr {
+				isMember = true
+				break
+			}
+		}
+	}
+
+	if isMember {
+		return (perm>>3)&mask == mask
+	}
+
+	return (perm & mask) == mask
+}
+
+// checkStickyBit enforces the POSIX sticky-bit rule: when a directory has the
+// sticky bit set, only the file owner, the directory owner, or root may
+// delete or rename entries within it.
+func checkStickyBit(dirMode, dirUid, targetUid, callerUid uint32) fuse.Status {
+	if dirMode&0o1000 == 0 {
+		return fuse.OK
+	}
+	if callerUid == 0 || callerUid == dirUid || callerUid == targetUid {
+		return fuse.OK
+	}
+	return fuse.EPERM
+}
+
+// openFlagsToAccessMask converts open(2) flags to an access permission mask.
+func openFlagsToAccessMask(flags uint32) uint32 {
+	switch flags & uint32(o_ACCMODE) {
+	case syscall.O_WRONLY:
+		return fuse.W_OK
+	case syscall.O_RDWR:
+		return fuse.R_OK | fuse.W_OK
+	default: // O_RDONLY
+		return fuse.R_OK
+	}
+}

weed/mount/weedfs_access_bench_test.go

@@ -0,0 +1,45 @@
+package mount
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+)
+
+// BenchmarkCachedLookupSupplementaryGroupIDs measures cache hit performance.
+func BenchmarkCachedLookupSupplementaryGroupIDs(b *testing.B) {
+	oldLookupSupplementaryGroupIDs := lookupSupplementaryGroupIDs
+	lookupSupplementaryGroupIDs = func(uid uint32) ([]string, error) {
+		return []string{"456", "789", "1011"}, nil
+	}
+	clearSupplementaryGroupCache()
+	defer func() {
+		lookupSupplementaryGroupIDs = oldLookupSupplementaryGroupIDs
+		clearSupplementaryGroupCache()
+	}()
+
+	cachedLookupSupplementaryGroupIDs(999)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		cachedLookupSupplementaryGroupIDs(999)
+	}
+}
+
+// BenchmarkHasAccessPermissionCheck measures permission checks with the cache.
+// Simulates repeated permission checks for the same user against different files.
+func BenchmarkHasAccessPermissionCheck(b *testing.B) {
+	oldLookupSupplementaryGroupIDs := lookupSupplementaryGroupIDs
+	lookupSupplementaryGroupIDs = func(uid uint32) ([]string, error) {
+		return []string{"456", "789", "1011"}, nil
+	}
+	clearSupplementaryGroupCache()
+	defer func() {
+		lookupSupplementaryGroupIDs = oldLookupSupplementaryGroupIDs
+		clearSupplementaryGroupCache()
+	}()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		hasAccess(999, 999, 123, uint32(i%10), 0o040, fuse.R_OK|fuse.W_OK)
+	}
+}

weed/mount/weedfs_access_perf_test.go

@@ -0,0 +1,93 @@
+package mount
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+)
+
+// TestPermissionCheckPerformance simulates permission checks during a large copy operation.
+// Shows that with caching, even 50,000 permission checks only trigger 1 system lookup.
+func TestPermissionCheckPerformance(t *testing.T) {
+	oldLookupSupplementaryGroupIDs := lookupSupplementaryGroupIDs
+	lookupCount := 0
+	lookupSupplementaryGroupIDs = func(uid uint32) ([]string, error) {
+		lookupCount++
+		return []string{"456", "789", "1011"}, nil
+	}
+	defer func() {
+		lookupSupplementaryGroupIDs = oldLookupSupplementaryGroupIDs
+	}()
+
+	// Simulate copying 10,000 files as non-root user.
+	// Each file access requires multiple permission checks.
+	// Without caching, this would trigger 50,000 system lookups!
+	fileCount := 10000
+	checksPerFile := 5
+
+	clearSupplementaryGroupCache()
+	start := time.Now()
+	for i := 0; i < fileCount; i++ {
+		for j := 0; j < checksPerFile; j++ {
+			gid := uint32(100 + (i % 10))
+			hasAccess(999, 999, 123, gid, 0o040, fuse.R_OK|fuse.W_OK)
+		}
+	}
+	elapsed := time.Since(start)
+
+	totalChecks := fileCount * checksPerFile
+	if lookupCount != 1 {
+		t.Fatalf("Expected 1 system lookup (cache hit for UID 999), got %d", lookupCount)
+	}
+
+	opsPerSecond := float64(totalChecks) / elapsed.Seconds()
+	fmt.Printf("\n=== Permission Check Performance Test (WITH CACHE) ===\n")
+	fmt.Printf("Files simulated: %d\n", fileCount)
+	fmt.Printf("Checks per file: %d\n", checksPerFile)
+	fmt.Printf("Total checks: %d\n", totalChecks)
+	fmt.Printf("System lookups: %d (would be %d without cache)\n", lookupCount, totalChecks)
+	fmt.Printf("Lookups eliminated: %d (%.1f%% reduction)\n", totalChecks-lookupCount,
+		(1-float64(lookupCount)/float64(totalChecks))*100)
+	fmt.Printf("Time elapsed: %v\n", elapsed)
+	fmt.Printf("Throughput: %.0f checks/sec\n", opsPerSecond)
+}
+
+// BenchmarkPermissionCheckScaling shows how performance scales with unique users.
+func BenchmarkPermissionCheckScaling(b *testing.B) {
+	oldLookupSupplementaryGroupIDs := lookupSupplementaryGroupIDs
+	lookupSupplementaryGroupIDs = func(uid uint32) ([]string, error) {
+		return []string{"456", "789", "1011"}, nil
+	}
+	clearSupplementaryGroupCache()
+	defer func() {
+		lookupSupplementaryGroupIDs = oldLookupSupplementaryGroupIDs
+		clearSupplementaryGroupCache()
+	}()
+
+	// Simulate repeated permission checks for the same user
+	// (typical single-user copy operation)
+	b.Run("single-user", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			hasAccess(999, 999, 123, 456, 0o040, fuse.R_OK|fuse.W_OK)
+		}
+	})
+
+	b.Run("multi-user-10", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			uid := uint32(1000 + (i % 10))
+			hasAccess(uid, 100, 123, 456, 0o040, fuse.R_OK|fuse.W_OK)
+		}
+	})
+
+	b.Run("multi-user-100", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			uid := uint32(1000 + (i % 100))
+			hasAccess(uid, 100, 123, 456, 0o040, fuse.R_OK|fuse.W_OK)
+		}
+	})
+}

weed/mount/weedfs_async_flush.go

@@ -0,0 +1,146 @@
+package mount
+
+import (
+	"context"
+	"time"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// asyncFlushItem holds the data needed for a background flush work item.
+type asyncFlushItem struct {
+	fh   *FileHandle
+	done chan struct{}
+}
+
+// startAsyncFlushWorkers launches a fixed pool of goroutines that process
+// background flush work items from asyncFlushCh. This bounds the number of
+// concurrent flush operations to prevent resource exhaustion (connections,
+// goroutines) when many files are closed rapidly (e.g., cp -r with writebackCache).
+func (wfs *WFS) startAsyncFlushWorkers(numWorkers int) {
+	wfs.asyncFlushCh = make(chan *asyncFlushItem, numWorkers*4)
+	for i := 0; i < numWorkers; i++ {
+		go wfs.asyncFlushWorker()
+	}
+}
+
+func (wfs *WFS) asyncFlushWorker() {
+	for item := range wfs.asyncFlushCh {
+		wfs.processAsyncFlushItem(item)
+	}
+}
+
+func (wfs *WFS) processAsyncFlushItem(item *asyncFlushItem) {
+	defer wfs.asyncFlushWg.Done()
+	defer func() {
+		// Remove from fhMap first (so AcquireFileHandle creates a fresh handle).
+		wfs.fhMap.RemoveFileHandle(item.fh.fh, item.fh.inode)
+		// Then signal completion (unblocks waitForPendingAsyncFlush).
+		close(item.done)
+		wfs.pendingAsyncFlushMu.Lock()
+		delete(wfs.pendingAsyncFlush, item.fh.inode)
+		wfs.pendingAsyncFlushMu.Unlock()
+	}()
+	wfs.completeAsyncFlush(item.fh)
+}
+
+// completeAsyncFlush performs the deferred data upload and metadata flush
+// that was skipped in doFlush() for writebackCache mode.
+//
+// This enables close() to return immediately for small file workloads (e.g., rsync),
+// while the actual I/O happens concurrently in the background.
+func (wfs *WFS) completeAsyncFlush(fh *FileHandle) {
+	glog.V(4).Infof("completeAsyncFlush inode %d fh %d saved=%s/%s dirtyMetadata=%v isDeleted=%v isRenamed=%v",
+		fh.inode, fh.fh, fh.savedDir, fh.savedName, fh.dirtyMetadata, fh.isDeleted, fh.isRenamed)
+
+	// Phase 1: Flush dirty pages — seals writable chunks, uploads to volume servers, and waits.
+	// The underlying UploadWithRetry already retries transient HTTP/gRPC errors internally,
+	// so a failure here indicates a persistent issue; the chunk data has been freed.
+	if err := fh.dirtyPages.FlushData(); err != nil {
+		glog.Errorf("completeAsyncFlush inode %d: data flush failed: %v", fh.inode, err)
+		// Data is lost at this point (chunks freed after internal retry exhaustion).
+		// Proceed to cleanup to avoid resource leaks and unmount hangs.
+	} else if fh.dirtyMetadata {
+		// Phase 2: Flush metadata unless the file was explicitly unlinked.
+		//
+		// isDeleted is set by the Unlink handler when it finds a draining
+		// handle.  In that case the filer entry is already gone and
+		// flushing would recreate it.  The uploaded chunks become orphans
+		// and are cleaned up by volume.fsck.
+		if fh.isDeleted || fh.isRenamed {
+			if fh.isDeleted {
+				glog.V(3).Infof("completeAsyncFlush inode %d: file was unlinked, skipping metadata flush", fh.inode)
+			} else {
+				glog.V(3).Infof("completeAsyncFlush inode %d: file was renamed, skipping old-path metadata flush (Rename handles it)", fh.inode)
+			}
+		} else if savedInode, found := wfs.inodeToPath.GetInode(util.FullPath(fh.savedDir).Child(fh.savedName)); !found || savedInode != fh.inode {
+			// The saved path no longer maps to this inode — the file was
+			// renamed (or deleted and recreated). Flushing metadata under
+			// the old path would re-insert a stale entry into the meta
+			// cache, breaking git's lock file protocol.
+			glog.V(3).Infof("completeAsyncFlush inode %d: saved path %s/%s no longer maps to this inode, skipping metadata flush",
+				fh.inode, fh.savedDir, fh.savedName)
+		} else {
+			// Resolve the current path for metadata flush.
+			//
+			// Try GetPath first — it reflects any rename that happened
+			// after close().  If the inode mapping is gone (Forget
+			// dropped it after the kernel's lookup count hit zero), fall
+			// back to the last path saved on the handle. Rename keeps
+			// that fallback current, so it is always the newest known path.
+			//
+			// Forget does NOT mean the file was deleted — it only means
+			// the kernel evicted its cache entry.
+			dir, name := fh.savedDir, fh.savedName
+			fileFullPath := util.FullPath(dir).Child(name)
+
+			if resolvedPath, status := wfs.inodeToPath.GetPath(fh.inode); status == fuse.OK {
+				dir, name = resolvedPath.DirAndName()
+				fileFullPath = resolvedPath
+			}
+
+			wfs.flushMetadataWithRetry(fh, dir, name, fileFullPath)
+		}
+	}
+
+	glog.V(3).Infof("completeAsyncFlush done inode %d fh %d", fh.inode, fh.fh)
+
+	// Phase 3: Destroy the upload pipeline and free resources.
+	fh.ReleaseHandle()
+}
+
+// flushMetadataWithRetry attempts to flush file metadata to the filer, retrying
+// with exponential backoff on transient errors. The chunk data is already on the
+// volume servers at this point; only the filer metadata reference needs persisting.
+func (wfs *WFS) flushMetadataWithRetry(fh *FileHandle, dir, name string, fileFullPath util.FullPath) {
+	err := retryMetadataFlush(context.Background(), func() error {
+		return wfs.flushMetadataToFiler(context.Background(), fh, dir, name, fh.asyncFlushUid, fh.asyncFlushGid)
+	}, func(nextAttempt, totalAttempts int, backoff time.Duration, err error) {
+		glog.Warningf("completeAsyncFlush %s: retrying metadata flush (attempt %d/%d) after %v: %v",
+			fileFullPath, nextAttempt, totalAttempts, backoff, err)
+	})
+	if err != nil {
+		glog.Errorf("completeAsyncFlush %s: metadata flush failed after %d attempts: %v - "+
+			"chunks are uploaded but NOT referenced in filer metadata; "+
+			"they will appear as orphans in volume.fsck",
+			fileFullPath, metadataFlushRetries+1, err)
+	}
+}
+
+// WaitForAsyncFlush waits for all pending background flush work items to complete.
+// Called before unmount cleanup to ensure no data is lost.
+func (wfs *WFS) WaitForAsyncFlush() {
+	wfs.asyncFlushWg.Wait()
+	// Shutdown reaches here from both the interrupt hook and the path that
+	// resumes once serving stops, so closing has to survive a second caller.
+	wfs.asyncFlushClose.Do(func() {
+		if wfs.asyncFlushCh != nil {
+			close(wfs.asyncFlushCh)
+		}
+		if wfs.streamMutate != nil {
+			wfs.streamMutate.Close()
+		}
+	})
+}

weed/mount/weedfs_attr.go

@@ -0,0 +1,541 @@
+package mount
+
+import (
+	"context"
+	"os"
+	"syscall"
+	"time"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func (wfs *WFS) GetAttr(cancel <-chan struct{}, input *fuse.GetAttrIn, out *fuse.AttrOut) (code fuse.Status) {
+	glog.V(4).Infof("GetAttr %v", input.NodeId)
+	if input.NodeId == 1 {
+		wfs.setRootAttr(out)
+		if wfs.option.PosixDirNlink {
+			wfs.applyDirNlink(&out.Attr, util.FullPath(wfs.option.FilerMountRootPath))
+		}
+		return fuse.OK
+	}
+
+	inode := input.NodeId
+	path, fh, entry, status := wfs.maybeReadEntry(inode)
+	if status != fuse.OK {
+		return status
+	}
+	out.AttrValid = wfs.attrValidSec
+	// An open handle's entry has two sets of writers: async upload workers
+	// append chunks under the LockedEntry lock, while Write and the metadata
+	// flush rewrite size, times and the whole chunk slice under the handle
+	// lock. Hold both for reading, in that order, so FileSize never iterates a
+	// slice mid-reallocation. Re-read the entry under them in case SetEntry
+	// swapped the pointer since maybeReadEntry.
+	if fh != nil {
+		fhActiveLock := wfs.fhLockTable.AcquireLock("GetAttr", fh.fh, util.SharedLock)
+		fh.entry.RLock()
+		entry = fh.entry.Entry
+		wfs.setAttrByPbEntry(&out.Attr, inode, entry, true)
+		fh.entry.RUnlock()
+		wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
+	} else {
+		wfs.setAttrByPbEntry(&out.Attr, inode, entry, true)
+	}
+	wfs.applyInMemoryAtime(&out.Attr, inode)
+	applyUnlinkedNlink(&out.Attr, path)
+	if entry.GetIsDirectory() {
+		wfs.applyInMemoryDirMtime(&out.Attr, inode)
+		if wfs.option.PosixDirNlink {
+			wfs.applyDirNlink(&out.Attr, path)
+		}
+	}
+	return status
+}
+
+func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse.AttrOut) (code fuse.Status) {
+
+	// Check quota including uncommitted writes for real-time enforcement
+	if wfs.IsOverQuotaWithUncommitted() {
+		return fuse.Status(syscall.ENOSPC)
+	}
+
+	path, fh, entry, status := wfs.maybeReadEntry(input.NodeId)
+	if status != fuse.OK || entry == nil {
+		return status
+	}
+	if size, ok := input.GetSize(); ok && fh != nil && fh.dirtyPages.HasWrites() && size < filer.FileSize(entry) {
+		// The truncation below trims chunks; dirty pages it cannot see. Left
+		// alone, pages beyond the new size come back with the next flush and
+		// grow the file again, so turn them into chunks first. Runs before
+		// the entry locks below: the flush takes its own.
+		ctx, cancelFunc := context.WithTimeout(context.Background(), metadataFlushTimeout)
+		flushStatus := wfs.doFlush(ctx, fh, input.Uid, input.Gid, false)
+		cancelFunc()
+		if flushStatus != fuse.OK {
+			return flushStatus
+		}
+	}
+	if fh != nil {
+		fh.entryLock.Lock()
+		defer fh.entryLock.Unlock()
+		// entry is the handle's shared LockedEntry.Entry. Async upload workers
+		// mutate its Chunks slice under the LockedEntry lock (AddChunks); hold
+		// that same lock so the truncate and FileSize reads below don't tear
+		// against a concurrent append. Re-read under the lock in case SetEntry
+		// swapped the pointer since maybeReadEntry, so we don't mutate an
+		// orphaned entry and lose the update.
+		fh.entry.Lock()
+		defer fh.entry.Unlock()
+		entry = fh.entry.Entry
+	}
+
+	wormEnforced, wormEnabled := wfs.wormEnforcedForEntry(path, entry)
+	if wormEnforced {
+		return fuse.EPERM
+	}
+
+	if size, ok := input.GetSize(); ok {
+		glog.V(4).Infof("%v setattr set size=%v chunks=%d", path, size, len(entry.GetChunks()))
+		// Invalidate the open-mtime cache so the next Open does not set
+		// FOPEN_KEEP_CACHE with stale kernel page cache data.
+		wfs.invalidateOpenMtimeCache(input.NodeId)
+		oldFileSize := filer.FileSize(entry)
+		if size < oldFileSize {
+			// fmt.Printf("truncate %v \n", fullPath)
+			var chunks []*filer_pb.FileChunk
+			var truncatedChunks []*filer_pb.FileChunk
+			for _, chunk := range entry.GetChunks() {
+				int64Size := int64(chunk.Size)
+				if chunk.Offset+int64Size > int64(size) {
+					// this chunk is truncated
+					int64Size = int64(size) - chunk.Offset
+					if int64Size > 0 {
+						chunks = append(chunks, chunk)
+						glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size)
+						chunk.Size = uint64(int64Size)
+					} else {
+						glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString())
+						truncatedChunks = append(truncatedChunks, chunk)
+					}
+				} else {
+					chunks = append(chunks, chunk)
+				}
+			}
+			// set the new chunks and reset entry cache
+			entry.Chunks = chunks
+			if fh != nil {
+				fh.entryChunkGroup.SetChunks(chunks)
+			}
+		}
+		truncNow := time.Now()
+		entry.Attributes.Mtime = truncNow.Unix()
+		entry.Attributes.MtimeNs = int32(truncNow.Nanosecond())
+		entry.Attributes.FileSize = size
+		if size > oldFileSize {
+			// The writes that fill the range will not grow the file, so they
+			// charge nothing; the growth is counted here or the quota never
+			// sees it. Matches Write and Fallocate.
+			wfs.AddUncommittedBytes(int64(size - oldFileSize))
+		}
+
+	}
+
+	if mode, ok := input.GetMode(); ok {
+		// commit the file to worm when it is set to readonly at the first time
+		if entry.WormEnforcedAtTsNs == 0 && wormEnabled && !hasWritePermission(mode) {
+			entry.WormEnforcedAtTsNs = time.Now().UnixNano()
+		}
+
+		// glog.V(4).Infof("setAttr mode %o", mode)
+		entry.Attributes.FileMode = chmod(entry.Attributes.FileMode, mode)
+		if input.NodeId == 1 {
+			wfs.option.MountMode = os.FileMode(chmod(uint32(wfs.option.MountMode), mode))
+		}
+	}
+
+	ownerChanged := false
+	if uid, ok := input.GetUID(); ok {
+		entry.Attributes.Uid = uid
+		ownerChanged = true
+		if input.NodeId == 1 {
+			wfs.option.MountUid = uid
+		}
+	}
+
+	if gid, ok := input.GetGID(); ok {
+		entry.Attributes.Gid = gid
+		ownerChanged = true
+		if input.NodeId == 1 {
+			wfs.option.MountGid = gid
+		}
+	}
+
+	// POSIX: clear SUID/SGID bits when ownership changes (unless caller is root).
+	if ownerChanged && input.Uid != 0 {
+		entry.Attributes.FileMode &^= 0o6000
+	}
+
+	if atime, ok := input.GetATime(); ok {
+		wfs.setAtime(input.NodeId, atime)
+	}
+
+	if mtime, ok := input.GetMTime(); ok {
+		entry.Attributes.Mtime = mtime.Unix()
+		entry.Attributes.MtimeNs = int32(mtime.Nanosecond())
+	}
+
+	// POSIX: update ctime on any metadata change.
+	now := time.Now()
+	entry.Attributes.Ctime = now.Unix()
+	entry.Attributes.CtimeNs = int32(now.Nanosecond())
+
+	out.AttrValid = wfs.attrValidSec
+	size, includeSize := input.GetSize()
+	if includeSize {
+		out.Attr.Size = size
+	}
+	wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry, !includeSize)
+	wfs.applyInMemoryAtime(&out.Attr, input.NodeId)
+	applyUnlinkedNlink(&out.Attr, path)
+
+	if fh != nil {
+		fh.dirtyMetadata = true
+		return fuse.OK
+	}
+	if path == "" {
+		// removed while open: the remembered entry is all there is to update
+		wfs.rememberRemovedDir(input.NodeId, entry)
+		return fuse.OK
+	}
+
+	return wfs.saveEntry(path, entry)
+
+}
+
+func (wfs *WFS) setRootAttr(out *fuse.AttrOut) {
+	now := uint64(time.Now().Unix())
+	out.AttrValid = 119
+	out.Ino = 1
+	setBlksize(&out.Attr, blockSize)
+	out.Uid = wfs.option.MountUid
+	out.Gid = wfs.option.MountGid
+	out.Mtime = now
+	out.Ctime = now
+	out.Atime = now
+	out.Mode = toSyscallType(os.ModeDir) | uint32(wfs.option.MountMode)
+	out.Nlink = 2
+}
+
+func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.Entry, calculateSize bool) {
+	out.Ino = inode
+	setBlksize(out, blockSize)
+	if entry == nil {
+		return
+	}
+	if entry.Attributes != nil && entry.Attributes.Inode != 0 {
+		out.Ino = entry.Attributes.Inode
+	}
+	if calculateSize {
+		out.Size = filer.FileSize(entry)
+	}
+	if entry.FileMode()&os.ModeSymlink != 0 {
+		out.Size = uint64(len(entry.Attributes.SymlinkTarget))
+	}
+	out.Blocks = (out.Size + blockSize - 1) / blockSize
+	out.Mtime = uint64(entry.Attributes.Mtime)
+	out.Mtimensec = uint32(entry.Attributes.MtimeNs)
+	if entry.Attributes.Ctime != 0 {
+		out.Ctime = uint64(entry.Attributes.Ctime)
+		out.Ctimensec = uint32(entry.Attributes.CtimeNs)
+	} else {
+		out.Ctime = uint64(entry.Attributes.Mtime)
+		out.Ctimensec = uint32(entry.Attributes.MtimeNs)
+	}
+	out.Atime = uint64(entry.Attributes.Mtime)
+	out.Atimensec = uint32(entry.Attributes.MtimeNs)
+	// In-memory atime overlay is applied by the caller via applyInMemoryAtime.
+	out.Mode = toSyscallMode(os.FileMode(entry.Attributes.FileMode))
+	if entry.IsDirectory {
+		out.Nlink = 2
+	} else if entry.HardLinkCounter > 0 {
+		out.Nlink = uint32(entry.HardLinkCounter)
+	} else {
+		out.Nlink = 1
+	}
+	out.Uid = entry.Attributes.Uid
+	out.Gid = entry.Attributes.Gid
+	out.Rdev = entry.Attributes.Rdev
+}
+
+func (wfs *WFS) setAttrByFilerEntry(out *fuse.Attr, inode uint64, entry *filer.Entry) {
+	out.Ino = inode
+	out.Size = entry.FileSize
+	if entry.Mode&os.ModeSymlink != 0 {
+		out.Size = uint64(len(entry.SymlinkTarget))
+	}
+	out.Blocks = (out.Size + blockSize - 1) / blockSize
+	setBlksize(out, blockSize)
+	out.Atime = uint64(entry.Attr.Mtime.Unix())
+	out.Atimensec = uint32(entry.Attr.Mtime.Nanosecond())
+	out.Mtime = uint64(entry.Attr.Mtime.Unix())
+	out.Mtimensec = uint32(entry.Attr.Mtime.Nanosecond())
+	if !entry.Attr.Ctime.IsZero() {
+		out.Ctime = uint64(entry.Attr.Ctime.Unix())
+		out.Ctimensec = uint32(entry.Attr.Ctime.Nanosecond())
+	} else {
+		out.Ctime = uint64(entry.Attr.Mtime.Unix())
+		out.Ctimensec = uint32(entry.Attr.Mtime.Nanosecond())
+	}
+	out.Mode = toSyscallMode(entry.Attr.Mode)
+	if entry.IsDirectory() {
+		out.Nlink = 2
+	} else if entry.HardLinkCounter > 0 {
+		out.Nlink = uint32(entry.HardLinkCounter)
+	} else {
+		out.Nlink = 1
+	}
+	out.Uid = entry.Attr.Uid
+	out.Gid = entry.Attr.Gid
+	out.Rdev = entry.Attr.Rdev
+}
+
+func (wfs *WFS) outputPbEntry(out *fuse.EntryOut, inode uint64, entry *filer_pb.Entry) {
+	out.NodeId = inode
+	out.Generation = 1
+	out.EntryValid = wfs.entryValidSec
+	out.AttrValid = wfs.attrValidSec
+	wfs.setAttrByPbEntry(&out.Attr, inode, entry, true)
+}
+
+func (wfs *WFS) outputFilerEntry(out *fuse.EntryOut, inode uint64, entry *filer.Entry) {
+	out.NodeId = inode
+	out.Generation = 1
+	out.EntryValid = wfs.entryValidSec
+	out.AttrValid = wfs.attrValidSec
+	wfs.setAttrByFilerEntry(&out.Attr, inode, entry)
+}
+
+// touchDirMtimeCtimeBest updates a directory's mtime and ctime using the
+// best strategy for the current mode:
+//   - WritebackCache: local meta cache only (no filer RPC)
+//   - Normal mode: filer UpdateEntry RPC for POSIX correctness
+func (wfs *WFS) touchDirMtimeCtimeBest(dirPath util.FullPath) {
+	if wfs.option.WritebackCache {
+		wfs.touchDirMtimeCtimeLocal(dirPath)
+	} else {
+		wfs.touchDirMtimeCtime(dirPath)
+	}
+}
+
+// touchDirMtimeCtime updates a directory's mtime and ctime on the filer.
+// POSIX requires this when entries are created or removed in the directory.
+func (wfs *WFS) touchDirMtimeCtime(dirPath util.FullPath) {
+	dirEntry, _, code := wfs.maybeLoadEntry(dirPath)
+	if code != fuse.OK || dirEntry == nil || dirEntry.Attributes == nil {
+		return
+	}
+	now := time.Now()
+	dirEntry.Attributes.Mtime = now.Unix()
+	dirEntry.Attributes.MtimeNs = int32(now.Nanosecond())
+	dirEntry.Attributes.Ctime = now.Unix()
+	dirEntry.Attributes.CtimeNs = int32(now.Nanosecond())
+	wfs.saveEntry(dirPath, dirEntry)
+}
+
+// touchDirMtimeCtimeLocal updates a directory's mtime and ctime in an in-memory
+// overlay, avoiding LevelDB reads and writes entirely. The overlay is applied
+// by applyInMemoryDirMtime when GetAttr/Lookup reads the directory's attributes.
+func (wfs *WFS) touchDirMtimeCtimeLocal(dirPath util.FullPath) {
+	if inode, found := wfs.inodeToPath.GetInode(dirPath); found {
+		wfs.setDirMtime(inode, time.Now())
+	}
+}
+
+const dirMtimeMapMaxSize = 8192
+
+func (wfs *WFS) setDirMtime(inode uint64, t time.Time) {
+	wfs.dirMtimeMu.Lock()
+	defer wfs.dirMtimeMu.Unlock()
+	if len(wfs.dirMtimeMap) >= dirMtimeMapMaxSize {
+		for k := range wfs.dirMtimeMap {
+			delete(wfs.dirMtimeMap, k)
+			break
+		}
+	}
+	wfs.dirMtimeMap[inode] = t
+}
+
+// applyInMemoryDirMtime overlays the in-memory mtime/ctime onto fuse.Attr
+// for directories that had recent child mutations.
+func (wfs *WFS) applyInMemoryDirMtime(out *fuse.Attr, inode uint64) {
+	wfs.dirMtimeMu.Lock()
+	if t, ok := wfs.dirMtimeMap[inode]; ok {
+		sec := uint64(t.Unix())
+		nsec := uint32(t.Nanosecond())
+		if sec > out.Mtime || (sec == out.Mtime && nsec > out.Mtimensec) {
+			out.Mtime = sec
+			out.Mtimensec = nsec
+			out.Ctime = sec
+			out.Ctimensec = nsec
+		}
+	}
+	wfs.dirMtimeMu.Unlock()
+}
+
+const atimeMapMaxSize = 8192
+
+// setAtime stores an in-memory atime for an inode. The map is bounded;
+// when full, a random entry is evicted.
+func (wfs *WFS) setAtime(inode uint64, t time.Time) {
+	wfs.atimeMu.Lock()
+	defer wfs.atimeMu.Unlock()
+	if len(wfs.atimeMap) >= atimeMapMaxSize {
+		// evict one random entry
+		for k := range wfs.atimeMap {
+			delete(wfs.atimeMap, k)
+			break
+		}
+	}
+	wfs.atimeMap[inode] = t
+}
+
+// applyInMemoryAtime overlays the in-memory atime onto a fuse.Attr if present.
+// forgetInMemoryTimes drops the overlays for an inode. All these maps are keyed
+// by inode and inodes are derived from the path, so a delete and recreate can
+// hand the same number to a different file — which would then inherit the
+// previous one's access or modification time, or a removed directory's entry.
+func (wfs *WFS) forgetInMemoryTimes(inode uint64) {
+	wfs.atimeMu.Lock()
+	delete(wfs.atimeMap, inode)
+	wfs.atimeMu.Unlock()
+
+	wfs.dirMtimeMu.Lock()
+	delete(wfs.dirMtimeMap, inode)
+	wfs.dirMtimeMu.Unlock()
+
+	wfs.removedDirMu.Lock()
+	delete(wfs.removedDirs, inode)
+	wfs.removedDirMu.Unlock()
+}
+
+// rememberRemovedDir keeps the last-known entry of a directory removed while
+// the kernel still references its inode. A directory has no file handle to
+// live on through, so this is what serves fstat/fchmod/f*xattr on a still-open
+// descriptor until the final forget. Mutations publish through here as well,
+// replacing the stored entry wholesale.
+//
+// The final forget's cleanup cannot miss an insert: Rmdir inserts inside
+// RemovePath's callback, under the same inode table lock the forget releases
+// under, and a publish runs inside a request whose open descriptor keeps the
+// kernel from issuing that forget at all.
+func (wfs *WFS) rememberRemovedDir(inode uint64, entry *filer_pb.Entry) {
+	wfs.removedDirMu.Lock()
+	if wfs.removedDirs == nil {
+		wfs.removedDirs = make(map[uint64]*filer_pb.Entry)
+	}
+	wfs.removedDirs[inode] = entry
+	wfs.removedDirMu.Unlock()
+}
+
+// removedDirEntry hands out a private copy: stored entries are never mutated
+// in place, so concurrent readers cannot see a half-applied change.
+func (wfs *WFS) removedDirEntry(inode uint64) *filer_pb.Entry {
+	wfs.removedDirMu.Lock()
+	entry := wfs.removedDirs[inode]
+	wfs.removedDirMu.Unlock()
+	if entry == nil {
+		return nil
+	}
+	return proto.Clone(entry).(*filer_pb.Entry)
+}
+
+func (wfs *WFS) applyInMemoryAtime(out *fuse.Attr, inode uint64) {
+	wfs.atimeMu.Lock()
+	if t, ok := wfs.atimeMap[inode]; ok {
+		out.Atime = uint64(t.Unix())
+		out.Atimensec = uint32(t.Nanosecond())
+	}
+	wfs.atimeMu.Unlock()
+}
+
+// applyUnlinkedNlink zeroes nlink for an inode no name points at any more: the
+// file was unlinked while open and lives on only through its handle. Every
+// reply carrying attributes must say so, or the kernel caches a live nlink.
+func applyUnlinkedNlink(out *fuse.Attr, path util.FullPath) {
+	if path == "" {
+		out.Nlink = 0
+	}
+}
+
+// applyDirNlink sets nlink = 2 + number_of_subdirectories for a directory.
+// Uses the in-memory subdirectory count tracked by mkdir/rmdir/rename.
+func (wfs *WFS) applyDirNlink(out *fuse.Attr, dirPath util.FullPath) {
+	count := wfs.inodeToPath.GetSubdirCount(dirPath)
+	if count > 0 {
+		out.Nlink = 2 + uint32(count)
+	}
+}
+
+func chmod(existing uint32, mode uint32) uint32 {
+	return existing&^07777 | mode&07777
+}
+
+const ownerWrite = 0o200
+const groupWrite = 0o020
+const otherWrite = 0o002
+
+func hasWritePermission(mode uint32) bool {
+	return (mode&ownerWrite != 0) || (mode&groupWrite != 0) || (mode&otherWrite != 0)
+}
+
+func toSyscallMode(mode os.FileMode) uint32 {
+	return toSyscallType(mode) | uint32(mode)
+}
+
+func toSyscallType(mode os.FileMode) uint32 {
+	switch mode & os.ModeType {
+	case os.ModeDir:
+		return syscall.S_IFDIR
+	case os.ModeSymlink:
+		return syscall.S_IFLNK
+	case os.ModeNamedPipe:
+		return syscall.S_IFIFO
+	case os.ModeSocket:
+		return syscall.S_IFSOCK
+	case os.ModeDevice:
+		return syscall.S_IFBLK
+	case os.ModeCharDevice:
+		return syscall.S_IFCHR
+	default:
+		return syscall.S_IFREG
+	}
+}
+
+func toOsFileType(mode uint32) os.FileMode {
+	switch mode & (syscall.S_IFMT & 0xffff) {
+	case syscall.S_IFDIR:
+		return os.ModeDir
+	case syscall.S_IFLNK:
+		return os.ModeSymlink
+	case syscall.S_IFIFO:
+		return os.ModeNamedPipe
+	case syscall.S_IFSOCK:
+		return os.ModeSocket
+	case syscall.S_IFBLK:
+		return os.ModeDevice
+	case syscall.S_IFCHR:
+		return os.ModeCharDevice
+	default:
+		return 0
+	}
+}
+
+func toOsFileMode(mode uint32) os.FileMode {
+	return toOsFileType(mode) | os.FileMode(mode&07777)
+}

weed/mount/weedfs_attr_darwin.go

@@ -0,0 +1,8 @@
+package mount
+
+import (
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+)
+
+func setBlksize(out *fuse.Attr, size uint32) {
+}

weed/mount/weedfs_attr_freebsd.go

@@ -0,0 +1,8 @@
+package mount
+
+import (
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+)
+
+func setBlksize(out *fuse.Attr, size uint32) {
+}

weed/mount/weedfs_attr_linux.go

@@ -0,0 +1,9 @@
+package mount
+
+import (
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+)
+
+func setBlksize(out *fuse.Attr, size uint32) {
+	out.Blksize = size
+}

weed/mount/weedfs_attr_quota_test.go

@@ -0,0 +1,38 @@
+package mount
+
+import (
+	"sync/atomic"
+	"testing"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+)
+
+// TestSetAttrChargesTheGrowth pins the quota accounting for a truncate up: the
+// writes that fill the range do not grow the file, so they charge nothing and
+// the growth has to be counted where it happens, as Write and Fallocate do.
+func TestSetAttrChargesTheGrowth(t *testing.T) {
+	atomic.StoreInt64(&uncommittedBytes, 0)
+	wfs, fh := newOpenFileHandle(t, 10)
+	wfs.option.Quota = 100 << 20
+
+	in := &fuse.SetAttrIn{}
+	in.NodeId = fh.inode
+	in.Valid = fuse.FATTR_SIZE
+	in.Size = 8192
+	var out fuse.AttrOut
+	if status := wfs.SetAttr(nil, in, &out); status != fuse.OK {
+		t.Fatalf("SetAttr size up: got %v, want OK", status)
+	}
+	if got := wfs.GetUncommittedBytes(); got != 8192-10 {
+		t.Fatalf("uncommitted bytes: got %d, want %d", got, 8192-10)
+	}
+
+	// Truncating back down frees space rather than consuming it.
+	in.Size = 0
+	if status := wfs.SetAttr(nil, in, &out); status != fuse.OK {
+		t.Fatalf("SetAttr size down: got %v, want OK", status)
+	}
+	if got := wfs.GetUncommittedBytes(); got != 8192-10 {
+		t.Fatalf("uncommitted bytes after a shrink: got %d, want %d", got, 8192-10)
+	}
+}

weed/mount/weedfs_attr_race_test.go

@@ -0,0 +1,155 @@
+package mount
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// TestAttrChunkRace guards the locking around an open handle's chunk slice.
+//
+// With writebackCache, async upload workers append chunks to an open file
+// handle's shared entry under the LockedEntry lock (FileHandle.AddChunks),
+// while metadata ops compute the file size by iterating entry.Chunks. SetAttr
+// and GetAttr used to read that slice without the LockedEntry lock, so a
+// concurrent append that reallocated the backing array produced a torn slice
+// read and a nil pointer dereference in filer.TotalSize. Run under -race.
+func TestAttrChunkRace(t *testing.T) {
+	wfs := &WFS{
+		option:         &Option{},
+		inodeToPath:    NewInodeToPath(util.FullPath("/"), 0),
+		fhMap:          NewFileHandleToInode(),
+		fhLockTable:    util.NewLockTable[FileHandleId](),
+		openMtimeCache: make(map[uint64][2]int64, 8),
+	}
+
+	const inode = uint64(42)
+	fullPath := util.FullPath("/dir/sample.txt")
+	wfs.inodeToPath.Lookup(fullPath, 1, false, false, inode, true)
+
+	entry := &filer_pb.Entry{
+		Name:       "sample.txt",
+		Attributes: &filer_pb.FuseAttributes{FileMode: 0644},
+	}
+	chunkGroup, err := filer.NewChunkGroup(nil, nil, nil, 1, nil, nil)
+	if err != nil {
+		t.Fatalf("NewChunkGroup: %v", err)
+	}
+	fh := &FileHandle{
+		fh:              FileHandleId(1),
+		inode:           inode,
+		wfs:             wfs,
+		entry:           &LockedEntry{Entry: entry},
+		entryChunkGroup: chunkGroup,
+	}
+	fh.dirtyPages = newPageWriter(fh, 1<<20)
+	wfs.fhMap.inode2fh[inode] = fh
+	wfs.fhMap.fh2inode[fh.fh] = inode
+
+	const iterations = 2000
+	var wg sync.WaitGroup
+	wg.Add(3)
+
+	// Async uploader: append chunks, reallocating the backing array.
+	go func() {
+		defer wg.Done()
+		for i := 0; i < iterations; i++ {
+			fh.AddChunks([]*filer_pb.FileChunk{{FileId: "x", Offset: int64(i), Size: 1}})
+		}
+	}()
+
+	// SetAttr: mtime-only recomputes FileSize by iterating chunks; a shrinking
+	// size takes the truncate path that rewrites entry.Chunks under the lock.
+	go func() {
+		defer wg.Done()
+		for i := 0; i < iterations; i++ {
+			in := &fuse.SetAttrIn{}
+			in.NodeId = inode
+			if i%2 == 0 {
+				in.Valid = fuse.FATTR_MTIME
+				in.Mtime = uint64(i)
+			} else {
+				in.Valid = fuse.FATTR_SIZE
+				in.Size = uint64(i % 8)
+			}
+			var out fuse.AttrOut
+			wfs.SetAttr(nil, in, &out)
+		}
+	}()
+
+	// GetAttr also computes FileSize by iterating chunks.
+	go func() {
+		defer wg.Done()
+		for i := 0; i < iterations; i++ {
+			in := &fuse.GetAttrIn{}
+			in.NodeId = inode
+			var out fuse.AttrOut
+			wfs.GetAttr(nil, in, &out)
+		}
+	}()
+
+	wg.Wait()
+}
+
+// TestReadFromChunksRace guards the read path's chunk-slice access. The read
+// path holds fh.entryLock (which excludes SetAttr) but not the LockedEntry lock
+// the async uploader appends under, so readFromChunks used to compute FileSize
+// and walk entry.Chunks while AddChunks reallocated the slice. Run under -race.
+func TestReadFromChunksRace(t *testing.T) {
+	wfs := &WFS{
+		option:      &Option{},
+		inodeToPath: NewInodeToPath(util.FullPath("/"), 0),
+		fhMap:       NewFileHandleToInode(),
+	}
+
+	const inode = uint64(42)
+	fullPath := util.FullPath("/dir/sample.txt")
+	wfs.inodeToPath.Lookup(fullPath, 1, false, false, inode, true)
+
+	// FileSize 0 forces readFromChunks down the filer.FileSize(chunks) branch.
+	entry := &filer_pb.Entry{
+		Name:       "sample.txt",
+		Attributes: &filer_pb.FuseAttributes{FileMode: 0644},
+	}
+	chunkGroup, err := filer.NewChunkGroup(nil, nil, nil, 1, nil, nil)
+	if err != nil {
+		t.Fatalf("NewChunkGroup: %v", err)
+	}
+	fh := &FileHandle{
+		fh:              FileHandleId(1),
+		inode:           inode,
+		wfs:             wfs,
+		entry:           &LockedEntry{Entry: entry},
+		entryChunkGroup: chunkGroup,
+	}
+	fh.dirtyPages = newPageWriter(fh, 1<<20)
+	wfs.fhMap.inode2fh[inode] = fh
+	wfs.fhMap.fh2inode[fh.fh] = inode
+
+	const iterations = 2000
+	var wg sync.WaitGroup
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		for i := 0; i < iterations; i++ {
+			fh.AddChunks([]*filer_pb.FileChunk{{FileId: "x", Offset: int64(i), Size: 1}})
+		}
+	}()
+
+	// A read past EOF returns before touching the volume tier, but only after
+	// the racy size/chunk snapshot has run.
+	go func() {
+		defer wg.Done()
+		buff := make([]byte, 16)
+		for i := 0; i < iterations; i++ {
+			fh.readFromChunks(buff, 1<<62)
+		}
+	}()
+
+	wg.Wait()
+}

weed/mount/weedfs_attr_unlinked_test.go

@@ -0,0 +1,292 @@
+package mount
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// newUnlinkedOpenFile builds a WFS holding one open handle whose name has
+// already been removed, the state a file is in between unlink() and the last
+// close() of a descriptor still pointing at it.
+func newUnlinkedOpenFile(t *testing.T) (*WFS, uint64, *FileHandle) {
+	t.Helper()
+
+	wfs := &WFS{
+		option:         &Option{ChunkSizeLimit: 1024, ConcurrentReaders: 1},
+		inodeToPath:    NewInodeToPath(util.FullPath("/"), 0),
+		fhMap:          NewFileHandleToInode(),
+		fhLockTable:    util.NewLockTable[FileHandleId](),
+		openMtimeCache: make(map[uint64][2]int64, 8),
+	}
+
+	const inode = uint64(42)
+	fullPath := util.FullPath("/dir/file")
+	wfs.inodeToPath.Lookup(fullPath, 1, false, false, inode, true)
+
+	entry := &filer_pb.Entry{
+		Name:       "file",
+		Attributes: &filer_pb.FuseAttributes{FileMode: 0644},
+	}
+	chunkGroup, err := filer.NewChunkGroup(nil, nil, nil, 1, nil, nil)
+	if err != nil {
+		t.Fatalf("NewChunkGroup: %v", err)
+	}
+	fh := &FileHandle{
+		fh:              FileHandleId(1),
+		inode:           inode,
+		wfs:             wfs,
+		entry:           &LockedEntry{Entry: entry},
+		entryChunkGroup: chunkGroup,
+	}
+	fh.dirtyPages = newPageWriter(fh, 1<<20)
+	fh.RememberPath(fullPath)
+	wfs.fhMap.inode2fh[inode] = fh
+	wfs.fhMap.fh2inode[fh.fh] = inode
+
+	wfs.inodeToPath.RemovePath(fullPath, nil)
+	fh.isDeleted = true
+
+	return wfs, inode, fh
+}
+
+// TestSetAttrOnUnlinkedOpenFile covers ftruncate/fchmod on a descriptor whose
+// file has been unlinked: POSIX keeps the open file alive, so these must not
+// fail with ENOENT.
+func TestSetAttrOnUnlinkedOpenFile(t *testing.T) {
+	wfs, inode, fh := newUnlinkedOpenFile(t)
+
+	in := &fuse.SetAttrIn{}
+	in.NodeId = inode
+	in.Valid = fuse.FATTR_SIZE
+	in.Size = 128
+	var out fuse.AttrOut
+	if status := wfs.SetAttr(nil, in, &out); status != fuse.OK {
+		t.Fatalf("SetAttr size on unlinked open file: got %v, want OK", status)
+	}
+	if out.Attr.Size != 128 {
+		t.Fatalf("SetAttr size: got %d, want 128", out.Attr.Size)
+	}
+	if got := fh.GetEntry().GetEntry().Attributes.FileSize; got != 128 {
+		t.Fatalf("handle entry FileSize: got %d, want 128", got)
+	}
+	if !fh.dirtyMetadata {
+		t.Fatal("SetAttr did not mark the handle dirty")
+	}
+	// The kernel caches what this reply carries, so it has to agree with fstat.
+	if out.Attr.Nlink != 0 {
+		t.Fatalf("SetAttr nlink: got %d, want 0", out.Attr.Nlink)
+	}
+
+	in = &fuse.SetAttrIn{}
+	in.NodeId = inode
+	in.Valid = fuse.FATTR_MODE
+	in.Mode = 0600
+	if status := wfs.SetAttr(nil, in, &out); status != fuse.OK {
+		t.Fatalf("SetAttr mode on unlinked open file: got %v, want OK", status)
+	}
+	if got := fh.GetEntry().GetEntry().Attributes.FileMode & 0777; got != 0600 {
+		t.Fatalf("handle entry FileMode: got %o, want 600", got)
+	}
+}
+
+// TestGetAttrOnUnlinkedOpenFile pins fstat on an unlinked descriptor: the size
+// stays visible and nlink drops to zero.
+func TestGetAttrOnUnlinkedOpenFile(t *testing.T) {
+	wfs, inode, fh := newUnlinkedOpenFile(t)
+	fh.GetEntry().GetEntry().Attributes.FileSize = 128
+
+	in := &fuse.GetAttrIn{}
+	in.NodeId = inode
+	var out fuse.AttrOut
+	if status := wfs.GetAttr(nil, in, &out); status != fuse.OK {
+		t.Fatalf("GetAttr on unlinked open file: got %v, want OK", status)
+	}
+	if out.Attr.Size != 128 {
+		t.Fatalf("GetAttr size: got %d, want 128", out.Attr.Size)
+	}
+	if out.Attr.Nlink != 0 {
+		t.Fatalf("GetAttr nlink: got %d, want 0", out.Attr.Nlink)
+	}
+}
+
+// TestXAttrOnUnlinkedOpenFile covers fsetxattr/fgetxattr/fremovexattr on a
+// descriptor whose file has been unlinked.
+func TestXAttrOnUnlinkedOpenFile(t *testing.T) {
+	wfs, inode, _ := newUnlinkedOpenFile(t)
+
+	setIn := &fuse.SetXAttrIn{}
+	setIn.NodeId = inode
+	if status := wfs.SetXAttr(nil, setIn, "user.k", []byte("v")); status != fuse.OK {
+		t.Fatalf("SetXAttr on unlinked open file: got %v, want OK", status)
+	}
+
+	header := &fuse.InHeader{NodeId: inode}
+	dest := make([]byte, 8)
+	n, status := wfs.GetXAttr(nil, header, "user.k", dest)
+	if status != fuse.OK {
+		t.Fatalf("GetXAttr on unlinked open file: got %v, want OK", status)
+	}
+	if string(dest[:n]) != "v" {
+		t.Fatalf("GetXAttr value: got %q, want %q", dest[:n], "v")
+	}
+
+	if status := wfs.RemoveXAttr(nil, header, "user.k"); status != fuse.OK {
+		t.Fatalf("RemoveXAttr on unlinked open file: got %v, want OK", status)
+	}
+}
+
+// newRemovedOpenDir builds a WFS holding one kernel-referenced directory whose
+// name rmdir has already dropped, the state between rmdir() and releasedir().
+func newRemovedOpenDir(t *testing.T) (*WFS, uint64) {
+	t.Helper()
+
+	wfs := &WFS{
+		option:      &Option{},
+		inodeToPath: NewInodeToPath(util.FullPath("/"), 0),
+		fhMap:       NewFileHandleToInode(),
+		fhLockTable: util.NewLockTable[FileHandleId](),
+		atimeMap:    make(map[uint64]time.Time, 8),
+		dirMtimeMap: make(map[uint64]time.Time, 8),
+	}
+
+	const inode = uint64(7)
+	fullPath := util.FullPath("/dir")
+	wfs.inodeToPath.Lookup(fullPath, 1, true, false, inode, true)
+
+	kept := false
+	wfs.inodeToPath.RemovePath(fullPath, func(removedInode uint64) {
+		kept = true
+		wfs.rememberRemovedDir(removedInode, &filer_pb.Entry{
+			Name:        "dir",
+			IsDirectory: true,
+			Attributes:  &filer_pb.FuseAttributes{FileMode: uint32(os.ModeDir | 0755), Uid: 1, Gid: 2},
+		})
+	})
+	if !kept {
+		t.Fatal("RemovePath did not report the inode as still referenced")
+	}
+
+	return wfs, inode
+}
+
+// TestSetAttrOnRemovedOpenDir covers fchmod/futimens on a descriptor whose
+// directory has been removed: the inode lives on until the final forget, so
+// these must not fail with ENOENT, and the change must be visible to a
+// following fstat.
+func TestSetAttrOnRemovedOpenDir(t *testing.T) {
+	wfs, inode := newRemovedOpenDir(t)
+
+	in := &fuse.SetAttrIn{}
+	in.NodeId = inode
+	in.Valid = fuse.FATTR_MODE
+	in.Mode = 0770
+	var out fuse.AttrOut
+	if status := wfs.SetAttr(nil, in, &out); status != fuse.OK {
+		t.Fatalf("SetAttr mode on removed open dir: got %v, want OK", status)
+	}
+	if out.Attr.Mode&0777 != 0770 {
+		t.Fatalf("SetAttr mode: got %o, want 770", out.Attr.Mode&0777)
+	}
+	if out.Attr.Nlink != 0 {
+		t.Fatalf("SetAttr nlink: got %d, want 0", out.Attr.Nlink)
+	}
+
+	in = &fuse.SetAttrIn{}
+	in.NodeId = inode
+	in.Valid = fuse.FATTR_MTIME
+	in.Mtime = 12345
+	if status := wfs.SetAttr(nil, in, &out); status != fuse.OK {
+		t.Fatalf("SetAttr mtime on removed open dir: got %v, want OK", status)
+	}
+
+	gin := &fuse.GetAttrIn{}
+	gin.NodeId = inode
+	if status := wfs.GetAttr(nil, gin, &out); status != fuse.OK {
+		t.Fatalf("GetAttr on removed open dir: got %v, want OK", status)
+	}
+	if out.Attr.Mode&0777 != 0770 {
+		t.Fatalf("GetAttr mode after chmod: got %o, want 770", out.Attr.Mode&0777)
+	}
+	if out.Attr.Mode&fuse.S_IFDIR == 0 {
+		t.Fatalf("GetAttr lost the directory type: %o", out.Attr.Mode)
+	}
+	if out.Attr.Mtime != 12345 {
+		t.Fatalf("GetAttr mtime after utimens: got %d, want 12345", out.Attr.Mtime)
+	}
+	if out.Attr.Uid != 1 || out.Attr.Gid != 2 {
+		t.Fatalf("GetAttr uid/gid: got %d/%d, want 1/2", out.Attr.Uid, out.Attr.Gid)
+	}
+	if out.Attr.Nlink != 0 {
+		t.Fatalf("GetAttr nlink: got %d, want 0", out.Attr.Nlink)
+	}
+}
+
+// TestXAttrOnRemovedOpenDir covers fsetxattr/fgetxattr/fremovexattr on a
+// descriptor whose directory has been removed.
+func TestXAttrOnRemovedOpenDir(t *testing.T) {
+	wfs, inode := newRemovedOpenDir(t)
+
+	setIn := &fuse.SetXAttrIn{}
+	setIn.NodeId = inode
+	if status := wfs.SetXAttr(nil, setIn, "user.k", []byte("v")); status != fuse.OK {
+		t.Fatalf("SetXAttr on removed open dir: got %v, want OK", status)
+	}
+
+	header := &fuse.InHeader{NodeId: inode}
+	dest := make([]byte, 8)
+	n, status := wfs.GetXAttr(nil, header, "user.k", dest)
+	if status != fuse.OK {
+		t.Fatalf("GetXAttr on removed open dir: got %v, want OK", status)
+	}
+	if string(dest[:n]) != "v" {
+		t.Fatalf("GetXAttr value: got %q, want %q", dest[:n], "v")
+	}
+
+	if status := wfs.RemoveXAttr(nil, header, "user.k"); status != fuse.OK {
+		t.Fatalf("RemoveXAttr on removed open dir: got %v, want OK", status)
+	}
+	if _, status := wfs.GetXAttr(nil, header, "user.k", dest); status != fuse.ENOATTR {
+		t.Fatalf("GetXAttr after remove: got %v, want ENOATTR", status)
+	}
+}
+
+// TestForgetReleasesRemovedOpenDir pins the cleanup: once the kernel drops its
+// last reference, the remembered entry goes with it.
+func TestForgetReleasesRemovedOpenDir(t *testing.T) {
+	wfs, inode := newRemovedOpenDir(t)
+
+	wfs.Forget(inode, 1)
+
+	in := &fuse.SetAttrIn{}
+	in.NodeId = inode
+	in.Valid = fuse.FATTR_MODE
+	in.Mode = 0700
+	var out fuse.AttrOut
+	if status := wfs.SetAttr(nil, in, &out); status != fuse.ENOENT {
+		t.Fatalf("SetAttr after forget: got %v, want ENOENT", status)
+	}
+	if len(wfs.removedDirs) != 0 {
+		t.Fatalf("removedDirs not cleaned up: %d entries", len(wfs.removedDirs))
+	}
+}
+
+// TestRemovePathSkipsUnreferencedInode pins the insert-forget ordering: once
+// the kernel's references are gone, RemovePath must not offer the inode for
+// retention, so nothing can be inserted that the forget's cleanup missed.
+func TestRemovePathSkipsUnreferencedInode(t *testing.T) {
+	itp := NewInodeToPath(util.FullPath("/"), 0)
+	fullPath := util.FullPath("/dir")
+	itp.Lookup(fullPath, 1, true, false, 7, true)
+	itp.Forget(7, 1, nil, nil)
+
+	itp.Lookup(fullPath, 1, true, false, 7, false)
+	itp.RemovePath(fullPath, func(inode uint64) {
+		t.Fatalf("RemovePath offered unreferenced inode %d for retention", inode)
+	})
+}

weed/mount/weedfs_attr_windows.go

@@ -0,0 +1,8 @@
+package mount
+
+import (
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+)
+
+func setBlksize(out *fuse.Attr, size uint32) {
+}

weed/mount/weedfs_chunk_flusher.go

@@ -0,0 +1,48 @@
+package mount
+
+import (
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+const (
+	proactiveFlushInterval  = 200 * time.Millisecond
+	proactiveIdleThreshold  = 500 * time.Millisecond
+	proactiveMaxHoldTime    = 5 * time.Second
+	proactiveFillRatioNumer = 9
+	proactiveFillRatioDenom = 10
+	proactiveFrontierLag    = 2
+)
+
+func (wfs *WFS) loopProactiveFlush() {
+	ticker := time.NewTicker(proactiveFlushInterval)
+	defer ticker.Stop()
+
+	glog.V(0).Infof("proactive chunk flusher started (idle=%v maxHold=%v)", proactiveIdleThreshold, proactiveMaxHoldTime)
+
+	for range ticker.C {
+		wfs.proactiveFlushOnce()
+	}
+}
+
+func (wfs *WFS) proactiveFlushOnce() {
+	nowNs := time.Now().UnixNano()
+	idleNs := proactiveIdleThreshold.Nanoseconds()
+	maxHoldNs := proactiveMaxHoldTime.Nanoseconds()
+	fillRatio := wfs.option.ChunkSizeLimit * proactiveFillRatioNumer / proactiveFillRatioDenom
+
+	var handles []*FileHandle
+	wfs.fhMap.RLock()
+	for _, fh := range wfs.fhMap.inode2fh {
+		if fh != nil && fh.dirtyPages != nil {
+			handles = append(handles, fh)
+		}
+	}
+	wfs.fhMap.RUnlock()
+
+	for _, fh := range handles {
+		isSeq := fh.dirtyPages.writerPattern.IsSequentialMode()
+		fh.dirtyPages.ProactiveFlush(nowNs, idleNs, maxHoldNs, fillRatio, proactiveFrontierLag, isSeq)
+	}
+}

weed/mount/weedfs_dir_lookup.go

@@ -0,0 +1,57 @@
+package mount
+
+import (
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+// Lookup is called by the kernel when the VFS wants to know
+// about a file inside a directory. Many lookup calls can
+// occur in parallel, but only one call happens for each (dir,
+// name) pair.
+
+func (wfs *WFS) Lookup(cancel <-chan struct{}, header *fuse.InHeader, name string, out *fuse.EntryOut) (code fuse.Status) {
+
+	var s fuse.Status
+	if name, s = checkName(name); s != fuse.OK {
+		return s
+	}
+
+	dirPath, code := wfs.inodeToPath.GetPath(header.NodeId)
+	if code != fuse.OK {
+		return
+	}
+
+	fullFilePath := dirPath.Child(name)
+
+	// Use shared lookup logic that checks cache first, then filer if needed
+	localEntry, _, status := wfs.lookupEntry(fullFilePath)
+	if status != fuse.OK {
+		return status
+	}
+
+	inode := wfs.inodeToPath.Lookup(fullFilePath, localEntry.Crtime.Unix(), localEntry.IsDirectory(), len(localEntry.HardLinkId) > 0, localEntry.Inode, true)
+
+	if fh, found := wfs.fhMap.FindFileHandle(inode); found {
+		fh.entryLock.RLock()
+		if entry := fh.GetEntry().GetEntry(); entry != nil {
+			glog.V(4).Infof("lookup opened file %s size %d", dirPath.Child(localEntry.Name()), filer.FileSize(entry))
+			localEntry = filer.FromPbEntry(string(dirPath), entry)
+		}
+		fh.entryLock.RUnlock()
+	}
+
+	wfs.outputFilerEntry(out, inode, localEntry)
+
+	if localEntry.IsDirectory() {
+		wfs.applyInMemoryDirMtime(&out.Attr, inode)
+		if wfs.option.PosixDirNlink {
+			wfs.applyDirNlink(&out.Attr, fullFilePath)
+		}
+	}
+
+	return fuse.OK
+
+}

weed/mount/weedfs_dir_lookup_cache_test.go

@@ -0,0 +1,333 @@
+package mount
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/metadata"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+
+	"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func newLookupCacheTestWFS(t *testing.T, ttlSec int) *WFS {
+	t.Helper()
+	uidGidMapper, err := meta_cache.NewUidGidMapper("", "")
+	if err != nil {
+		t.Fatalf("create uid/gid mapper: %v", err)
+	}
+	root := util.FullPath("/")
+	wfs := &WFS{
+		signature:         1,
+		inodeToPath:       NewInodeToPath(root, ttlSec),
+		fhMap:             NewFileHandleToInode(),
+		fhLockTable:       util.NewLockTable[FileHandleId](),
+		hardLinkLockTable: util.NewLockTable[string](),
+		option: &Option{
+			ChunkSizeLimit:     1024,
+			ConcurrentReaders:  1,
+			VolumeServerAccess: "filerProxy",
+			FilerAddresses: []pb.ServerAddress{
+				pb.NewServerAddressWithGrpcPort("127.0.0.1:1", 1),
+			},
+			GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+			UidGidMapper:   uidGidMapper,
+		},
+	}
+	wfs.metaCache = meta_cache.NewMetaCache(
+		filepath.Join(t.TempDir(), "meta"),
+		uidGidMapper, root, false,
+		func(path util.FullPath) { wfs.inodeToPath.MarkChildrenCached(path) },
+		func(path util.FullPath) bool { return wfs.inodeToPath.IsChildrenCached(path) },
+		wfs.onEntryInvalidation, nil,
+	)
+	t.Cleanup(wfs.metaCache.Shutdown)
+	return wfs
+}
+
+type lookupCacheTestFiler struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+	entries      []*filer_pb.Entry
+	snapshotTsNs int64
+	listErr      error
+	listCalls    atomic.Int32
+	lookupCalls  atomic.Int32
+	mu           sync.Mutex
+	listGate     chan struct{}
+	startedCh    chan struct{}
+	startedOnce  sync.Once
+}
+
+func (s *lookupCacheTestFiler) setListGate(gate, started chan struct{}) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.listGate = gate
+	s.startedCh = started
+	s.startedOnce = sync.Once{}
+}
+
+func (s *lookupCacheTestFiler) signalStarted() {
+	s.startedOnce.Do(func() {
+		if s.startedCh != nil {
+			close(s.startedCh)
+		}
+	})
+}
+
+func (s *lookupCacheTestFiler) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
+	s.lookupCalls.Add(1)
+	return &filer_pb.LookupDirectoryEntryResponse{
+		Entry: &filer_pb.Entry{
+			Name:       req.Name,
+			Attributes: &filer_pb.FuseAttributes{FileSize: 123, FileMode: 0100644},
+		},
+	}, nil
+}
+
+func (s *lookupCacheTestFiler) ListEntries(req *filer_pb.ListEntriesRequest, stream filer_pb.SeaweedFiler_ListEntriesServer) error {
+	s.listCalls.Add(1)
+	s.mu.Lock()
+	gate := s.listGate
+	s.mu.Unlock()
+	if gate != nil {
+		s.signalStarted()
+		<-gate
+	}
+	if s.listErr != nil {
+		return s.listErr
+	}
+	for _, e := range s.entries {
+		if err := stream.Send(&filer_pb.ListEntriesResponse{Entry: e}); err != nil {
+			return err
+		}
+	}
+	if s.snapshotTsNs != 0 {
+		stream.SetTrailer(metadata.Pairs(filer_pb.ListSnapshotTsNsTrailerKey, strconv.FormatInt(s.snapshotTsNs, 10)))
+	}
+	return nil
+}
+
+// expireDirCache marks a cached directory's TTL as elapsed without invalidating
+// it, reproducing the state the kernel leaves the mount in when it still holds
+// the directory listing (FOPEN_CACHE_DIR | FOPEN_KEEP_CACHE) past cacheMetaTtlSec.
+func expireDirCache(t *testing.T, itp *InodeToPath, dir util.FullPath) {
+	t.Helper()
+	itp.Lock()
+	defer itp.Unlock()
+	d := itp.dirStateOf(dir)
+	if d == nil {
+		t.Fatalf("expireDirCache: no dirState for %s", dir)
+	}
+	if !d.isChildrenCached {
+		t.Fatalf("expireDirCache: %s not cached", dir)
+	}
+	d.cachedExpiresTime = time.Now().Add(-time.Second)
+}
+
+// TestLookupRebuildsExpiredDirectoryCache reproduces issue #11262: after the
+// metadata cache TTL elapses the kernel can still serve directory listings
+// from its page cache, so ReadDir never runs and EnsureVisited is not called.
+// Metadata-heavy listings (e.g. `ls --color`) then reach Lookup, which must
+// rebuild the directory cache once instead of issuing one LookupEntry RPC
+// per entry.
+func TestLookupRebuildsExpiredDirectoryCache(t *testing.T) {
+	wfs := newLookupCacheTestWFS(t, 60)
+	dir := util.FullPath("/dir")
+	wfs.inodeToPath.Lookup(dir, time.Now().Unix(), true, false, 0, true)
+
+	fake := &lookupCacheTestFiler{
+		entries: []*filer_pb.Entry{
+			{Name: "a", Attributes: &filer_pb.FuseAttributes{FileSize: 1, FileMode: 0100644}},
+			{Name: "b", Attributes: &filer_pb.FuseAttributes{FileSize: 2, FileMode: 0100644}},
+			{Name: "c", Attributes: &filer_pb.FuseAttributes{FileSize: 3, FileMode: 0100644}},
+		},
+		snapshotTsNs: 5000,
+	}
+	startFakeFiler(t, wfs, fake)
+
+	if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dir, 0); err != nil {
+		t.Fatalf("EnsureVisited: %v", err)
+	}
+	if !wfs.inodeToPath.IsChildrenCached(dir) {
+		t.Fatal("directory should be cached after EnsureVisited")
+	}
+
+	expireDirCache(t, wfs.inodeToPath, dir)
+	if wfs.inodeToPath.IsChildrenCached(dir) {
+		t.Fatal("directory cache should be expired")
+	}
+
+	listCallsBefore := fake.listCalls.Load()
+	lookupCallsBefore := fake.lookupCalls.Load()
+
+	for _, name := range []string{"a", "b", "c"} {
+		entry, _, status := wfs.lookupEntry(dir.Child(name))
+		if status != fuse.OK {
+			t.Fatalf("lookupEntry %s: %v", name, status)
+		}
+		if entry == nil || entry.Name() != name {
+			t.Fatalf("lookupEntry %s: got %v", name, entry)
+		}
+	}
+
+	listCalls := fake.listCalls.Load() - listCallsBefore
+	lookupCalls := fake.lookupCalls.Load() - lookupCallsBefore
+	if lookupCalls != 0 {
+		t.Errorf("LookupDirectoryEntry RPCs = %d, want 0 (expired dir cache should be rebuilt, not read per-entry)", lookupCalls)
+	}
+	if listCalls != 1 {
+		t.Errorf("ListEntries RPCs = %d, want 1 (expired dir cache should be rebuilt once)", listCalls)
+	}
+}
+
+// TestLookupRebuildDeduplicatesConcurrentRebuilds exercises the singleflight in
+// EnsureVisited: concurrent lookups into the same expired directory share one
+// rebuild rather than each issuing its own ListEntries.
+func TestLookupRebuildDeduplicatesConcurrentRebuilds(t *testing.T) {
+	wfs := newLookupCacheTestWFS(t, 60)
+	dir := util.FullPath("/dir")
+	wfs.inodeToPath.Lookup(dir, time.Now().Unix(), true, false, 0, true)
+
+	fake := &lookupCacheTestFiler{
+		entries: []*filer_pb.Entry{
+			{Name: "a", Attributes: &filer_pb.FuseAttributes{FileSize: 1, FileMode: 0100644}},
+			{Name: "b", Attributes: &filer_pb.FuseAttributes{FileSize: 2, FileMode: 0100644}},
+			{Name: "c", Attributes: &filer_pb.FuseAttributes{FileSize: 3, FileMode: 0100644}},
+		},
+		snapshotTsNs: 5000,
+	}
+	startFakeFiler(t, wfs, fake)
+
+	if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dir, 0); err != nil {
+		t.Fatalf("EnsureVisited: %v", err)
+	}
+	expireDirCache(t, wfs.inodeToPath, dir)
+
+	started := make(chan struct{})
+	gate := make(chan struct{})
+	fake.setListGate(gate, started)
+
+	listCallsBefore := fake.listCalls.Load()
+	names := []string{"a", "b", "c"}
+	var wg sync.WaitGroup
+	errs := make([]error, len(names))
+	for i, name := range names {
+		wg.Add(1)
+		go func(i int, name string) {
+			defer wg.Done()
+			if _, _, status := wfs.lookupEntry(dir.Child(name)); status != fuse.OK {
+				errs[i] = fmt.Errorf("lookupEntry %s: %v", name, status)
+			}
+		}(i, name)
+	}
+
+	<-started
+	// Hold the rebuild so the other lookups reach the singleflight while it
+	// is in flight, exercising the deduplication rather than serializing.
+	time.Sleep(50 * time.Millisecond)
+	close(gate)
+	wg.Wait()
+
+	for _, err := range errs {
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	if got := fake.listCalls.Load() - listCallsBefore; got != 1 {
+		t.Errorf("rebuild ListEntries RPCs = %d, want 1 (concurrent lookups should share one rebuild)", got)
+	}
+	if got := fake.lookupCalls.Load(); got != 0 {
+		t.Errorf("LookupDirectoryEntry RPCs = %d, want 0", got)
+	}
+}
+
+// TestLookupRebuildCooldownAfterFailure guards the failure path: a failed
+// rebuild records the attempt so an immediate retry is suppressed (per-entry
+// RPC fallback), and once the cooldown elapses and the filer recovers a later
+// lookup rebuilds the cache again.
+func TestLookupRebuildCooldownAfterFailure(t *testing.T) {
+	wfs := newLookupCacheTestWFS(t, 60)
+	dir := util.FullPath("/dir")
+	wfs.inodeToPath.Lookup(dir, time.Now().Unix(), true, false, 0, true)
+
+	fake := &lookupCacheTestFiler{
+		entries: []*filer_pb.Entry{
+			{Name: "a", Attributes: &filer_pb.FuseAttributes{FileSize: 1, FileMode: 0100644}},
+		},
+		snapshotTsNs: 5000,
+	}
+	startFakeFiler(t, wfs, fake)
+
+	if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dir, 0); err != nil {
+		t.Fatalf("EnsureVisited: %v", err)
+	}
+	expireDirCache(t, wfs.inodeToPath, dir)
+
+	// First lookup: rebuild fails, falls through to the per-entry RPC.
+	fake.listErr = errors.New("persistent listing failure")
+	listCallsBefore := fake.listCalls.Load()
+	lookupCallsBefore := fake.lookupCalls.Load()
+	if entry, _, status := wfs.lookupEntry(dir.Child("a")); status != fuse.OK || entry == nil || entry.Name() != "a" {
+		t.Fatalf("lookupEntry a after failed rebuild: status=%v entry=%v", status, entry)
+	}
+	if got := fake.lookupCalls.Load() - lookupCallsBefore; got != 1 {
+		t.Errorf("first lookup LookupDirectoryEntry RPCs = %d, want 1 (fall through to per-entry RPC)", got)
+	}
+	if got := fake.listCalls.Load() - listCallsBefore; got == 0 {
+		t.Error("first lookup should have attempted a rebuild")
+	}
+
+	// Immediate retry: cooldown suppresses the rebuild, falls through again.
+	listCallsBefore = fake.listCalls.Load()
+	lookupCallsBefore = fake.lookupCalls.Load()
+	if _, _, status := wfs.lookupEntry(dir.Child("a")); status != fuse.OK {
+		t.Fatalf("second lookupEntry a: %v", status)
+	}
+	if got := fake.listCalls.Load() - listCallsBefore; got != 0 {
+		t.Errorf("second lookup ListEntries RPCs = %d, want 0 (cooldown suppresses rebuild)", got)
+	}
+	if got := fake.lookupCalls.Load() - lookupCallsBefore; got != 1 {
+		t.Errorf("second lookup LookupDirectoryEntry RPCs = %d, want 1", got)
+	}
+
+	// Filer recovers and the cooldown elapses: the next lookup rebuilds.
+	fake.listErr = nil
+	advanceRebuildAttempt(t, wfs.inodeToPath, dir, expiredDirRebuildCooldown+time.Second)
+	listCallsBefore = fake.listCalls.Load()
+	lookupCallsBefore = fake.lookupCalls.Load()
+	if _, _, status := wfs.lookupEntry(dir.Child("a")); status != fuse.OK {
+		t.Fatalf("third lookupEntry a: %v", status)
+	}
+	if got := fake.listCalls.Load() - listCallsBefore; got != 1 {
+		t.Errorf("third lookup ListEntries RPCs = %d, want 1 (rebuild after cooldown)", got)
+	}
+	if got := fake.lookupCalls.Load() - lookupCallsBefore; got != 0 {
+		t.Errorf("third lookup LookupDirectoryEntry RPCs = %d, want 0 (served from rebuilt cache)", got)
+	}
+}
+
+// advanceRebuildAttempt moves a directory's last rebuild attempt back by d,
+// simulating the cooldown having elapsed without sleeping.
+func advanceRebuildAttempt(t *testing.T, itp *InodeToPath, dir util.FullPath, d time.Duration) {
+	t.Helper()
+	itp.Lock()
+	defer itp.Unlock()
+	state := itp.dirStateOf(dir)
+	if state == nil {
+		t.Fatalf("advanceRebuildAttempt: no dirState for %s", dir)
+	}
+	state.lastRebuildAttempt = time.Now().Add(-d)
+}

weed/mount/weedfs_dir_mkrm.go

@@ -0,0 +1,217 @@
+package mount
+
+import (
+	"context"
+	"errors"
+	"os"
+	"syscall"
+	"time"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+/** Create a directory
+ *
+ * Note that the mode argument may not have the type specification
+ * bits set, i.e. S_ISDIR(mode) can be false.  To obtain the
+ * correct directory type bits use  mode|S_IFDIR
+ * */
+func (wfs *WFS) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out *fuse.EntryOut) (code fuse.Status) {
+
+	if wfs.IsOverQuotaWithUncommitted() {
+		return fuse.Status(syscall.ENOSPC)
+	}
+
+	var s fuse.Status
+	if name, s = checkName(name); s != fuse.OK {
+		return s
+	}
+
+	now := time.Now().Unix()
+
+	dirFullPath, code := wfs.inodeToPath.GetPath(in.NodeId)
+	if code != fuse.OK {
+		return
+	}
+
+	entryFullPath := dirFullPath.Child(name)
+
+	// Pre-allocate the mount's local inode and stamp it into the create
+	// request so both the mount and the filer agree on object identity from
+	// the start. Without this, the filer assigns its own inode in CreateEntry
+	// and the cached entry then reports a different value than the one we
+	// return to the kernel here.
+	inode := wfs.inodeToPath.AllocateInode(entryFullPath, now)
+	newEntry := &filer_pb.Entry{
+		Name:        name,
+		IsDirectory: true,
+		Attributes: &filer_pb.FuseAttributes{
+			Mtime:    now,
+			Crtime:   now,
+			Ctime:    now,
+			FileMode: uint32(os.ModeDir) | in.Mode,
+			Uid:      in.Uid,
+			Gid:      in.Gid,
+			Inode:    inode,
+		},
+	}
+
+	wfs.mapPbIdFromLocalToFiler(newEntry)
+	// Defer restoring to local uid/gid AFTER the entry is sent to the filer
+	// but BEFORE outputPbEntry writes attributes to the kernel.  We restore
+	// explicitly below instead of using defer so the kernel gets local values.
+
+	request := &filer_pb.CreateEntryRequest{
+		// Defensive: dirFullPath is clean by construction for mount-originated
+		// mutations, but could carry invalid-UTF-8 bytes if metaCache was
+		// populated from a non-gRPC source (direct store write, legacy import).
+		// Sanitizing here keeps the marshal strictly per-request on the off
+		// chance invalid bytes do reach us.
+		Directory:                dirFullPath.Sanitized(),
+		Entry:                    newEntry,
+		Signatures:               []int32{wfs.signature},
+		SkipCheckParentDirectory: true,
+		// mkdir(2) is exclusive by contract: creating an existing path must
+		// fail with EEXIST. Without OExcl the filer treats a concurrent
+		// duplicate as an update and reports success to both callers; the
+		// kernel's pre-mkdir lookup only catches the duplicate when the
+		// winner's create is already visible, which a cross-node race defeats.
+		// The filer routes an OExcl create to the entry's ring owner, so its
+		// per-path lock arbitrates every creator in the cluster.
+		OExcl: true,
+	}
+
+	glog.V(1).Infof("mkdir: %v", request)
+	resp, err := wfs.streamCreateEntry(context.Background(), request)
+	if err != nil {
+		glog.V(0).Infof("mkdir %s: %v", entryFullPath, err)
+	} else {
+		event := resp.GetMetadataEvent()
+		if event == nil {
+			event = metadataCreateEvent(string(dirFullPath), newEntry)
+		}
+		if applyErr := wfs.applyLocalMetadataEvent(context.Background(), event); applyErr != nil {
+			glog.Warningf("mkdir %s: best-effort metadata apply failed: %v", entryFullPath, applyErr)
+			wfs.inodeToPath.InvalidateChildrenCache(dirFullPath)
+		}
+		wfs.inodeToPath.TouchDirectory(dirFullPath)
+		wfs.touchDirMtimeCtimeBest(dirFullPath)
+		wfs.inodeToPath.AdjustSubdirCount(dirFullPath, 1)
+	}
+
+	glog.V(3).Infof("mkdir %s: %v", entryFullPath, err)
+
+	if err != nil {
+		wfs.mapPbIdFromFilerToLocal(newEntry)
+		if errors.Is(err, filer_pb.ErrEntryAlreadyExists) {
+			// Lost a create race: the path exists on the filer but not yet in
+			// this mount's cache. Drop the parent's children cache so the next
+			// lookup fetches the winner's entry instead of waiting for the
+			// metadata subscription to deliver it.
+			wfs.inodeToPath.InvalidateChildrenCache(dirFullPath)
+			return fuse.Status(syscall.EEXIST)
+		}
+		return fuse.EIO
+	}
+
+	// Map uid/gid back to local-space before writing attributes to the
+	// kernel.  The kernel (especially macFUSE) caches these and uses them
+	// for subsequent permission checks on children.
+	wfs.mapPbIdFromFilerToLocal(newEntry)
+
+	inode = wfs.inodeToPath.Lookup(entryFullPath, newEntry.Attributes.Crtime, true, false, inode, true)
+
+	// The newly created directory is guaranteed to be empty, so mark it as
+	// cached immediately to avoid a needless filer round-trip on the first
+	// Lookup or ReadDir inside this directory.
+	wfs.inodeToPath.MarkChildrenCached(entryFullPath)
+
+	wfs.outputPbEntry(out, inode, newEntry)
+
+	return fuse.OK
+
+}
+
+/** Remove a directory */
+func (wfs *WFS) Rmdir(cancel <-chan struct{}, header *fuse.InHeader, name string) (code fuse.Status) {
+
+	if name == "." {
+		return fuse.Status(syscall.EINVAL)
+	}
+	if name == ".." {
+		return fuse.Status(syscall.ENOTEMPTY)
+	}
+
+	// Sanitize before it reaches DeleteEntryRequest.Name; see sanitizeFuseName.
+	name = sanitizeFuseName(name)
+
+	dirFullPath, code := wfs.inodeToPath.GetPath(header.NodeId)
+	if code != fuse.OK {
+		return
+	}
+	entryFullPath := dirFullPath.Child(name)
+
+	targetEntry, _, targetCode := wfs.maybeLoadEntry(entryFullPath)
+	if targetCode != fuse.OK {
+		targetEntry = nil
+	}
+
+	// POSIX: enforce sticky bit on the parent directory.
+	if dirEntry, _, dirCode := wfs.maybeLoadEntry(dirFullPath); dirCode == fuse.OK && dirEntry != nil && dirEntry.Attributes != nil {
+		targetUid := uint32(0)
+		if targetEntry != nil && targetEntry.Attributes != nil {
+			targetUid = targetEntry.Attributes.Uid
+		}
+		if code := checkStickyBit(dirEntry.Attributes.FileMode, dirEntry.Attributes.Uid, targetUid, header.Uid); code != fuse.OK {
+			return code
+		}
+	}
+
+	glog.V(3).Infof("remove directory: %v", entryFullPath)
+	deleteReq := &filer_pb.DeleteEntryRequest{
+		Directory:            string(dirFullPath),
+		Name:                 name,
+		IsDeleteData:         true,
+		IgnoreRecursiveError: true, // ignore recursion error since the OS should manage it
+		Signatures:           []int32{wfs.signature},
+	}
+	resp, err := wfs.streamDeleteEntry(context.Background(), deleteReq)
+	if err != nil {
+		glog.V(1).Infof("remove %s: %v", entryFullPath, err)
+		if filer.IsNonEmptyFolderError(err) {
+			return fuse.Status(syscall.ENOTEMPTY)
+		}
+		return fuse.ENOENT
+	}
+
+	event := metadataDeleteEvent(string(dirFullPath), name, true)
+	if resp != nil && resp.MetadataEvent != nil {
+		event = resp.MetadataEvent
+	}
+	if applyErr := wfs.applyLocalMetadataEvent(context.Background(), event); applyErr != nil {
+		glog.Warningf("rmdir %s: best-effort metadata apply failed: %v", entryFullPath, applyErr)
+		wfs.inodeToPath.InvalidateChildrenCache(dirFullPath)
+	}
+	// The filer serialized the delete against concurrent updates and returned
+	// the entry as it stood; the snapshot loaded above may predate one.
+	if oldEntry := resp.GetMetadataEvent().GetEventNotification().GetOldEntry(); oldEntry.GetAttributes() != nil {
+		targetEntry = proto.Clone(oldEntry).(*filer_pb.Entry)
+		wfs.mapPbIdFromFilerToLocal(targetEntry)
+	}
+	wfs.inodeToPath.RemovePath(entryFullPath, func(inode uint64) {
+		if targetEntry != nil {
+			wfs.rememberRemovedDir(inode, targetEntry)
+		}
+	})
+	wfs.inodeToPath.TouchDirectory(dirFullPath)
+	wfs.touchDirMtimeCtimeBest(dirFullPath)
+	wfs.inodeToPath.AdjustSubdirCount(dirFullPath, -1)
+
+	return fuse.OK
+
+}

weed/mount/weedfs_dir_read.go

@@ -0,0 +1,517 @@
+package mount
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type DirectoryHandleId uint64
+
+const (
+	directoryStreamBaseOffset = 2 // . & ..
+	batchSize                 = 1000
+)
+
+// readdirContext marks the meta cache listing as reading attributes only. A
+// readdir never looks at a chunk list, and building one per child is most of
+// the cost of decoding a wide directory.
+var readdirContext = filer_pb.WithChunksOmitted(context.Background())
+
+// DirectoryHandle represents an open directory handle.
+// It maintains state for directory listing pagination and is protected by a mutex
+// to handle concurrent readdir operations from NFS-Ganesha and other multi-threaded clients.
+type DirectoryHandle struct {
+	sync.Mutex
+	isFinished        bool
+	entryStream       []*filer.Entry
+	entryStreamOffset uint64
+	// lastListedName is how far the store itself reached, which runs ahead of
+	// the last visible entry whenever children are dropped as expired.
+	lastListedName string
+	snapshotTsNs   int64 // snapshot timestamp for consistent readdir in direct mode
+}
+
+func (dh *DirectoryHandle) reset() {
+	dh.isFinished = false
+	dh.lastListedName = ""
+	dh.snapshotTsNs = 0
+	// Nil out pointers to allow garbage collection of old entries,
+	// then reuse the slice's capacity to avoid re-allocations.
+	for i := range dh.entryStream {
+		dh.entryStream[i] = nil
+	}
+	dh.entryStream = dh.entryStream[:0]
+	dh.entryStreamOffset = directoryStreamBaseOffset
+}
+
+// dropConsumed releases the entries the client has already walked past.
+// Offsets are indexes into the stream from entryStreamOffset, so advancing the
+// two together keeps them lined up; one entry is kept back because the next
+// batch resumes from the name immediately before the offset.
+func (dh *DirectoryHandle) dropConsumed(offset uint64) {
+	if offset < dh.entryStreamOffset {
+		return
+	}
+	trim := int(offset-dh.entryStreamOffset) - 1
+	if trim <= 0 || trim > len(dh.entryStream) {
+		return
+	}
+	copy(dh.entryStream, dh.entryStream[trim:])
+	for i := len(dh.entryStream) - trim; i < len(dh.entryStream); i++ {
+		dh.entryStream[i] = nil
+	}
+	dh.entryStream = dh.entryStream[:len(dh.entryStream)-trim]
+	dh.entryStreamOffset += uint64(trim)
+}
+
+type DirectoryHandleToInode struct {
+	sync.Mutex
+	dir2inode map[DirectoryHandleId]*DirectoryHandle
+}
+
+func NewDirectoryHandleToInode() *DirectoryHandleToInode {
+	return &DirectoryHandleToInode{
+		dir2inode: make(map[DirectoryHandleId]*DirectoryHandle),
+	}
+}
+
+func (wfs *WFS) AcquireDirectoryHandle() (DirectoryHandleId, *DirectoryHandle) {
+	fh := DirectoryHandleId(util.RandomUint64())
+
+	wfs.dhMap.Lock()
+	defer wfs.dhMap.Unlock()
+	dh := &DirectoryHandle{}
+	dh.reset()
+	wfs.dhMap.dir2inode[fh] = dh
+	return fh, dh
+}
+
+func (wfs *WFS) GetDirectoryHandle(dhid DirectoryHandleId) *DirectoryHandle {
+	wfs.dhMap.Lock()
+	defer wfs.dhMap.Unlock()
+	if dh, found := wfs.dhMap.dir2inode[dhid]; found {
+		return dh
+	}
+	dh := &DirectoryHandle{}
+	dh.reset()
+	wfs.dhMap.dir2inode[dhid] = dh
+	return dh
+}
+
+func (wfs *WFS) ReleaseDirectoryHandle(dhid DirectoryHandleId) {
+	wfs.dhMap.Lock()
+	defer wfs.dhMap.Unlock()
+	delete(wfs.dhMap.dir2inode, dhid)
+}
+
+// Directory handling
+
+/** Open directory
+ *
+ * Unless the 'default_permissions' mount option is given,
+ * this method should check if opendir is permitted for this
+ * directory. Optionally opendir may also return an arbitrary
+ * filehandle in the fuse_file_info structure, which will be
+ * passed to readdir, releasedir and fsyncdir.
+ */
+func (wfs *WFS) OpenDir(cancel <-chan struct{}, input *fuse.OpenIn, out *fuse.OpenOut) (code fuse.Status) {
+	if !wfs.inodeToPath.HasInode(input.NodeId) {
+		return fuse.ENOENT
+	}
+	dhid, _ := wfs.AcquireDirectoryHandle()
+	out.Fh = uint64(dhid)
+	// Let the kernel keep the listing in the directory's page cache, so
+	// reopening the directory does not reach the mount at all. Local mutations
+	// drop that cache in the kernel; remote ones arrive through the metadata
+	// subscription, which notifies the kernel per changed directory. A kernel
+	// too old for the flag ignores it.
+	out.OpenFlags |= fuse.FOPEN_CACHE_DIR | fuse.FOPEN_KEEP_CACHE
+	return fuse.OK
+}
+
+/** Release directory
+ *
+ * If the directory has been removed after the call to opendir, the
+ * path parameter will be NULL.
+ */
+func (wfs *WFS) ReleaseDir(input *fuse.ReleaseIn) {
+	wfs.ReleaseDirectoryHandle(DirectoryHandleId(input.Fh))
+}
+
+/** Synchronize directory contents
+ *
+ * If the directory has been removed after the call to opendir, the
+ * path parameter will be NULL.
+ *
+ * If the datasync parameter is non-zero, then only the user data
+ * should be flushed, not the meta data
+ */
+func (wfs *WFS) FsyncDir(cancel <-chan struct{}, input *fuse.FsyncIn) (code fuse.Status) {
+	return fuse.OK
+}
+
+/** Read directory
+ *
+ * The filesystem may choose between two modes of operation:
+ *
+ * 1) The readdir implementation ignores the offset parameter, and
+ * passes zero to the filler function's offset.  The filler
+ * function will not return '1' (unless an error happens), so the
+ * whole directory is read in a single readdir operation.
+ *
+ * 2) The readdir implementation keeps track of the offsets of the
+ * directory entries.  It uses the offset parameter and always
+ * passes non-zero offset to the filler function.  When the buffer
+ * is full (or an error happens) the filler function will return
+ * '1'.
+ */
+func (wfs *WFS) ReadDir(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
+	return wfs.doReadDirectory(input, fuseDirEntryList{out}, false)
+}
+
+func (wfs *WFS) ReadDirPlus(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
+	return wfs.doReadDirectory(input, fuseDirEntryList{out}, true)
+}
+
+func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out DirEntrySink, isPlusMode bool) fuse.Status {
+	// Get the directory handle and lock it for the duration of this operation.
+	// This serializes concurrent readdir calls on the same handle, fixing the
+	// race condition that caused hangs with NFS-Ganesha.
+	dh := wfs.GetDirectoryHandle(DirectoryHandleId(input.Fh))
+	dh.Lock()
+	defer dh.Unlock()
+
+	if input.Offset == 0 {
+		dh.reset()
+	} else if input.Offset < dh.entryStreamOffset {
+		// Seeking back before what the handle still holds. Start the directory
+		// again rather than reporting nothing; the preload below refills up to
+		// the requested offset.
+		dh.reset()
+	} else if dh.isFinished && input.Offset >= dh.entryStreamOffset {
+		entryCurrentIndex := input.Offset - dh.entryStreamOffset
+		if uint64(len(dh.entryStream)) <= entryCurrentIndex {
+			return fuse.OK
+		}
+	}
+
+	dirPath, code := wfs.inodeToPath.GetPath(input.NodeId)
+	if code != fuse.OK {
+		return code
+	}
+	wfs.inodeToPath.TouchDirectory(dirPath)
+
+	var dirEntry fuse.DirEntry
+	// Only a reference makes a child worth entering in the inode table: without
+	// one nothing ever arrives to take the entry back out again.
+	takesLookupRef := isPlusMode && out.TakesLookupRef()
+
+	// index is the position in entryStream, used to calculate the offset for next readdir
+	processEachEntryFn := func(entry *filer.Entry, index int64) bool {
+		dirEntry.Name = entry.Name()
+		dirEntry.Mode = toSyscallMode(entry.Mode)
+		// Rebuild only for a sanitized name: that is the one a LOOKUP carries.
+		childPath := entry.FullPath
+		if !strings.HasSuffix(string(childPath), dirEntry.Name) {
+			childPath = dirPath.Child(dirEntry.Name)
+		}
+		var inode uint64
+		if takesLookupRef {
+			inode = wfs.inodeToPath.Lookup(childPath, entry.Crtime.Unix(), entry.IsDirectory(), len(entry.HardLinkId) > 0, entry.Inode, false)
+		} else {
+			inode = wfs.inodeToPath.InodeForListing(childPath, entry.Crtime.Unix(), entry.Inode)
+		}
+		dirEntry.Ino = inode
+
+		// Set Off to the next offset so client can resume from correct position
+		dirEntry.Off = dh.entryStreamOffset + uint64(index) + 1
+
+		if !isPlusMode {
+			if !out.AddEntry(dirEntry) {
+				return false
+			}
+		} else {
+			entryOut := out.AddEntryPlus(dirEntry)
+			if entryOut == nil {
+				return false
+			}
+			if fh, found := wfs.fhMap.FindFileHandle(inode); found {
+				glog.V(4).Infof("readdir opened file %s", childPath)
+				entry = filer.FromPbEntry(string(dirPath), fh.GetEntry().GetEntry())
+			}
+			wfs.outputFilerEntry(entryOut, inode, entry)
+			// Taken only once the entry is really in the sink, so one that did not
+			// fit leaves no reference behind. The fallback covers a racing Forget.
+			if takesLookupRef && !wfs.inodeToPath.IncrementNlookup(inode) {
+				wfs.inodeToPath.Lookup(childPath, entry.Crtime.Unix(), entry.IsDirectory(), len(entry.HardLinkId) > 0, entry.Inode, true)
+			}
+		}
+		return true
+	}
+
+	if input.Offset < directoryStreamBaseOffset {
+		if !isPlusMode {
+			if input.Offset == 0 {
+				out.AddEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: ".", Off: 1})
+			}
+			out.AddEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: "..", Off: 2})
+		} else {
+			if input.Offset == 0 {
+				out.AddEntryPlus(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: ".", Off: 1})
+			}
+			out.AddEntryPlus(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: "..", Off: 2})
+		}
+		input.Offset = directoryStreamBaseOffset
+	}
+
+	var lastEntryName string
+
+	// Both the cached and the direct walk page through the same stream, and a
+	// directory read through is precisely one too big to hold, so drop the
+	// consumed head before either of them appends to it.
+	dh.dropConsumed(input.Offset)
+
+	if wfs.inodeToPath.ShouldReadDirectoryDirect(dirPath) {
+		return wfs.readDirectoryDirect(input, out, dh, dirPath, processEachEntryFn)
+	}
+
+	// Read from cache first, then load next batch if needed
+	if input.Offset >= dh.entryStreamOffset {
+		// Handle case: new handle with non-zero offset but empty cache
+		// This happens when NFS-Ganesha opens multiple directory handles
+		if len(dh.entryStream) == 0 && input.Offset > dh.entryStreamOffset {
+			skipCount := int64(input.Offset - dh.entryStreamOffset)
+
+			if err := wfs.ensureDirectoryVisited(dirPath); err != nil {
+				var tooLarge *meta_cache.DirectoryTooLargeError
+				if errors.As(err, &tooLarge) {
+					return wfs.readDirectoryDirect(input, out, dh, dirPath, processEachEntryFn)
+				}
+				glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
+				return fuse.EIO
+			}
+
+			// Serve the maintained-but-unverified cache if the filer is unreachable.
+			if err := meta_cache.EnsureListingFresh(context.Background(), wfs.metaCache, wfs, dirPath, ""); err != nil {
+				if errors.Is(err, meta_cache.ErrRefreshRangeTooLarge) {
+					// re-tile with a full rebuild; serve this request direct
+					wfs.purgeDirectoryCache(dirPath)
+					return wfs.readDirectoryDirect(input, out, dh, dirPath, processEachEntryFn)
+				}
+				glog.V(1).Infof("refresh %s sections: %v", dirPath, err)
+			}
+
+			// Load entries from beginning to fill cache up to the requested offset
+			storeLastName, loadErr := wfs.metaCache.ListDirectoryEntries(readdirContext, dirPath, "", false, skipCount+int64(batchSize), func(entry *filer.Entry) (bool, error) {
+				dh.entryStream = append(dh.entryStream, entry)
+				return true, nil
+			})
+			dh.lastListedName = storeLastName
+			if loadErr != nil {
+				glog.Errorf("list meta cache: %v", loadErr)
+				return fuse.EIO
+			}
+		}
+
+		if input.Offset > dh.entryStreamOffset {
+			entryPreviousIndex := (input.Offset - dh.entryStreamOffset) - 1
+			if uint64(len(dh.entryStream)) > entryPreviousIndex {
+				lastEntryName = dh.entryStream[entryPreviousIndex].Name()
+			} else {
+				// The stream runs from the directory's first child, so failing to
+				// reach the entry before this offset means the directory has since
+				// shrunk past it. Listing on from an empty name would replay the
+				// directory from the start and hand the client every name twice.
+				dh.isFinished = true
+				return fuse.OK
+			}
+		}
+
+		entryCurrentIndex := int64(input.Offset - dh.entryStreamOffset)
+		for int64(len(dh.entryStream)) > entryCurrentIndex {
+			entry := dh.entryStream[entryCurrentIndex]
+			if processEachEntryFn(entry, entryCurrentIndex) {
+				lastEntryName = entry.Name()
+				entryCurrentIndex++
+			} else {
+				return fuse.OK
+			}
+		}
+
+		// Cache exhausted, load next batch
+		if err := wfs.ensureDirectoryVisited(dirPath); err != nil {
+			var tooLarge *meta_cache.DirectoryTooLargeError
+			if errors.As(err, &tooLarge) {
+				// The direct path keeps the same pagination state on dh, so it
+				// carries on from wherever the cached walk reached.
+				return wfs.readDirectoryDirect(input, out, dh, dirPath, processEachEntryFn)
+			}
+			glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
+			return fuse.EIO
+		}
+
+		// Page from where the store itself reached, not from the last entry the
+		// sink saw. An expired child is counted against the batch and then
+		// dropped, so resuming from the last visible name would re-read it every
+		// round and never get past a batch that was entirely expired.
+		if dh.lastListedName > lastEntryName {
+			lastEntryName = dh.lastListedName
+		}
+
+		// Serve the maintained-but-unverified cache if the filer is unreachable.
+		if err := meta_cache.EnsureListingFresh(context.Background(), wfs.metaCache, wfs, dirPath, lastEntryName); err != nil {
+			if errors.Is(err, meta_cache.ErrRefreshRangeTooLarge) {
+				// re-tile with a full rebuild; serve this request direct
+				wfs.purgeDirectoryCache(dirPath)
+				return wfs.readDirectoryDirect(input, out, dh, dirPath, processEachEntryFn)
+			}
+			glog.V(1).Infof("refresh %s sections: %v", dirPath, err)
+		}
+
+		bufferFull := false
+		storeLastName, loadErr := wfs.metaCache.ListDirectoryEntries(readdirContext, dirPath, lastEntryName, false, int64(batchSize), func(entry *filer.Entry) (bool, error) {
+			currentIndex := int64(len(dh.entryStream))
+			dh.entryStream = append(dh.entryStream, entry)
+			if !processEachEntryFn(entry, currentIndex) {
+				bufferFull = true
+				return false, nil
+			}
+			return true, nil
+		})
+		if loadErr != nil {
+			glog.Errorf("list meta cache: %v", loadErr)
+			return fuse.EIO
+		}
+		dh.lastListedName = storeLastName
+
+		// The store reaching nothing is the only sound end-of-directory signal:
+		// a batch can come back short because entries expired, not because the
+		// directory ran out.
+		if !bufferFull && storeLastName == "" {
+			dh.isFinished = true
+		}
+	}
+
+	return fuse.OK
+}
+
+// ensureDirectoryVisited pulls the directory into the local cache, unless it is
+// too large to cache: then the directory is marked read-through, so later
+// listings go straight to the filer without re-asking.
+func (wfs *WFS) ensureDirectoryVisited(dirPath util.FullPath) error {
+	err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath, wfs.option.CacheDirMaxEntries)
+	var tooLarge *meta_cache.DirectoryTooLargeError
+	if errors.As(err, &tooLarge) {
+		wfs.inodeToPath.MarkDirectoryReadThrough(dirPath, time.Now())
+	}
+	return err
+}
+
+func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out DirEntrySink, dh *DirectoryHandle, dirPath util.FullPath, processEachEntryFn func(entry *filer.Entry, index int64) bool) fuse.Status {
+	var lastEntryName string
+
+	if input.Offset >= dh.entryStreamOffset {
+		if len(dh.entryStream) == 0 && input.Offset > dh.entryStreamOffset {
+			skipCount := uint32(input.Offset-dh.entryStreamOffset) + batchSize
+			entries, snapshotTs, err := loadDirectoryEntriesDirect(readdirContext, wfs, wfs.option.UidGidMapper, dirPath, "", false, skipCount, dh.snapshotTsNs, wfs.option.IncludeSystemEntries)
+			if err != nil {
+				glog.Errorf("list filer directory: %v", err)
+				return fuse.EIO
+			}
+			dh.entryStream = append(dh.entryStream, entries...)
+			if dh.snapshotTsNs == 0 {
+				dh.snapshotTsNs = snapshotTs
+			}
+		}
+
+		if input.Offset > dh.entryStreamOffset {
+			entryPreviousIndex := (input.Offset - dh.entryStreamOffset) - 1
+			if uint64(len(dh.entryStream)) > entryPreviousIndex {
+				lastEntryName = dh.entryStream[entryPreviousIndex].Name()
+			} else {
+				// See the cached path: the directory shrank past this offset, and
+				// resuming from an empty name would replay it from the start.
+				dh.isFinished = true
+				return fuse.OK
+			}
+		}
+
+		entryCurrentIndex := int64(input.Offset - dh.entryStreamOffset)
+		for int64(len(dh.entryStream)) > entryCurrentIndex {
+			entry := dh.entryStream[entryCurrentIndex]
+			if processEachEntryFn(entry, entryCurrentIndex) {
+				lastEntryName = entry.Name()
+				entryCurrentIndex++
+			} else {
+				return fuse.OK
+			}
+		}
+
+		entries, snapshotTs, err := loadDirectoryEntriesDirect(readdirContext, wfs, wfs.option.UidGidMapper, dirPath, lastEntryName, false, batchSize, dh.snapshotTsNs, wfs.option.IncludeSystemEntries)
+		if err != nil {
+			glog.Errorf("list filer directory: %v", err)
+			return fuse.EIO
+		}
+		if dh.snapshotTsNs == 0 {
+			dh.snapshotTsNs = snapshotTs
+		}
+
+		bufferFull := false
+		for _, entry := range entries {
+			currentIndex := int64(len(dh.entryStream))
+			dh.entryStream = append(dh.entryStream, entry)
+			if !processEachEntryFn(entry, currentIndex) {
+				bufferFull = true
+				break
+			}
+		}
+		if !bufferFull && len(entries) < int(batchSize) {
+			dh.isFinished = true
+			// After a full successful read-through listing, exit direct mode
+			// so subsequent reads can use the cache instead of hitting the filer.
+			wfs.inodeToPath.MarkDirectoryRefreshed(dirPath, time.Now())
+		}
+	}
+
+	return fuse.OK
+}
+
+func loadDirectoryEntriesDirect(ctx context.Context, client filer_pb.FilerClient, uidGidMapper *meta_cache.UidGidMapper, dirPath util.FullPath, startFileName string, includeStart bool, limit uint32, snapshotTsNs int64, includeSystemEntries bool) ([]*filer.Entry, int64, error) {
+	// limit can be a client-supplied resume offset rather than a batch size, so
+	// preallocating for it would size the slice from where the caller happened to
+	// seek. Reserve a batch and let append find the rest.
+	prealloc := limit
+	if prealloc > batchSize {
+		prealloc = batchSize
+	}
+	entries := make([]*filer.Entry, 0, prealloc)
+	var actualSnapshotTsNs int64
+	err := client.WithFilerClient(false, func(sc filer_pb.SeaweedFilerClient) error {
+		var innerErr error
+		actualSnapshotTsNs, innerErr = filer_pb.DoSeaweedListWithSnapshot(ctx, sc, dirPath, "", func(entry *filer_pb.Entry, isLast bool) error {
+			if !includeSystemEntries && meta_cache.IsHiddenSystemEntry(string(dirPath), entry.Name) {
+				return nil
+			}
+			if uidGidMapper != nil && entry.Attributes != nil {
+				entry.Attributes.Uid, entry.Attributes.Gid = uidGidMapper.FilerToLocal(entry.Attributes.Uid, entry.Attributes.Gid)
+			}
+			entries = append(entries, filer.FromPbEntry(string(dirPath), entry))
+			return nil
+		}, startFileName, includeStart, limit, snapshotTsNs)
+		return innerErr
+	})
+	if err != nil {
+		return nil, actualSnapshotTsNs, err
+	}
+	return entries, actualSnapshotTsNs, nil
+}

weed/mount/weedfs_dir_read_bench_test.go

@@ -0,0 +1,233 @@
+package mount
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+const benchDirEntryCount = 200000
+
+// benchSink drains a readdir the way a front end does. sinkLimit is how many
+// entries one round accepts, standing in for the kernel reply buffer or the
+// WinFsp adapter's batch.
+type benchSink struct {
+	plus      bool
+	takesRef  bool
+	sinkLimit int
+	count     int
+	inodes    []uint64
+	lastOff   uint64
+	attrs     []fuse.EntryOut
+}
+
+func (s *benchSink) reset() {
+	s.count = 0
+	s.inodes = s.inodes[:0]
+	s.attrs = s.attrs[:0]
+}
+
+func (s *benchSink) AddEntry(entry fuse.DirEntry) bool {
+	if s.count >= s.sinkLimit {
+		return false
+	}
+	s.count++
+	s.lastOff = entry.Off
+	s.inodes = append(s.inodes, entry.Ino)
+	return true
+}
+
+func (s *benchSink) AddEntryPlus(entry fuse.DirEntry) *fuse.EntryOut {
+	if s.count >= s.sinkLimit {
+		return nil
+	}
+	s.count++
+	s.lastOff = entry.Off
+	s.inodes = append(s.inodes, entry.Ino)
+	s.attrs = append(s.attrs, fuse.EntryOut{})
+	return &s.attrs[len(s.attrs)-1]
+}
+
+func (s *benchSink) TakesLookupRef() bool { return s.takesRef }
+
+func inodeTableSize(i *InodeToPath) int {
+	i.RLock()
+	defer i.RUnlock()
+	return len(i.inode2path)
+}
+
+func newBenchWFS(tb testing.TB, dir util.FullPath, n int) *WFS {
+	tb.Helper()
+
+	uidGidMapper, err := meta_cache.NewUidGidMapper("", "")
+	if err != nil {
+		tb.Fatalf("uid/gid mapper: %v", err)
+	}
+
+	root := util.FullPath("/")
+	option := &Option{
+		ChunkSizeLimit:     1024,
+		ConcurrentReaders:  1,
+		VolumeServerAccess: "filerProxy",
+		FilerAddresses:     []pb.ServerAddress{pb.NewServerAddressWithGrpcPort("127.0.0.1:1", 1)},
+		GrpcDialOption:     grpc.WithTransportCredentials(insecure.NewCredentials()),
+		FilerMountRootPath: "/",
+		MountUid:           99,
+		MountGid:           100,
+		MountMode:          0o777,
+		MountMtime:         time.Now(),
+		MountCtime:         time.Now(),
+		UidGidMapper:       uidGidMapper,
+	}
+
+	wfs := &WFS{
+		option:            option,
+		signature:         1,
+		inodeToPath:       NewInodeToPath(root, 0),
+		fhMap:             NewFileHandleToInode(),
+		dhMap:             NewDirectoryHandleToInode(),
+		fhLockTable:       util.NewLockTable[FileHandleId](),
+		hardLinkLockTable: util.NewLockTable[string](),
+	}
+	wfs.metaCache = meta_cache.NewMetaCache(
+		filepath.Join(tb.TempDir(), "meta"),
+		uidGidMapper,
+		root,
+		false,
+		func(path util.FullPath) { wfs.inodeToPath.MarkChildrenCached(path) },
+		func(path util.FullPath) bool { return wfs.inodeToPath.IsChildrenCached(path) },
+		func(meta_cache.EntryInvalidation) {},
+		nil,
+	)
+	tb.Cleanup(wfs.metaCache.Shutdown)
+
+	now := time.Now()
+	ctx := context.Background()
+	if err := wfs.metaCache.InsertEntry(ctx, &filer.Entry{
+		FullPath: dir,
+		Attr:     filer.Attr{Mode: os.ModeDir | 0o755, Mtime: now, Crtime: now, Uid: 99, Gid: 100},
+	}, 0); err != nil {
+		tb.Fatalf("insert dir: %v", err)
+	}
+	for i := 0; i < n; i++ {
+		child := dir.Child(fmt.Sprintf("image-%08d.jpg", i))
+		if err := wfs.metaCache.InsertEntry(ctx, &filer.Entry{
+			FullPath: child,
+			// The filer stamps an inode on every entry it stores, so a listing
+			// arrives with one and never has to derive its own.
+			Attr: filer.Attr{Mode: 0o644, Mtime: now, Crtime: now, Uid: 99, Gid: 100, FileSize: 4 << 20, Inode: child.AsInode(now.Unix())},
+			// A real file has chunks, and building them is most of what decoding
+			// an entry costs.
+			// No FileId: BeforeEntrySerialization reparses that legacy string
+			// over Fid on the way in, which would make every entry's chunk
+			// byte-identical instead of varying per file.
+			Chunks: []*filer_pb.FileChunk{{
+				Size: 4 << 20, ModifiedTsNs: now.UnixNano(),
+				ETag: "1a2b3c4d5e6f7890",
+				Fid:  &filer_pb.FileId{VolumeId: 3, FileKey: uint64(i), Cookie: 0x1637037d},
+			}},
+		}, 0); err != nil {
+			tb.Fatalf("insert entry %d: %v", i, err)
+		}
+	}
+
+	// Mark the tree cached so the listing is served from the meta cache and no
+	// filer client is dialled.
+	wfs.inodeToPath.MarkChildrenCached(root)
+	wfs.inodeToPath.Lookup(dir, now.Unix(), true, false, 0, true)
+	wfs.inodeToPath.MarkChildrenCached(dir)
+
+	return wfs
+}
+
+// walkOnce enumerates the whole directory the way a front end does: repeated
+// rounds against one handle until the listing runs dry, returning whatever
+// references the round took.
+func walkOnce(tb testing.TB, wfs *WFS, dirInode uint64, sink *benchSink, forgets bool) int {
+	dhid, _ := wfs.AcquireDirectoryHandle()
+	defer wfs.ReleaseDirectoryHandle(dhid)
+
+	total := 0
+	offset := uint64(0)
+	for {
+		sink.reset()
+		status := wfs.doReadDirectory(&fuse.ReadIn{
+			InHeader: fuse.InHeader{NodeId: dirInode},
+			Fh:       uint64(dhid),
+			Offset:   offset,
+			Size:     1 << 20,
+		}, sink, sink.plus)
+		if status != fuse.OK {
+			tb.Fatalf("readdir: %v", status)
+		}
+		if sink.count == 0 {
+			return total
+		}
+		total += sink.count
+		if forgets {
+			// HasInode keeps this to the entries that really hold a reference,
+			// so a listing that took none is not charged for a bogus Forget.
+			for _, ino := range sink.inodes {
+				if wfs.inodeToPath.HasInode(ino) {
+					wfs.Forget(ino, 1)
+				}
+			}
+		}
+		if sink.lastOff <= offset {
+			return total
+		}
+		offset = sink.lastOff
+	}
+}
+
+var benchCases = []struct {
+	name string
+	plus bool
+	// ref mirrors the sink's TakesLookupRef. The kernel's reports true whatever
+	// the mode, exactly as fuseDirEntryList does; it is the mode that decides
+	// whether a reference is actually granted.
+	ref bool
+	// forgets is what the front end does after a round: the kernel returns one
+	// FORGET per readdirplus entry and none for a plain readdir, and the WinFsp
+	// adapter hands back everything a round gave it.
+	forgets bool
+}{
+	{"kernel_readdir", false, true, false},
+	{"kernel_readdirplus", true, true, true},
+	{"winfsp_readdirplus", true, false, true},
+}
+
+func BenchmarkReadDirectory(b *testing.B) {
+	dir := util.FullPath("/images")
+	for _, tc := range benchCases {
+		b.Run(tc.name, func(b *testing.B) {
+			wfs := newBenchWFS(b, dir, benchDirEntryCount)
+			dirInode, _ := wfs.inodeToPath.GetInode(dir)
+			sink := &benchSink{plus: tc.plus, takesRef: tc.ref, sinkLimit: 4096}
+
+			b.ResetTimer()
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				if got := walkOnce(b, wfs, dirInode, sink, tc.forgets); got != benchDirEntryCount+2 {
+					b.Fatalf("listed %d entries, want %d", got, benchDirEntryCount+2)
+				}
+			}
+			b.StopTimer()
+			// What the listing left in the inode table, over the root and the
+			// directory itself.
+			b.ReportMetric(float64(inodeTableSize(wfs.inodeToPath)), "inodes_left")
+		})
+	}
+}

weed/mount/weedfs_dir_read_cache_flags_test.go

@@ -0,0 +1,32 @@
+package mount
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// TestOpenDirEnablesKernelListingCache pins the reply flags: without them the
+// kernel calls back for every enumeration, and losing them would silently
+// revert repeat listings to full walks of the mount.
+func TestOpenDirEnablesKernelListingCache(t *testing.T) {
+	dir := util.FullPath("/images")
+	wfs := newBenchWFS(t, dir, 4)
+	dirInode, _ := wfs.inodeToPath.GetInode(dir)
+
+	var out fuse.OpenOut
+	if status := wfs.OpenDir(nil, &fuse.OpenIn{InHeader: fuse.InHeader{NodeId: dirInode}}, &out); status != fuse.OK {
+		t.Fatalf("OpenDir: %v", status)
+	}
+	defer wfs.ReleaseDir(&fuse.ReleaseIn{Fh: out.Fh})
+
+	for _, want := range []struct {
+		name string
+		flag uint32
+	}{{"FOPEN_CACHE_DIR", fuse.FOPEN_CACHE_DIR}, {"FOPEN_KEEP_CACHE", fuse.FOPEN_KEEP_CACHE}} {
+		if out.OpenFlags&want.flag == 0 {
+			t.Errorf("OpenDir reply lacks %s", want.name)
+		}
+	}
+}

weed/mount/weedfs_dir_read_chunks_test.go

@@ -0,0 +1,55 @@
+package mount
+
+import (
+	"context"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// TestListDirectoryEntriesOmitsChunks covers the wiring the readdir speedup
+// rests on: that the context marker actually reaches the store's decode. Every
+// other test exercises the decoder directly, so a refactor that stopped
+// threading the context would revert the optimisation silently.
+func TestListDirectoryEntriesOmitsChunks(t *testing.T) {
+	dir := util.FullPath("/images")
+	wfs := newBenchWFS(t, dir, 4)
+
+	for _, tc := range []struct {
+		name       string
+		ctx        context.Context
+		wantChunks bool
+	}{
+		{"plain listing keeps chunks", context.Background(), true},
+		{"marked listing drops chunks", filer_pb.WithChunksOmitted(context.Background()), false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			var seen int
+			_, err := wfs.metaCache.ListDirectoryEntries(tc.ctx, dir, "", false, 100, func(entry *filer.Entry) (bool, error) {
+				seen++
+				if got := len(entry.Chunks) > 0; got != tc.wantChunks {
+					t.Errorf("%s: has chunks = %v, want %v", entry.Name(), got, tc.wantChunks)
+				}
+				// The size has to survive either way, since that is what the
+				// readdir reports.
+				if entry.FileSize != 4<<20 {
+					t.Errorf("%s: FileSize = %d, want %d", entry.Name(), entry.FileSize, 4<<20)
+				}
+				return true, nil
+			})
+			if err != nil {
+				t.Fatalf("list: %v", err)
+			}
+			if seen != 4 {
+				t.Fatalf("listed %d entries, want 4", seen)
+			}
+		})
+	}
+
+	// readdirContext is what weedfs_dir_read.go actually passes.
+	if !filer_pb.ChunksOmitted(readdirContext) {
+		t.Error("readdirContext does not carry the chunks-omitted marker")
+	}
+}

weed/mount/weedfs_dir_read_pagination_test.go

@@ -0,0 +1,359 @@
+package mount
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// pagingSink collects names, accepting at most limit per round the way a kernel
+// reply buffer does.
+type pagingSink struct {
+	limit   int
+	names   []string
+	lastOff uint64
+	round   int
+}
+
+func (s *pagingSink) AddEntry(entry fuse.DirEntry) bool {
+	if s.round >= s.limit {
+		return false
+	}
+	s.round++
+	s.names = append(s.names, entry.Name)
+	s.lastOff = entry.Off
+	return true
+}
+
+func (s *pagingSink) AddEntryPlus(entry fuse.DirEntry) *fuse.EntryOut { return nil }
+func (s *pagingSink) TakesLookupRef() bool                            { return true }
+
+func newPagingWFS(tb testing.TB, dir util.FullPath, names []string, ttlSec int32) *WFS {
+	tb.Helper()
+	uidGidMapper, err := meta_cache.NewUidGidMapper("", "")
+	if err != nil {
+		tb.Fatalf("uid/gid mapper: %v", err)
+	}
+	root := util.FullPath("/")
+	wfs := &WFS{
+		option: &Option{
+			ChunkSizeLimit: 1024, ConcurrentReaders: 1, VolumeServerAccess: "filerProxy",
+			FilerAddresses:     []pb.ServerAddress{pb.NewServerAddressWithGrpcPort("127.0.0.1:1", 1)},
+			GrpcDialOption:     grpc.WithTransportCredentials(insecure.NewCredentials()),
+			FilerMountRootPath: "/", MountUid: 99, MountGid: 100, MountMode: 0o777,
+			MountMtime: time.Now(), MountCtime: time.Now(), UidGidMapper: uidGidMapper,
+		},
+		signature: 1, inodeToPath: NewInodeToPath(root, 0),
+		fhMap: NewFileHandleToInode(), dhMap: NewDirectoryHandleToInode(),
+		fhLockTable: util.NewLockTable[FileHandleId](), hardLinkLockTable: util.NewLockTable[string](),
+	}
+	wfs.metaCache = meta_cache.NewMetaCache(
+		filepath.Join(tb.TempDir(), "meta"), uidGidMapper, root, false,
+		func(p util.FullPath) { wfs.inodeToPath.MarkChildrenCached(p) },
+		func(p util.FullPath) bool { return wfs.inodeToPath.IsChildrenCached(p) },
+		func(meta_cache.EntryInvalidation) {}, nil,
+	)
+	tb.Cleanup(wfs.metaCache.Shutdown)
+
+	now := time.Now()
+	ctx := context.Background()
+	if err := wfs.metaCache.InsertEntry(ctx, &filer.Entry{
+		FullPath: dir,
+		Attr:     filer.Attr{Mode: os.ModeDir | 0o755, Mtime: now, Crtime: now},
+	}, 0); err != nil {
+		tb.Fatalf("insert dir: %v", err)
+	}
+	for _, name := range names {
+		child := dir.Child(name)
+		attr := filer.Attr{Mode: 0o644, Mtime: now, Crtime: now, FileSize: 1, Inode: child.AsInode(now.Unix())}
+		if ttlSec > 0 {
+			// Expired well in the past, so the meta cache drops it after the
+			// store has already counted it against the limit.
+			attr.TtlSec = ttlSec
+			attr.Crtime = now.Add(-time.Duration(ttlSec+60) * time.Second)
+		}
+		if err := wfs.metaCache.InsertEntry(ctx, &filer.Entry{FullPath: child, Attr: attr}, 0); err != nil {
+			tb.Fatalf("insert %s: %v", name, err)
+		}
+	}
+	wfs.inodeToPath.MarkChildrenCached(root)
+	wfs.inodeToPath.Lookup(dir, now.Unix(), true, false, 0, true)
+	wfs.inodeToPath.MarkChildrenCached(dir)
+	return wfs
+}
+
+// TestReadDirResumePastShrunkDirectory covers a client resuming a fresh handle
+// at a cookie from an earlier, larger listing. The directory has since shrunk
+// past that offset, and the readdir used to fall back to listing from the first
+// child again, handing back names the client had already consumed.
+func TestReadDirResumePastShrunkDirectory(t *testing.T) {
+	dir := util.FullPath("/d")
+	var names []string
+	for i := 0; i < 100; i++ {
+		names = append(names, fmt.Sprintf("f%03d", i))
+	}
+	wfs := newPagingWFS(t, dir, names, 0)
+	dirInode, _ := wfs.inodeToPath.GetInode(dir)
+
+	dhid, _ := wfs.AcquireDirectoryHandle()
+	defer wfs.ReleaseDirectoryHandle(dhid)
+
+	sink := &pagingSink{limit: 4096}
+	status := wfs.doReadDirectory(&fuse.ReadIn{
+		InHeader: fuse.InHeader{NodeId: dirInode},
+		Fh:       uint64(dhid),
+		Offset:   152, // past the end of a directory that now holds 100
+		Size:     1 << 20,
+	}, sink, false)
+	if status != fuse.OK {
+		t.Fatalf("readdir: %v", status)
+	}
+	if len(sink.names) != 0 {
+		t.Errorf("resuming past the end returned %d entries (first %q), want none", len(sink.names), sink.names[0])
+	}
+}
+
+// TestReadDirWithExpiredEntries covers a batch the store fills to the limit but
+// whose entries the meta cache then drops as expired. The post-filter count used
+// to be read as end-of-directory, silently hiding every later child.
+func TestReadDirWithExpiredEntries(t *testing.T) {
+	dir := util.FullPath("/d")
+	// One full store batch of entries that all expire, then live ones behind
+	// them. Nothing survives the first batch, which is the case that latched.
+	var names []string
+	for i := 0; i < batchSize; i++ {
+		names = append(names, fmt.Sprintf("a%05d", i))
+	}
+	wfs := newPagingWFS(t, dir, names, 30)
+
+	live := []string{"z001", "z002", "z003"}
+	now := time.Now()
+	for _, name := range live {
+		child := dir.Child(name)
+		if err := wfs.metaCache.InsertEntry(context.Background(), &filer.Entry{
+			FullPath: child,
+			Attr:     filer.Attr{Mode: 0o644, Mtime: now, Crtime: now, FileSize: 1, Inode: child.AsInode(now.Unix())},
+		}, 0); err != nil {
+			t.Fatalf("insert %s: %v", name, err)
+		}
+	}
+
+	dirInode, _ := wfs.inodeToPath.GetInode(dir)
+	dhid, _ := wfs.AcquireDirectoryHandle()
+	defer wfs.ReleaseDirectoryHandle(dhid)
+
+	sink := &pagingSink{limit: 4096}
+	var offset uint64
+	for round := 0; round < 20; round++ {
+		sink.round = 0
+		status := wfs.doReadDirectory(&fuse.ReadIn{
+			InHeader: fuse.InHeader{NodeId: dirInode},
+			Fh:       uint64(dhid),
+			Offset:   offset,
+			Size:     1 << 20,
+		}, sink, false)
+		if status != fuse.OK {
+			t.Fatalf("readdir: %v", status)
+		}
+		if sink.lastOff <= offset {
+			break
+		}
+		offset = sink.lastOff
+	}
+
+	var seen []string
+	for _, n := range sink.names {
+		if n != "." && n != ".." {
+			seen = append(seen, n)
+		}
+	}
+	if len(seen) != len(live) {
+		t.Fatalf("listed %d entries %v, want the %d live ones %v", len(seen), seen, len(live), live)
+	}
+}
+
+// TestReadDirTrimsConsumedEntries checks that a walk does not accumulate the
+// whole directory in the handle. Offsets index into the stream from
+// entryStreamOffset, so the two have to advance together or the listing
+// silently misaligns.
+func TestReadDirTrimsConsumedEntries(t *testing.T) {
+	dir := util.FullPath("/d")
+	const total = 5000
+	var names []string
+	for i := 0; i < total; i++ {
+		names = append(names, fmt.Sprintf("f%05d", i))
+	}
+	wfs := newPagingWFS(t, dir, names, 0)
+	dirInode, _ := wfs.inodeToPath.GetInode(dir)
+
+	dhid, dh := wfs.AcquireDirectoryHandle()
+	defer wfs.ReleaseDirectoryHandle(dhid)
+
+	// A small sink forces many rounds, which is when the stream would grow.
+	sink := &pagingSink{limit: 64}
+	var offset uint64
+	var seen []string
+	peak := 0
+	for round := 0; round < 500; round++ {
+		sink.round = 0
+		before := len(sink.names)
+		status := wfs.doReadDirectory(&fuse.ReadIn{
+			InHeader: fuse.InHeader{NodeId: dirInode},
+			Fh:       uint64(dhid),
+			Offset:   offset,
+			Size:     1 << 20,
+		}, sink, false)
+		if status != fuse.OK {
+			t.Fatalf("readdir: %v", status)
+		}
+		if n := len(dh.entryStream); n > peak {
+			peak = n
+		}
+		if len(sink.names) == before || sink.lastOff <= offset {
+			break
+		}
+		offset = sink.lastOff
+	}
+	for _, n := range sink.names {
+		if n != "." && n != ".." {
+			seen = append(seen, n)
+		}
+	}
+
+	if len(seen) != total {
+		t.Fatalf("listed %d entries, want %d", len(seen), total)
+	}
+	for i, name := range seen {
+		if want := fmt.Sprintf("f%05d", i); name != want {
+			t.Fatalf("entry %d is %q, want %q -- trimming misaligned the offsets", i, name, want)
+		}
+	}
+	// Without trimming the handle ends up holding every entry.
+	if peak >= total {
+		t.Errorf("handle held %d entries at peak, want well under the %d in the directory", peak, total)
+	}
+}
+
+// listingFilerServer serves a fixed, sorted child list the way the filer
+// paginates one.
+type listingFilerServer struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+	names []string
+}
+
+func (s *listingFilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream filer_pb.SeaweedFiler_ListEntriesServer) error {
+	var sent uint32
+	for _, name := range s.names {
+		if name < req.StartFromFileName || (name == req.StartFromFileName && !req.InclusiveStartFrom) {
+			continue
+		}
+		if req.Limit > 0 && sent >= req.Limit {
+			break
+		}
+		if err := stream.Send(&filer_pb.ListEntriesResponse{Entry: &filer_pb.Entry{
+			Name:       name,
+			Attributes: &filer_pb.FuseAttributes{FileMode: 0o644, FileSize: 1},
+		}}); err != nil {
+			return err
+		}
+		sent++
+	}
+	return nil
+}
+
+// TestReadDirDirectTrimsConsumedEntries is the same check for a directory read
+// through to the filer. That path is taken precisely for directories too big to
+// cache, so holding the whole walk in the handle is where a listing turns into
+// gigabytes of resident memory.
+func TestReadDirDirectTrimsConsumedEntries(t *testing.T) {
+	dir := util.FullPath("/d")
+	const total = 5000
+	var names []string
+	for i := 0; i < total; i++ {
+		names = append(names, fmt.Sprintf("f%05d", i))
+	}
+	wfs := newPagingWFS(t, dir, nil, 0)
+	startFakeFiler(t, wfs, &listingFilerServer{names: names})
+	dirInode, _ := wfs.inodeToPath.GetInode(dir)
+	if !wfs.inodeToPath.MarkDirectoryReadThrough(dir, time.Now()) {
+		t.Fatal("directory did not enter read-through mode")
+	}
+
+	dhid, dh := wfs.AcquireDirectoryHandle()
+	defer wfs.ReleaseDirectoryHandle(dhid)
+
+	sink := &pagingSink{limit: 64}
+	var offset uint64
+	var seen []string
+	peak := 0
+	for round := 0; round < 500; round++ {
+		sink.round = 0
+		before := len(sink.names)
+		status := wfs.doReadDirectory(&fuse.ReadIn{
+			InHeader: fuse.InHeader{NodeId: dirInode},
+			Fh:       uint64(dhid),
+			Offset:   offset,
+			Size:     1 << 20,
+		}, sink, false)
+		if status != fuse.OK {
+			t.Fatalf("readdir: %v", status)
+		}
+		if n := len(dh.entryStream); n > peak {
+			peak = n
+		}
+		if len(sink.names) == before || sink.lastOff <= offset {
+			break
+		}
+		offset = sink.lastOff
+	}
+	for _, n := range sink.names {
+		if n != "." && n != ".." {
+			seen = append(seen, n)
+		}
+	}
+
+	if len(seen) != total {
+		t.Fatalf("listed %d entries, want %d", len(seen), total)
+	}
+	for i, name := range seen {
+		if want := fmt.Sprintf("f%05d", i); name != want {
+			t.Fatalf("entry %d is %q, want %q -- trimming misaligned the offsets", i, name, want)
+		}
+	}
+	if peak >= total {
+		t.Errorf("handle held %d entries at peak, want well under the %d in the directory", peak, total)
+	}
+}
+
+// The table has to key on the sanitized name, not the stored bytes.
+func TestReadDirPlusSanitizedName(t *testing.T) {
+	dir := util.FullPath("/images")
+	const rawName = "bad\xffname.jpg"
+	wfs := newPagingWFS(t, dir, []string{"good.jpg", rawName}, 0)
+	dirInode, _ := wfs.inodeToPath.GetInode(dir)
+
+	sink := &benchSink{plus: true, takesRef: true, sinkLimit: 16}
+	if got := walkOnce(t, wfs, dirInode, sink, false); got != 4 {
+		t.Fatalf("listed %d entries, want 4 (. .. and two children)", got)
+	}
+
+	sanitized := dir.Child(util.SanitizeUTF8Name(rawName))
+	if !wfs.inodeToPath.HasPath(sanitized) {
+		t.Errorf("%s not in the inode table", sanitized)
+	}
+	if wfs.inodeToPath.HasPath(dir.Child(rawName)) {
+		t.Errorf("%s keyed on the unsanitized name", dir.Child(rawName))
+	}
+}

weed/mount/weedfs_dir_read_test.go

@@ -0,0 +1,122 @@
+package mount
+
+import (
+	"context"
+	"io"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+)
+
+type directoryListStream struct {
+	responses []*filer_pb.ListEntriesResponse
+	index     int
+}
+
+func (s *directoryListStream) Recv() (*filer_pb.ListEntriesResponse, error) {
+	if s.index >= len(s.responses) {
+		return nil, io.EOF
+	}
+	resp := s.responses[s.index]
+	s.index++
+	return resp, nil
+}
+
+func (s *directoryListStream) Header() (metadata.MD, error) { return metadata.MD{}, nil }
+func (s *directoryListStream) Trailer() metadata.MD         { return metadata.MD{} }
+func (s *directoryListStream) CloseSend() error             { return nil }
+func (s *directoryListStream) Context() context.Context     { return context.Background() }
+func (s *directoryListStream) SendMsg(any) error            { return nil }
+func (s *directoryListStream) RecvMsg(any) error            { return nil }
+
+type directoryListClient struct {
+	filer_pb.SeaweedFilerClient
+	responses []*filer_pb.ListEntriesResponse
+}
+
+func (c *directoryListClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
+	return &directoryListStream{responses: c.responses}, nil
+}
+
+type directoryFilerAccessor struct {
+	client filer_pb.SeaweedFilerClient
+}
+
+func (a *directoryFilerAccessor) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+	return fn(a.client)
+}
+
+func (a *directoryFilerAccessor) AdjustedUrl(*filer_pb.Location) string { return "" }
+func (a *directoryFilerAccessor) GetDataCenter() string                 { return "" }
+
+func TestLoadDirectoryEntriesDirectFiltersHiddenEntriesAndMapsIds(t *testing.T) {
+	mapper, err := meta_cache.NewUidGidMapper("10:1000", "20:2000")
+	if err != nil {
+		t.Fatalf("uid/gid mapper: %v", err)
+	}
+
+	client := &directoryFilerAccessor{
+		client: &directoryListClient{
+			responses: []*filer_pb.ListEntriesResponse{
+				{
+					Entry: &filer_pb.Entry{
+						Name: "topics",
+						Attributes: &filer_pb.FuseAttributes{
+							Uid: 1000,
+							Gid: 2000,
+						},
+					},
+				},
+				{
+					Entry: &filer_pb.Entry{
+						Name: "visible",
+						Attributes: &filer_pb.FuseAttributes{
+							Uid: 1000,
+							Gid: 2000,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	entries, _, err := loadDirectoryEntriesDirect(context.Background(), client, mapper, util.FullPath("/"), "", false, 10, 0, false)
+	if err != nil {
+		t.Fatalf("loadDirectoryEntriesDirect: %v", err)
+	}
+	if got := len(entries); got != 1 {
+		t.Fatalf("entry count = %d, want 1", got)
+	}
+	if entries[0].Name() != "visible" {
+		t.Fatalf("entry name = %q, want visible", entries[0].Name())
+	}
+	if entries[0].Attr.Uid != 10 || entries[0].Attr.Gid != 20 {
+		t.Fatalf("mapped uid/gid = %d/%d, want 10/20", entries[0].Attr.Uid, entries[0].Attr.Gid)
+	}
+}
+
+func TestLoadDirectoryEntriesDirectShowsSystemEntriesWhenEnabled(t *testing.T) {
+	client := &directoryFilerAccessor{
+		client: &directoryListClient{
+			responses: []*filer_pb.ListEntriesResponse{
+				{Entry: &filer_pb.Entry{Name: "topics"}},
+				{Entry: &filer_pb.Entry{Name: "visible"}},
+			},
+		},
+	}
+
+	entries, _, err := loadDirectoryEntriesDirect(context.Background(), client, nil, util.FullPath("/"), "", false, 10, 0, true)
+	if err != nil {
+		t.Fatalf("loadDirectoryEntriesDirect: %v", err)
+	}
+	if got := len(entries); got != 2 {
+		t.Fatalf("entry count = %d, want 2", got)
+	}
+	if entries[0].Name() != "topics" || entries[1].Name() != "visible" {
+		t.Fatalf("entry names = %q, %q, want topics, visible", entries[0].Name(), entries[1].Name())
+	}
+}

weed/mount/weedfs_file_copy_range.go

@@ -0,0 +1,514 @@
+package mount
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+	request_id "github.com/seaweedfs/seaweedfs/weed/util/request_id"
+)
+
+type serverSideWholeFileCopyOutcome uint8
+
+const (
+	serverSideWholeFileCopyNotCommitted serverSideWholeFileCopyOutcome = iota
+	serverSideWholeFileCopyCommitted
+	serverSideWholeFileCopyAmbiguous
+)
+
+type wholeFileServerCopyRequest struct {
+	srcPath       util.FullPath
+	dstPath       util.FullPath
+	sourceSize    int64
+	srcInode      uint64
+	srcMtime      int64
+	dstInode      uint64
+	dstMtime      int64
+	dstSize       int64
+	sourceMime    string
+	sourceMd5     []byte
+	copyRequestID string
+}
+
+// performServerSideWholeFileCopy is a package-level seam so tests can override
+// the filer call without standing up an HTTP endpoint.
+var performServerSideWholeFileCopy = func(cancel <-chan struct{}, wfs *WFS, copyRequest wholeFileServerCopyRequest) (*filer_pb.Entry, entryVersion, serverSideWholeFileCopyOutcome, error) {
+	return wfs.copyEntryViaFiler(cancel, copyRequest)
+}
+
+// filerCopyRequestTimeout bounds the mount->filer POST so a stalled copy does
+// not block copy_file_range workers indefinitely.
+const filerCopyRequestTimeout = 60 * time.Second
+
+// filerCopyReadbackTimeout gives the follow-up metadata reload a fresh deadline
+// after the filer already accepted the copy request.
+const filerCopyReadbackTimeout = 15 * time.Second
+
+// CopyFileRange copies data from one file to another from and to specified offsets.
+//
+// See https://man7.org/linux/man-pages/man2/copy_file_range.2.html
+// See https://github.com/libfuse/libfuse/commit/fe4f9428fc403fa8b99051f52d84ea5bd13f3855
+/**
+ * Copy a range of data from one file to another
+ *
+ * Niels de Vos: • libfuse: add copy_file_range() support
+ *
+ * Performs an optimized copy between two file descriptors without the
+ * additional cost of transferring data through the FUSE kernel module
+ * to user space (glibc) and then back into the FUSE filesystem again.
+ *
+ * In case this method is not implemented, applications are expected to
+ * fall back to a regular file copy.   (Some glibc versions did this
+ * emulation automatically, but the emulation has been removed from all
+ * glibc release branches.)
+ */
+func (wfs *WFS) CopyFileRange(cancel <-chan struct{}, in *fuse.CopyFileRangeIn) (written uint32, code fuse.Status) {
+	// flags must equal 0 for this syscall as of now
+	if in.Flags != 0 {
+		return 0, fuse.EINVAL
+	}
+
+	// files must exist
+	fhOut := wfs.GetHandle(FileHandleId(in.FhOut))
+	if fhOut == nil {
+		return 0, fuse.EBADF
+	}
+	fhIn := wfs.GetHandle(FileHandleId(in.FhIn))
+	if fhIn == nil {
+		return 0, fuse.EBADF
+	}
+
+	// lock source and target file handles
+	fhOutActiveLock := fhOut.wfs.fhLockTable.AcquireLock("CopyFileRange", fhOut.fh, util.ExclusiveLock)
+	defer fhOut.wfs.fhLockTable.ReleaseLock(fhOut.fh, fhOutActiveLock)
+
+	if fhOut.entry == nil {
+		return 0, fuse.ENOENT
+	}
+
+	if fhIn.fh != fhOut.fh {
+		fhInActiveLock := fhIn.wfs.fhLockTable.AcquireLock("CopyFileRange", fhIn.fh, util.SharedLock)
+		defer fhIn.wfs.fhLockTable.ReleaseLock(fhIn.fh, fhInActiveLock)
+	}
+
+	// directories are not supported
+	if fhIn.entry.IsDirectory || fhOut.entry.IsDirectory {
+		return 0, fuse.EISDIR
+	}
+
+	glog.V(4).Infof(
+		"CopyFileRange %s fhIn %d -> %s fhOut %d, [%d,%d) -> [%d,%d)",
+		fhIn.FullPath(), fhIn.fh,
+		fhOut.FullPath(), fhOut.fh,
+		in.OffIn, in.OffIn+in.Len,
+		in.OffOut, in.OffOut+in.Len,
+	)
+
+	if written, handled, status := wfs.tryServerSideWholeFileCopy(cancel, in, fhIn, fhOut); handled {
+		return written, status
+	}
+
+	// Concurrent copy operations could allocate too much memory, so we want to
+	// throttle our concurrency, scaling with the number of writers the mount
+	// was configured with.
+	if wfs.concurrentCopiersSem != nil {
+		wfs.concurrentCopiersSem <- struct{}{}
+		defer func() { <-wfs.concurrentCopiersSem }()
+	}
+
+	// We want to stream the copy operation to avoid allocating massive buffers.
+	nowUnixNano := time.Now().UnixNano()
+	totalCopied := int64(0)
+	buff := wfs.copyBufferPool.Get().([]byte)
+	defer wfs.copyBufferPool.Put(buff)
+	for {
+		// Comply with cancellation as best as we can, given that the underlying
+		// IO functions aren't cancellation-aware.
+		select {
+		case <-cancel:
+			glog.Warningf("canceled CopyFileRange for %s (copied %d)",
+				fhIn.FullPath(), totalCopied)
+			return uint32(totalCopied), fuse.EINTR
+		default: // keep going
+		}
+
+		// We can save one IO by breaking early if we already know the next read
+		// will result in zero bytes.
+		remaining := int64(in.Len) - totalCopied
+		readLen := min(remaining, int64(len(buff)))
+		if readLen == 0 {
+			break
+		}
+
+		// Perform the read
+		offsetIn := totalCopied + int64(in.OffIn)
+		numBytesRead, err := readDataByFileHandle(
+			buff[:readLen], fhIn, offsetIn)
+		if err != nil {
+			glog.Warningf("file handle read %s %d (total %d): %v",
+				fhIn.FullPath(), numBytesRead, totalCopied, err)
+			return 0, fuse.EIO
+		}
+
+		// Break if we're done copying (no more bytes to read)
+		if numBytesRead == 0 {
+			break
+		}
+
+		offsetOut := int64(in.OffOut) + totalCopied
+
+		// Detect mime type only during the beginning of our stream, since
+		// DetectContentType is expecting some of the first 512 bytes of the
+		// file. See [http.DetectContentType] for details.
+		if offsetOut <= 512 {
+			fhOut.contentType = http.DetectContentType(buff[:numBytesRead])
+		}
+
+		// Perform the write
+		fhOut.dirtyPages.writerPattern.MonitorWriteAt(offsetOut, int(numBytesRead))
+		if err := fhOut.dirtyPages.AddPage(
+			offsetOut,
+			buff[:numBytesRead],
+			fhOut.dirtyPages.writerPattern.IsSequentialMode(),
+			nowUnixNano); err != nil {
+			glog.Errorf("AddPage error: %v", err)
+			return 0, writeErrorToFuseStatus(err)
+		}
+
+		// Accumulate for the next loop iteration
+		totalCopied += numBytesRead
+	}
+
+	if totalCopied == 0 {
+		return 0, fuse.OK
+	}
+
+	fhOut.entry.Attributes.FileSize = uint64(max(
+		totalCopied+int64(in.OffOut),
+		int64(fhOut.entry.Attributes.FileSize),
+	))
+	fhOut.entry.Content = nil
+	fhOut.dirtyMetadata = true
+
+	written = uint32(totalCopied)
+	return written, fuse.OK
+}
+
+func (wfs *WFS) tryServerSideWholeFileCopy(cancel <-chan struct{}, in *fuse.CopyFileRangeIn, fhIn, fhOut *FileHandle) (written uint32, handled bool, code fuse.Status) {
+	copyRequest, ok := wholeFileServerCopyCandidate(fhIn, fhOut, in)
+	if !ok {
+		return 0, false, fuse.OK
+	}
+
+	glog.V(1).Infof("CopyFileRange server-side copy %s => %s (%d bytes)", copyRequest.srcPath, copyRequest.dstPath, copyRequest.sourceSize)
+
+	entry, entryVersionTsNs, outcome, err := performServerSideWholeFileCopy(cancel, wfs, copyRequest)
+	switch outcome {
+	case serverSideWholeFileCopyCommitted:
+		if err != nil {
+			glog.Warningf("CopyFileRange server-side copy %s => %s committed but local refresh failed: %v", copyRequest.srcPath, copyRequest.dstPath, err)
+		} else {
+			glog.V(1).Infof("CopyFileRange server-side copy %s => %s completed (%d bytes)", copyRequest.srcPath, copyRequest.dstPath, copyRequest.sourceSize)
+		}
+		wfs.applyServerSideWholeFileCopyResult(fhIn, fhOut, copyRequest.dstPath, entry, entryVersionTsNs, copyRequest.sourceSize)
+		return uint32(copyRequest.sourceSize), true, fuse.OK
+	case serverSideWholeFileCopyAmbiguous:
+		glog.Warningf("CopyFileRange server-side copy %s => %s outcome ambiguous: %v", copyRequest.srcPath, copyRequest.dstPath, err)
+		return 0, true, fuse.EIO
+	default:
+		glog.V(0).Infof("CopyFileRange server-side copy %s => %s fallback to chunk copy: %v", copyRequest.srcPath, copyRequest.dstPath, err)
+		return 0, false, fuse.OK
+	}
+}
+
+func (wfs *WFS) applyServerSideWholeFileCopyResult(fhIn, fhOut *FileHandle, dstPath util.FullPath, entry *filer_pb.Entry, entryVersionTsNs entryVersion, sourceSize int64) {
+	if entry == nil {
+		entry = synthesizeLocalEntryForServerSideWholeFileCopy(fhIn, fhOut, sourceSize)
+		entryVersionTsNs = entryVersion{}
+	}
+	if entry == nil {
+		glog.Warningf("CopyFileRange server-side copy %s left no local entry to apply", dstPath)
+		return
+	}
+
+	fhOut.SetEntry(entry)
+	// Enroll the install in the versioned-base protocol: the copy's own
+	// event must read as a no-op against this base, not as a foreign change
+	// that destroys writes made to the destination after the copy.
+	fhOut.baseEntry.Store(proto.Clone(entry).(*filer_pb.Entry))
+	fhOut.advanceEntryVersion(entryVersionTsNs.tsNs, entryVersionTsNs.signature)
+	fhOut.RememberPath(dstPath)
+	if entry.Attributes != nil {
+		fhOut.contentType = entry.Attributes.Mime
+	}
+	fhOut.dirtyMetadata = false
+	wfs.updateServerSideWholeFileCopyMetaCache(dstPath, entry)
+	// Note: we intentionally skip fuseServer.InodeNotify/EntryNotify here.
+	// This runs inside the CopyFileRange request handler; those notifies
+	// would write onto the same /dev/fuse fd the kernel is still waiting
+	// on for this request's reply, deadlocking the mount. The kernel will
+	// re-read attrs once its attr-cache TTL expires.
+}
+
+func (wfs *WFS) updateServerSideWholeFileCopyMetaCache(dstPath util.FullPath, entry *filer_pb.Entry) {
+	if wfs.metaCache == nil || entry == nil {
+		return
+	}
+
+	dir, _ := dstPath.DirAndName()
+	event := metadataUpdateEvent(dir, entry)
+	if applyErr := wfs.applyLocalMetadataEvent(context.Background(), event); applyErr != nil {
+		glog.Warningf("CopyFileRange metadata update %s: %v", dstPath, applyErr)
+		wfs.purgeDirectoryCache(util.FullPath(dir))
+	}
+}
+
+func synthesizeLocalEntryForServerSideWholeFileCopy(fhIn, fhOut *FileHandle, sourceSize int64) *filer_pb.Entry {
+	dstEntry := fhOut.GetEntry().GetEntry()
+	if dstEntry == nil {
+		return nil
+	}
+
+	localEntry := proto.Clone(dstEntry).(*filer_pb.Entry)
+	if localEntry.Attributes == nil {
+		localEntry.Attributes = &filer_pb.FuseAttributes{}
+	}
+
+	if srcEntry := fhIn.GetEntry().GetEntry(); srcEntry != nil {
+		srcEntryCopy := proto.Clone(srcEntry).(*filer_pb.Entry)
+		localEntry.Content = srcEntryCopy.Content
+		localEntry.Chunks = srcEntryCopy.Chunks
+		if srcEntryCopy.Attributes != nil {
+			localEntry.Attributes.Mime = srcEntryCopy.Attributes.Mime
+			localEntry.Attributes.Md5 = srcEntryCopy.Attributes.Md5
+		}
+	}
+
+	copyNow := time.Now()
+	localEntry.Attributes.FileSize = uint64(sourceSize)
+	localEntry.Attributes.Mtime = copyNow.Unix()
+	localEntry.Attributes.MtimeNs = int32(copyNow.Nanosecond())
+	localEntry.Attributes.Ctime = copyNow.Unix()
+	localEntry.Attributes.CtimeNs = int32(copyNow.Nanosecond())
+	return localEntry
+}
+
+func wholeFileServerCopyCandidate(fhIn, fhOut *FileHandle, in *fuse.CopyFileRangeIn) (copyRequest wholeFileServerCopyRequest, ok bool) {
+	if fhIn == nil || fhOut == nil || in == nil {
+		glog.V(4).Infof("server-side copy: skipped (nil handle or input)")
+		return wholeFileServerCopyRequest{}, false
+	}
+	if fhIn.fh == fhOut.fh {
+		glog.V(4).Infof("server-side copy: skipped (same file handle)")
+		return wholeFileServerCopyRequest{}, false
+	}
+	if fhIn.dirtyMetadata || fhOut.dirtyMetadata {
+		glog.V(4).Infof("server-side copy: skipped (dirty metadata: in=%v out=%v)", fhIn.dirtyMetadata, fhOut.dirtyMetadata)
+		return wholeFileServerCopyRequest{}, false
+	}
+	if in.OffIn != 0 || in.OffOut != 0 {
+		glog.V(4).Infof("server-side copy: skipped (non-zero offsets: in=%d out=%d)", in.OffIn, in.OffOut)
+		return wholeFileServerCopyRequest{}, false
+	}
+
+	srcEntry := fhIn.GetEntry()
+	dstEntry := fhOut.GetEntry()
+	if srcEntry == nil || dstEntry == nil {
+		glog.V(4).Infof("server-side copy: skipped (nil entry: src=%v dst=%v)", srcEntry == nil, dstEntry == nil)
+		return wholeFileServerCopyRequest{}, false
+	}
+	if srcEntry.IsDirectory || dstEntry.IsDirectory {
+		glog.V(4).Infof("server-side copy: skipped (directory)")
+		return wholeFileServerCopyRequest{}, false
+	}
+
+	srcPbEntry := srcEntry.GetEntry()
+	dstPbEntry := dstEntry.GetEntry()
+	if srcPbEntry == nil || dstPbEntry == nil || srcPbEntry.Attributes == nil || dstPbEntry.Attributes == nil {
+		glog.V(4).Infof("server-side copy: skipped (missing entry attributes)")
+		return wholeFileServerCopyRequest{}, false
+	}
+
+	sourceSize := int64(filer.FileSize(srcPbEntry))
+	// go-fuse exposes CopyFileRange's return value as uint32, so the fast path
+	// should only claim copies that can be reported without truncation.
+	if sourceSize <= 0 || sourceSize > math.MaxUint32 || int64(in.Len) < sourceSize {
+		glog.V(4).Infof("server-side copy: skipped (size mismatch: sourceSize=%d len=%d)", sourceSize, in.Len)
+		return wholeFileServerCopyRequest{}, false
+	}
+
+	dstSize := int64(filer.FileSize(dstPbEntry))
+	if dstSize != 0 || len(dstPbEntry.GetChunks()) > 0 || len(dstPbEntry.Content) > 0 {
+		glog.V(4).Infof("server-side copy: skipped (destination not empty)")
+		return wholeFileServerCopyRequest{}, false
+	}
+
+	srcPath := fhIn.FullPath()
+	dstPath := fhOut.FullPath()
+	if srcPath == "" || dstPath == "" || srcPath == dstPath {
+		glog.V(4).Infof("server-side copy: skipped (invalid paths: src=%q dst=%q)", srcPath, dstPath)
+		return wholeFileServerCopyRequest{}, false
+	}
+
+	if srcPbEntry.Attributes.Inode == 0 || dstPbEntry.Attributes.Inode == 0 {
+		glog.V(4).Infof("server-side copy: skipped (missing inode preconditions: src=%d dst=%d)", srcPbEntry.Attributes.Inode, dstPbEntry.Attributes.Inode)
+		return wholeFileServerCopyRequest{}, false
+	}
+
+	return wholeFileServerCopyRequest{
+		srcPath:       srcPath,
+		dstPath:       dstPath,
+		sourceSize:    sourceSize,
+		srcInode:      srcPbEntry.Attributes.Inode,
+		srcMtime:      srcPbEntry.Attributes.Mtime,
+		dstInode:      dstPbEntry.Attributes.Inode,
+		dstMtime:      dstPbEntry.Attributes.Mtime,
+		dstSize:       dstSize,
+		sourceMime:    srcPbEntry.Attributes.Mime,
+		sourceMd5:     append([]byte(nil), srcPbEntry.Attributes.Md5...),
+		copyRequestID: request_id.New(),
+	}, true
+}
+
+func (wfs *WFS) copyEntryViaFiler(cancel <-chan struct{}, copyRequest wholeFileServerCopyRequest) (*filer_pb.Entry, entryVersion, serverSideWholeFileCopyOutcome, error) {
+	baseCtx, baseCancel := context.WithCancel(context.Background())
+	defer baseCancel()
+
+	if cancel != nil {
+		go func() {
+			select {
+			case <-cancel:
+				baseCancel()
+			case <-baseCtx.Done():
+			}
+		}()
+	}
+
+	postCtx, postCancel := context.WithTimeout(baseCtx, filerCopyRequestTimeout)
+	defer postCancel()
+
+	httpClient := util_http.GetGlobalHttpClient()
+	if httpClient == nil {
+		var err error
+		httpClient, err = util_http.NewGlobalHttpClient()
+		if err != nil {
+			return nil, entryVersion{}, serverSideWholeFileCopyNotCommitted, fmt.Errorf("create filer copy http client: %w", err)
+		}
+	}
+
+	copyURL := &url.URL{
+		Scheme: httpClient.GetHttpScheme(),
+		Host:   wfs.getCurrentFiler().ToHttpAddress(),
+		Path:   string(copyRequest.dstPath),
+	}
+	query := copyURL.Query()
+	query.Set(filer.CopyQueryParamFrom, string(copyRequest.srcPath))
+	query.Set(filer.CopyQueryParamOverwrite, "true")
+	query.Set(filer.CopyQueryParamDataOnly, "true")
+	query.Set(filer.CopyQueryParamRequestID, copyRequest.copyRequestID)
+	query.Set(filer.CopyQueryParamSourceInode, fmt.Sprintf("%d", copyRequest.srcInode))
+	query.Set(filer.CopyQueryParamSourceMtime, fmt.Sprintf("%d", copyRequest.srcMtime))
+	query.Set(filer.CopyQueryParamSourceSize, fmt.Sprintf("%d", copyRequest.sourceSize))
+	query.Set(filer.CopyQueryParamDestinationInode, fmt.Sprintf("%d", copyRequest.dstInode))
+	query.Set(filer.CopyQueryParamDestinationMtime, fmt.Sprintf("%d", copyRequest.dstMtime))
+	query.Set(filer.CopyQueryParamDestinationSize, fmt.Sprintf("%d", copyRequest.dstSize))
+	copyURL.RawQuery = query.Encode()
+
+	req, err := http.NewRequestWithContext(postCtx, http.MethodPost, copyURL.String(), nil)
+	if err != nil {
+		return nil, entryVersion{}, serverSideWholeFileCopyNotCommitted, fmt.Errorf("create filer copy request: %w", err)
+	}
+	if jwt := wfs.filerCopyJWT(); jwt != "" {
+		req.Header.Set("Authorization", security.BearerPrefix+string(jwt))
+	}
+
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return wfs.confirmServerSideWholeFileCopyAfterAmbiguousRequest(baseCtx, copyRequest, fmt.Errorf("execute filer copy request: %w", err))
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return nil, entryVersion{}, serverSideWholeFileCopyNotCommitted, fmt.Errorf("filer copy %s => %s failed: status %d: %s", copyRequest.srcPath, copyRequest.dstPath, resp.StatusCode, string(body))
+	}
+
+	readbackCtx, readbackCancel := context.WithTimeout(baseCtx, filerCopyReadbackTimeout)
+	defer readbackCancel()
+
+	entry, logTsNs, logSignature, err := filer_pb.GetEntry(readbackCtx, wfs, copyRequest.dstPath)
+	entryVersionTsNs := entryVersion{tsNs: logTsNs, signature: logSignature}
+	if err != nil {
+		return nil, entryVersion{}, serverSideWholeFileCopyCommitted, fmt.Errorf("reload copied entry %s: %w", copyRequest.dstPath, err)
+	}
+	if entry == nil {
+		return nil, entryVersion{}, serverSideWholeFileCopyCommitted, fmt.Errorf("reload copied entry %s: not found", copyRequest.dstPath)
+	}
+	if entry.Attributes != nil && wfs.option != nil && wfs.option.UidGidMapper != nil {
+		entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.FilerToLocal(entry.Attributes.Uid, entry.Attributes.Gid)
+	}
+
+	return entry, entryVersionTsNs, serverSideWholeFileCopyCommitted, nil
+}
+
+func (wfs *WFS) confirmServerSideWholeFileCopyAfterAmbiguousRequest(baseCtx context.Context, copyRequest wholeFileServerCopyRequest, requestErr error) (*filer_pb.Entry, entryVersion, serverSideWholeFileCopyOutcome, error) {
+	readbackCtx, readbackCancel := context.WithTimeout(baseCtx, filerCopyReadbackTimeout)
+	defer readbackCancel()
+
+	entry, logTsNs, logSignature, err := filer_pb.GetEntry(readbackCtx, wfs, copyRequest.dstPath)
+	entryVersionTsNs := entryVersion{tsNs: logTsNs, signature: logSignature}
+	if err == nil && entry != nil && entryMatchesServerSideWholeFileCopy(copyRequest, entry) {
+		if entry.Attributes != nil && wfs.option != nil && wfs.option.UidGidMapper != nil {
+			entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.FilerToLocal(entry.Attributes.Uid, entry.Attributes.Gid)
+		}
+		return entry, entryVersionTsNs, serverSideWholeFileCopyCommitted, nil
+	}
+
+	if err != nil {
+		return nil, entryVersion{}, serverSideWholeFileCopyAmbiguous, fmt.Errorf("%w; post-copy readback failed: %v", requestErr, err)
+	}
+	if entry == nil {
+		return nil, entryVersion{}, serverSideWholeFileCopyAmbiguous, fmt.Errorf("%w; destination %s was not readable after the ambiguous request", requestErr, copyRequest.dstPath)
+	}
+	return nil, entryVersion{}, serverSideWholeFileCopyAmbiguous, fmt.Errorf("%w; destination %s did not match the requested copy after the ambiguous request", requestErr, copyRequest.dstPath)
+}
+
+func entryMatchesServerSideWholeFileCopy(copyRequest wholeFileServerCopyRequest, entry *filer_pb.Entry) bool {
+	if entry == nil || entry.Attributes == nil {
+		return false
+	}
+	if copyRequest.dstInode != 0 && entry.Attributes.Inode != copyRequest.dstInode {
+		return false
+	}
+	if entry.Attributes.FileSize != uint64(copyRequest.sourceSize) {
+		return false
+	}
+	if copyRequest.sourceMime != "" && entry.Attributes.Mime != copyRequest.sourceMime {
+		return false
+	}
+	if len(copyRequest.sourceMd5) > 0 && !bytes.Equal(entry.Attributes.Md5, copyRequest.sourceMd5) {
+		return false
+	}
+	return true
+}
+
+func (wfs *WFS) filerCopyJWT() security.EncodedJwt {
+	if wfs.option == nil || len(wfs.option.FilerSigningKey) == 0 {
+		return ""
+	}
+	return security.GenJwtForFilerServer(wfs.option.FilerSigningKey, wfs.option.FilerSigningExpiresAfterSec)
+}