snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/mount/weedfs_dir_read_pagination_test.go

@@ -0,0 +1,359 @@
+package mount
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// pagingSink collects names, accepting at most limit per round the way a kernel
+// reply buffer does.
+type pagingSink struct {
+	limit   int
+	names   []string
+	lastOff uint64
+	round   int
+}
+
+func (s *pagingSink) AddEntry(entry fuse.DirEntry) bool {
+	if s.round >= s.limit {
+		return false
+	}
+	s.round++
+	s.names = append(s.names, entry.Name)
+	s.lastOff = entry.Off
+	return true
+}
+
+func (s *pagingSink) AddEntryPlus(entry fuse.DirEntry) *fuse.EntryOut { return nil }
+func (s *pagingSink) TakesLookupRef() bool                            { return true }
+
+func newPagingWFS(tb testing.TB, dir util.FullPath, names []string, ttlSec int32) *WFS {
+	tb.Helper()
+	uidGidMapper, err := meta_cache.NewUidGidMapper("", "")
+	if err != nil {
+		tb.Fatalf("uid/gid mapper: %v", err)
+	}
+	root := util.FullPath("/")
+	wfs := &WFS{
+		option: &Option{
+			ChunkSizeLimit: 1024, ConcurrentReaders: 1, VolumeServerAccess: "filerProxy",
+			FilerAddresses:     []pb.ServerAddress{pb.NewServerAddressWithGrpcPort("127.0.0.1:1", 1)},
+			GrpcDialOption:     grpc.WithTransportCredentials(insecure.NewCredentials()),
+			FilerMountRootPath: "/", MountUid: 99, MountGid: 100, MountMode: 0o777,
+			MountMtime: time.Now(), MountCtime: time.Now(), UidGidMapper: uidGidMapper,
+		},
+		signature: 1, inodeToPath: NewInodeToPath(root, 0),
+		fhMap: NewFileHandleToInode(), dhMap: NewDirectoryHandleToInode(),
+		fhLockTable: util.NewLockTable[FileHandleId](), hardLinkLockTable: util.NewLockTable[string](),
+	}
+	wfs.metaCache = meta_cache.NewMetaCache(
+		filepath.Join(tb.TempDir(), "meta"), uidGidMapper, root, false,
+		func(p util.FullPath) { wfs.inodeToPath.MarkChildrenCached(p) },
+		func(p util.FullPath) bool { return wfs.inodeToPath.IsChildrenCached(p) },
+		func(meta_cache.EntryInvalidation) {}, nil,
+	)
+	tb.Cleanup(wfs.metaCache.Shutdown)
+
+	now := time.Now()
+	ctx := context.Background()
+	if err := wfs.metaCache.InsertEntry(ctx, &filer.Entry{
+		FullPath: dir,
+		Attr:     filer.Attr{Mode: os.ModeDir | 0o755, Mtime: now, Crtime: now},
+	}, 0); err != nil {
+		tb.Fatalf("insert dir: %v", err)
+	}
+	for _, name := range names {
+		child := dir.Child(name)
+		attr := filer.Attr{Mode: 0o644, Mtime: now, Crtime: now, FileSize: 1, Inode: child.AsInode(now.Unix())}
+		if ttlSec > 0 {
+			// Expired well in the past, so the meta cache drops it after the
+			// store has already counted it against the limit.
+			attr.TtlSec = ttlSec
+			attr.Crtime = now.Add(-time.Duration(ttlSec+60) * time.Second)
+		}
+		if err := wfs.metaCache.InsertEntry(ctx, &filer.Entry{FullPath: child, Attr: attr}, 0); err != nil {
+			tb.Fatalf("insert %s: %v", name, err)
+		}
+	}
+	wfs.inodeToPath.MarkChildrenCached(root)
+	wfs.inodeToPath.Lookup(dir, now.Unix(), true, false, 0, true)
+	wfs.inodeToPath.MarkChildrenCached(dir)
+	return wfs
+}
+
+// TestReadDirResumePastShrunkDirectory covers a client resuming a fresh handle
+// at a cookie from an earlier, larger listing. The directory has since shrunk
+// past that offset, and the readdir used to fall back to listing from the first
+// child again, handing back names the client had already consumed.
+func TestReadDirResumePastShrunkDirectory(t *testing.T) {
+	dir := util.FullPath("/d")
+	var names []string
+	for i := 0; i < 100; i++ {
+		names = append(names, fmt.Sprintf("f%03d", i))
+	}
+	wfs := newPagingWFS(t, dir, names, 0)
+	dirInode, _ := wfs.inodeToPath.GetInode(dir)
+
+	dhid, _ := wfs.AcquireDirectoryHandle()
+	defer wfs.ReleaseDirectoryHandle(dhid)
+
+	sink := &pagingSink{limit: 4096}
+	status := wfs.doReadDirectory(&fuse.ReadIn{
+		InHeader: fuse.InHeader{NodeId: dirInode},
+		Fh:       uint64(dhid),
+		Offset:   152, // past the end of a directory that now holds 100
+		Size:     1 << 20,
+	}, sink, false)
+	if status != fuse.OK {
+		t.Fatalf("readdir: %v", status)
+	}
+	if len(sink.names) != 0 {
+		t.Errorf("resuming past the end returned %d entries (first %q), want none", len(sink.names), sink.names[0])
+	}
+}
+
+// TestReadDirWithExpiredEntries covers a batch the store fills to the limit but
+// whose entries the meta cache then drops as expired. The post-filter count used
+// to be read as end-of-directory, silently hiding every later child.
+func TestReadDirWithExpiredEntries(t *testing.T) {
+	dir := util.FullPath("/d")
+	// One full store batch of entries that all expire, then live ones behind
+	// them. Nothing survives the first batch, which is the case that latched.
+	var names []string
+	for i := 0; i < batchSize; i++ {
+		names = append(names, fmt.Sprintf("a%05d", i))
+	}
+	wfs := newPagingWFS(t, dir, names, 30)
+
+	live := []string{"z001", "z002", "z003"}
+	now := time.Now()
+	for _, name := range live {
+		child := dir.Child(name)
+		if err := wfs.metaCache.InsertEntry(context.Background(), &filer.Entry{
+			FullPath: child,
+			Attr:     filer.Attr{Mode: 0o644, Mtime: now, Crtime: now, FileSize: 1, Inode: child.AsInode(now.Unix())},
+		}, 0); err != nil {
+			t.Fatalf("insert %s: %v", name, err)
+		}
+	}
+
+	dirInode, _ := wfs.inodeToPath.GetInode(dir)
+	dhid, _ := wfs.AcquireDirectoryHandle()
+	defer wfs.ReleaseDirectoryHandle(dhid)
+
+	sink := &pagingSink{limit: 4096}
+	var offset uint64
+	for round := 0; round < 20; round++ {
+		sink.round = 0
+		status := wfs.doReadDirectory(&fuse.ReadIn{
+			InHeader: fuse.InHeader{NodeId: dirInode},
+			Fh:       uint64(dhid),
+			Offset:   offset,
+			Size:     1 << 20,
+		}, sink, false)
+		if status != fuse.OK {
+			t.Fatalf("readdir: %v", status)
+		}
+		if sink.lastOff <= offset {
+			break
+		}
+		offset = sink.lastOff
+	}
+
+	var seen []string
+	for _, n := range sink.names {
+		if n != "." && n != ".." {
+			seen = append(seen, n)
+		}
+	}
+	if len(seen) != len(live) {
+		t.Fatalf("listed %d entries %v, want the %d live ones %v", len(seen), seen, len(live), live)
+	}
+}
+
+// TestReadDirTrimsConsumedEntries checks that a walk does not accumulate the
+// whole directory in the handle. Offsets index into the stream from
+// entryStreamOffset, so the two have to advance together or the listing
+// silently misaligns.
+func TestReadDirTrimsConsumedEntries(t *testing.T) {
+	dir := util.FullPath("/d")
+	const total = 5000
+	var names []string
+	for i := 0; i < total; i++ {
+		names = append(names, fmt.Sprintf("f%05d", i))
+	}
+	wfs := newPagingWFS(t, dir, names, 0)
+	dirInode, _ := wfs.inodeToPath.GetInode(dir)
+
+	dhid, dh := wfs.AcquireDirectoryHandle()
+	defer wfs.ReleaseDirectoryHandle(dhid)
+
+	// A small sink forces many rounds, which is when the stream would grow.
+	sink := &pagingSink{limit: 64}
+	var offset uint64
+	var seen []string
+	peak := 0
+	for round := 0; round < 500; round++ {
+		sink.round = 0
+		before := len(sink.names)
+		status := wfs.doReadDirectory(&fuse.ReadIn{
+			InHeader: fuse.InHeader{NodeId: dirInode},
+			Fh:       uint64(dhid),
+			Offset:   offset,
+			Size:     1 << 20,
+		}, sink, false)
+		if status != fuse.OK {
+			t.Fatalf("readdir: %v", status)
+		}
+		if n := len(dh.entryStream); n > peak {
+			peak = n
+		}
+		if len(sink.names) == before || sink.lastOff <= offset {
+			break
+		}
+		offset = sink.lastOff
+	}
+	for _, n := range sink.names {
+		if n != "." && n != ".." {
+			seen = append(seen, n)
+		}
+	}
+
+	if len(seen) != total {
+		t.Fatalf("listed %d entries, want %d", len(seen), total)
+	}
+	for i, name := range seen {
+		if want := fmt.Sprintf("f%05d", i); name != want {
+			t.Fatalf("entry %d is %q, want %q -- trimming misaligned the offsets", i, name, want)
+		}
+	}
+	// Without trimming the handle ends up holding every entry.
+	if peak >= total {
+		t.Errorf("handle held %d entries at peak, want well under the %d in the directory", peak, total)
+	}
+}
+
+// listingFilerServer serves a fixed, sorted child list the way the filer
+// paginates one.
+type listingFilerServer struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+	names []string
+}
+
+func (s *listingFilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream filer_pb.SeaweedFiler_ListEntriesServer) error {
+	var sent uint32
+	for _, name := range s.names {
+		if name < req.StartFromFileName || (name == req.StartFromFileName && !req.InclusiveStartFrom) {
+			continue
+		}
+		if req.Limit > 0 && sent >= req.Limit {
+			break
+		}
+		if err := stream.Send(&filer_pb.ListEntriesResponse{Entry: &filer_pb.Entry{
+			Name:       name,
+			Attributes: &filer_pb.FuseAttributes{FileMode: 0o644, FileSize: 1},
+		}}); err != nil {
+			return err
+		}
+		sent++
+	}
+	return nil
+}
+
+// TestReadDirDirectTrimsConsumedEntries is the same check for a directory read
+// through to the filer. That path is taken precisely for directories too big to
+// cache, so holding the whole walk in the handle is where a listing turns into
+// gigabytes of resident memory.
+func TestReadDirDirectTrimsConsumedEntries(t *testing.T) {
+	dir := util.FullPath("/d")
+	const total = 5000
+	var names []string
+	for i := 0; i < total; i++ {
+		names = append(names, fmt.Sprintf("f%05d", i))
+	}
+	wfs := newPagingWFS(t, dir, nil, 0)
+	startFakeFiler(t, wfs, &listingFilerServer{names: names})
+	dirInode, _ := wfs.inodeToPath.GetInode(dir)
+	if !wfs.inodeToPath.MarkDirectoryReadThrough(dir, time.Now()) {
+		t.Fatal("directory did not enter read-through mode")
+	}
+
+	dhid, dh := wfs.AcquireDirectoryHandle()
+	defer wfs.ReleaseDirectoryHandle(dhid)
+
+	sink := &pagingSink{limit: 64}
+	var offset uint64
+	var seen []string
+	peak := 0
+	for round := 0; round < 500; round++ {
+		sink.round = 0
+		before := len(sink.names)
+		status := wfs.doReadDirectory(&fuse.ReadIn{
+			InHeader: fuse.InHeader{NodeId: dirInode},
+			Fh:       uint64(dhid),
+			Offset:   offset,
+			Size:     1 << 20,
+		}, sink, false)
+		if status != fuse.OK {
+			t.Fatalf("readdir: %v", status)
+		}
+		if n := len(dh.entryStream); n > peak {
+			peak = n
+		}
+		if len(sink.names) == before || sink.lastOff <= offset {
+			break
+		}
+		offset = sink.lastOff
+	}
+	for _, n := range sink.names {
+		if n != "." && n != ".." {
+			seen = append(seen, n)
+		}
+	}
+
+	if len(seen) != total {
+		t.Fatalf("listed %d entries, want %d", len(seen), total)
+	}
+	for i, name := range seen {
+		if want := fmt.Sprintf("f%05d", i); name != want {
+			t.Fatalf("entry %d is %q, want %q -- trimming misaligned the offsets", i, name, want)
+		}
+	}
+	if peak >= total {
+		t.Errorf("handle held %d entries at peak, want well under the %d in the directory", peak, total)
+	}
+}
+
+// The table has to key on the sanitized name, not the stored bytes.
+func TestReadDirPlusSanitizedName(t *testing.T) {
+	dir := util.FullPath("/images")
+	const rawName = "bad\xffname.jpg"
+	wfs := newPagingWFS(t, dir, []string{"good.jpg", rawName}, 0)
+	dirInode, _ := wfs.inodeToPath.GetInode(dir)
+
+	sink := &benchSink{plus: true, takesRef: true, sinkLimit: 16}
+	if got := walkOnce(t, wfs, dirInode, sink, false); got != 4 {
+		t.Fatalf("listed %d entries, want 4 (. .. and two children)", got)
+	}
+
+	sanitized := dir.Child(util.SanitizeUTF8Name(rawName))
+	if !wfs.inodeToPath.HasPath(sanitized) {
+		t.Errorf("%s not in the inode table", sanitized)
+	}
+	if wfs.inodeToPath.HasPath(dir.Child(rawName)) {
+		t.Errorf("%s keyed on the unsanitized name", dir.Child(rawName))
+	}
+}