snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/mount/meta_cache/oversized_dir_test.go

@@ -0,0 +1,138 @@
+package meta_cache
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+)
+
+func listResponses(n int) []*filer_pb.ListEntriesResponse {
+	responses := make([]*filer_pb.ListEntriesResponse, 0, n)
+	for i := 0; i < n; i++ {
+		responses = append(responses, &filer_pb.ListEntriesResponse{
+			Entry: &filer_pb.Entry{
+				Name: fmt.Sprintf("f%05d", i),
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime: 1, Mtime: 1, FileMode: 0o644, FileSize: 3,
+				},
+			},
+		})
+	}
+	return responses
+}
+
+// TestEnsureVisitedRefusesOversizedDirectory checks that a directory past the
+// limit is not cached, that the refusal is remembered, and that the partial
+// build leaves nothing behind in the local store.
+func TestEnsureVisitedRefusesOversizedDirectory(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
+	defer mc.Shutdown()
+
+	accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(10)}}
+
+	err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 5)
+	var tooLarge *DirectoryTooLargeError
+	if !errors.As(err, &tooLarge) {
+		t.Fatalf("EnsureVisited = %v, want DirectoryTooLargeError", err)
+	}
+	if tooLarge.Path != util.FullPath("/dir") {
+		t.Fatalf("oversized path = %s, want /dir", tooLarge.Path)
+	}
+	if mc.IsDirectoryCached(util.FullPath("/dir")) {
+		t.Error("oversized directory reported cached")
+	}
+	// The aborted build must leave no partial children to be served later.
+	count := 0
+	if _, err := mc.ListDirectoryEntries(context.Background(), util.FullPath("/dir"), "", false, 100, func(e *filer.Entry) (bool, error) {
+		count++
+		return true, nil
+	}); err != nil {
+		t.Fatalf("list: %v", err)
+	}
+	if count != 0 {
+		t.Errorf("local store still holds %d children of the aborted build", count)
+	}
+
+	// A second visit fails fast without streaming to the limit again.
+	if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 5); !errors.As(err, &tooLarge) {
+		t.Fatalf("second EnsureVisited = %v, want DirectoryTooLargeError", err)
+	}
+}
+
+// TestEnsureVisitedStepsOverOversizedAncestor checks a huge ancestor does not
+// wedge the caching of its subdirectories.
+func TestEnsureVisitedStepsOverOversizedAncestor(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
+	defer mc.Shutdown()
+	mc.markOversized(util.FullPath("/huge"))
+
+	accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(3)}}
+	if err := EnsureVisited(mc, accessor, util.FullPath("/huge/sub"), 5); err != nil {
+		t.Fatalf("EnsureVisited under an oversized ancestor: %v", err)
+	}
+	if !mc.IsDirectoryCached(util.FullPath("/huge/sub")) {
+		t.Error("subdirectory of an oversized ancestor was not cached")
+	}
+	if mc.IsDirectoryCached(util.FullPath("/huge")) {
+		t.Error("oversized ancestor became cached")
+	}
+}
+
+// TestEnsureVisitedUnderTheLimitStillCaches pins that the gate does not change
+// behaviour for ordinary directories.
+func TestEnsureVisitedUnderTheLimitStillCaches(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
+	defer mc.Shutdown()
+
+	accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(5)}}
+	if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 5); err != nil {
+		t.Fatalf("EnsureVisited: %v", err)
+	}
+	if !mc.IsDirectoryCached(util.FullPath("/dir")) {
+		t.Error("directory at the limit was not cached")
+	}
+}
+
+// pathListClient serves canned listings per directory, so one visit can see
+// directories of different sizes.
+type pathListClient struct {
+	filer_pb.SeaweedFilerClient
+	perDir map[string][]*filer_pb.ListEntriesResponse
+}
+
+func (c *pathListClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
+	return &buildListStream{responses: c.perDir[in.Directory]}, nil
+}
+
+// TestEnsureVisitedAncestorFoundOversizedMidVisit covers the first discovery:
+// the ancestor's refusal must neither cancel the descendant's build nor be
+// reported as the descendant's own.
+func TestEnsureVisitedAncestorFoundOversizedMidVisit(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
+	defer mc.Shutdown()
+
+	accessor := &buildFilerAccessor{client: &pathListClient{perDir: map[string][]*filer_pb.ListEntriesResponse{
+		"/huge":     listResponses(10),
+		"/huge/sub": listResponses(3),
+	}}}
+
+	if err := EnsureVisited(mc, accessor, util.FullPath("/huge/sub"), 5); err != nil {
+		t.Fatalf("EnsureVisited: %v", err)
+	}
+	if !mc.IsDirectoryCached(util.FullPath("/huge/sub")) {
+		t.Error("descendant of a just-discovered oversized ancestor was not cached")
+	}
+	if mc.IsDirectoryCached(util.FullPath("/huge")) {
+		t.Error("oversized ancestor became cached")
+	}
+	if !mc.isOversized(util.FullPath("/huge")) {
+		t.Error("oversized ancestor was not remembered")
+	}
+}