upstream-seaweedfs/weed/mount/meta_cache/oversized_dir_test.go

package meta_cache

import (
	"context"
	"errors"
	"fmt"
	"testing"

	"github.com/seaweedfs/seaweedfs/weed/filer"

	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"google.golang.org/grpc"
)

func listResponses(n int) []*filer_pb.ListEntriesResponse {
	responses := make([]*filer_pb.ListEntriesResponse, 0, n)
	for i := 0; i < n; i++ {
		responses = append(responses, &filer_pb.ListEntriesResponse{
			Entry: &filer_pb.Entry{
				Name: fmt.Sprintf("f%05d", i),
				Attributes: &filer_pb.FuseAttributes{
					Crtime: 1, Mtime: 1, FileMode: 0o644, FileSize: 3,
				},
			},
		})
	}
	return responses
}

// TestEnsureVisitedRefusesOversizedDirectory checks that a directory past the
// limit is not cached, that the refusal is remembered, and that the partial
// build leaves nothing behind in the local store.
func TestEnsureVisitedRefusesOversizedDirectory(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
	defer mc.Shutdown()

	accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(10)}}

	err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 5)
	var tooLarge *DirectoryTooLargeError
	if !errors.As(err, &tooLarge) {
		t.Fatalf("EnsureVisited = %v, want DirectoryTooLargeError", err)
	}
	if tooLarge.Path != util.FullPath("/dir") {
		t.Fatalf("oversized path = %s, want /dir", tooLarge.Path)
	}
	if mc.IsDirectoryCached(util.FullPath("/dir")) {
		t.Error("oversized directory reported cached")
	}
	// The aborted build must leave no partial children to be served later.
	count := 0
	if _, err := mc.ListDirectoryEntries(context.Background(), util.FullPath("/dir"), "", false, 100, func(e *filer.Entry) (bool, error) {
		count++
		return true, nil
	}); err != nil {
		t.Fatalf("list: %v", err)
	}
	if count != 0 {
		t.Errorf("local store still holds %d children of the aborted build", count)
	}

	// A second visit fails fast without streaming to the limit again.
	if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 5); !errors.As(err, &tooLarge) {
		t.Fatalf("second EnsureVisited = %v, want DirectoryTooLargeError", err)
	}
}

// TestEnsureVisitedStepsOverOversizedAncestor checks a huge ancestor does not
// wedge the caching of its subdirectories.
func TestEnsureVisitedStepsOverOversizedAncestor(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
	defer mc.Shutdown()
	mc.markOversized(util.FullPath("/huge"))

	accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(3)}}
	if err := EnsureVisited(mc, accessor, util.FullPath("/huge/sub"), 5); err != nil {
		t.Fatalf("EnsureVisited under an oversized ancestor: %v", err)
	}
	if !mc.IsDirectoryCached(util.FullPath("/huge/sub")) {
		t.Error("subdirectory of an oversized ancestor was not cached")
	}
	if mc.IsDirectoryCached(util.FullPath("/huge")) {
		t.Error("oversized ancestor became cached")
	}
}

// TestEnsureVisitedUnderTheLimitStillCaches pins that the gate does not change
// behaviour for ordinary directories.
func TestEnsureVisitedUnderTheLimitStillCaches(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
	defer mc.Shutdown()

	accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(5)}}
	if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 5); err != nil {
		t.Fatalf("EnsureVisited: %v", err)
	}
	if !mc.IsDirectoryCached(util.FullPath("/dir")) {
		t.Error("directory at the limit was not cached")
	}
}

// pathListClient serves canned listings per directory, so one visit can see
// directories of different sizes.
type pathListClient struct {
	filer_pb.SeaweedFilerClient
	perDir map[string][]*filer_pb.ListEntriesResponse
}

func (c *pathListClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
	return &buildListStream{responses: c.perDir[in.Directory]}, nil
}

// TestEnsureVisitedAncestorFoundOversizedMidVisit covers the first discovery:
// the ancestor's refusal must neither cancel the descendant's build nor be
// reported as the descendant's own.
func TestEnsureVisitedAncestorFoundOversizedMidVisit(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
	defer mc.Shutdown()

	accessor := &buildFilerAccessor{client: &pathListClient{perDir: map[string][]*filer_pb.ListEntriesResponse{
		"/huge":     listResponses(10),
		"/huge/sub": listResponses(3),
	}}}

	if err := EnsureVisited(mc, accessor, util.FullPath("/huge/sub"), 5); err != nil {
		t.Fatalf("EnsureVisited: %v", err)
	}
	if !mc.IsDirectoryCached(util.FullPath("/huge/sub")) {
		t.Error("descendant of a just-discovered oversized ancestor was not cached")
	}
	if mc.IsDirectoryCached(util.FullPath("/huge")) {
		t.Error("oversized ancestor became cached")
	}
	if !mc.isOversized(util.FullPath("/huge")) {
		t.Error("oversized ancestor was not remembered")
	}
}