upstream-seaweedfs/weed/filer/empty_folder_cleanup/empty_folder_cleaner_test.go

package empty_folder_cleanup

import (
	"context"
	"errors"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

type mockFilerOps struct {
	countFn       func(path util.FullPath) (int, error)
	deleteFn      func(path util.FullPath) error
	attrsFn       func(path util.FullPath) (map[string][]byte, error)
	isDirKeyObjFn func(path util.FullPath) (bool, error)
	ensureDirFn   func(path util.FullPath, attrs DirectoryAttributes) error
	dirAttrsFn    func(path util.FullPath) (DirectoryAttributes, error)
}

func (m *mockFilerOps) EnsureDirectoryEntry(_ context.Context, p util.FullPath, attrs DirectoryAttributes) error {
	if m.ensureDirFn == nil {
		return nil
	}
	return m.ensureDirFn(p, attrs)
}

func (m *mockFilerOps) DirectoryAttributes(_ context.Context, p util.FullPath) (DirectoryAttributes, error) {
	if m.dirAttrsFn == nil {
		return DirectoryAttributes{Mode: 0755}, nil
	}
	return m.dirAttrsFn(p)
}

func (m *mockFilerOps) CountDirectoryEntries(_ context.Context, dirPath util.FullPath, _ int) (int, error) {
	if m.countFn == nil {
		return 0, nil
	}
	return m.countFn(dirPath)
}

func (m *mockFilerOps) DeleteEntryMetaAndData(_ context.Context, p util.FullPath, _, _, _, _ bool, _ []int32, _ int64) error {
	if m.deleteFn == nil {
		return nil
	}
	return m.deleteFn(p)
}

func (m *mockFilerOps) GetEntryAttributes(_ context.Context, p util.FullPath) (map[string][]byte, error) {
	if m.attrsFn == nil {
		return nil, nil
	}
	return m.attrsFn(p)
}

func (m *mockFilerOps) IsDirectoryKeyObject(_ context.Context, p util.FullPath) (bool, error) {
	if m.isDirKeyObjFn == nil {
		return false, nil
	}
	return m.isDirKeyObjFn(p)
}

func Test_isUnderPath(t *testing.T) {
	tests := []struct {
		name     string
		child    string
		parent   string
		expected bool
	}{
		{"child under parent", "/buckets/mybucket/folder/file.txt", "/buckets", true},
		{"child is parent", "/buckets", "/buckets", true},
		{"child not under parent", "/other/path", "/buckets", false},
		{"empty parent", "/any/path", "", true},
		{"root parent", "/any/path", "/", true},
		{"parent with trailing slash", "/buckets/mybucket", "/buckets/", true},
		{"similar prefix but not under", "/buckets-other/file", "/buckets", false},
		{"deeply nested", "/buckets/a/b/c/d/e/f", "/buckets/a/b", true},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result := isUnderPath(tt.child, tt.parent)
			if result != tt.expected {
				t.Errorf("isUnderPath(%q, %q) = %v, want %v", tt.child, tt.parent, result, tt.expected)
			}
		})
	}
}

func Test_isUnderBucketPath(t *testing.T) {
	tests := []struct {
		name       string
		directory  string
		bucketPath string
		expected   bool
	}{
		// Should NOT process - bucket path itself
		{"bucket path itself", "/buckets", "/buckets", false},
		// Should NOT process - bucket directory (immediate child)
		{"bucket directory", "/buckets/mybucket", "/buckets", false},
		// Should process - folder inside bucket
		{"folder in bucket", "/buckets/mybucket/folder", "/buckets", true},
		// Should process - nested folder
		{"nested folder", "/buckets/mybucket/a/b/c", "/buckets", true},
		// Should NOT process - outside buckets
		{"outside buckets", "/other/path", "/buckets", false},
		// Empty bucket path allows all
		{"empty bucket path", "/any/path", "", true},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result := isUnderBucketPath(tt.directory, tt.bucketPath)
			if result != tt.expected {
				t.Errorf("isUnderBucketPath(%q, %q) = %v, want %v", tt.directory, tt.bucketPath, result, tt.expected)
			}
		})
	}
}

func Test_isMultipartUploadsPath(t *testing.T) {
	uploads := s3_constants.MultipartUploadsFolder
	tests := []struct {
		name       string
		directory  string
		bucketPath string
		expected   bool
	}{
		{"staging root", "/buckets/mybucket/" + uploads, "/buckets", true},
		{"upload marker dir", "/buckets/mybucket/" + uploads + "/abc123", "/buckets", true},
		{"nested under staging", "/buckets/mybucket/" + uploads + "/abc123/hashstates", "/buckets", true},
		{"normal folder", "/buckets/mybucket/folder", "/buckets", false},
		{"registry own uploads dir", "/buckets/mybucket/docker/registry/v2/_uploads/x", "/buckets", false},
		{"bucket itself", "/buckets/mybucket", "/buckets", false},
		{"outside buckets", "/other/" + uploads, "/buckets", false},
		{"empty bucket path", "/buckets/mybucket/" + uploads, "", false},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := isMultipartUploadsPath(tt.bucketPath, tt.directory); got != tt.expected {
				t.Errorf("isMultipartUploadsPath(%q, %q) = %v, want %v", tt.bucketPath, tt.directory, got, tt.expected)
			}
		})
	}
}

func TestEmptyFolderCleaner_OnDeleteEvent_skipsMultipartUploads(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	now := time.Now()
	uploads := "/buckets/mybucket/" + s3_constants.MultipartUploadsFolder
	cleaner.OnDeleteEvent(uploads, "abc123", true, now)
	cleaner.OnDeleteEvent(uploads+"/abc123", "0001.part", false, now)
	if cleaner.GetPendingCleanupCount() != 0 {
		t.Fatalf("multipart staging paths must not be queued, got %d pending", cleaner.GetPendingCleanupCount())
	}

	// A normal folder is still queued, proving the guard is scoped to .uploads.
	cleaner.OnDeleteEvent("/buckets/mybucket/folder", "file.txt", false, now)
	if cleaner.GetPendingCleanupCount() != 1 {
		t.Fatalf("normal folder should be queued, got %d pending", cleaner.GetPendingCleanupCount())
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_executeCleanup_skipsMultipartUploads(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	var deleted []string
	mock := &mockFilerOps{
		countFn:  func(util.FullPath) (int, error) { return 0, nil },
		deleteFn: func(p util.FullPath) error { deleted = append(deleted, string(p)); return nil },
	}

	cleaner := &EmptyFolderCleaner{
		filer:                 mock,
		lockRing:              lockRing,
		host:                  "filer1:8888",
		bucketPath:            "/buckets",
		enabled:               true,
		maxCountCheck:         DefaultMaxCountCheck,
		cacheExpiry:           DefaultCacheExpiry,
		folderCounts:          make(map[string]*folderState),
		bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
		cleanupQueue:          NewCleanupQueue(1000, 10*time.Minute),
		stopCh:                make(chan struct{}),
	}

	uploads := "/buckets/mybucket/" + s3_constants.MultipartUploadsFolder
	cleaner.executeCleanup(uploads, "abc123")
	cleaner.executeCleanup(uploads+"/abc123", "0001.part")
	if len(deleted) != 0 {
		t.Fatalf("multipart staging paths must not be deleted, got %v", deleted)
	}

	// An empty normal folder is deleted, proving the guard did not disable cleanup.
	cleaner.executeCleanup("/buckets/mybucket/folder", "file.txt")
	if len(deleted) != 1 || deleted[0] != "/buckets/mybucket/folder" {
		t.Fatalf("normal empty folder should be deleted, got %v", deleted)
	}
}

func TestEmptyFolderCleaner_restoreFoldersWrittenDuringDelete(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	newCleaner := func(mock *mockFilerOps) *EmptyFolderCleaner {
		return &EmptyFolderCleaner{
			filer:                 mock,
			lockRing:              lockRing,
			host:                  "filer1:8888",
			bucketPath:            "/buckets",
			enabled:               true,
			maxCountCheck:         DefaultMaxCountCheck,
			cacheExpiry:           DefaultCacheExpiry,
			folderCounts:          make(map[string]*folderState),
			bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
			deleted:               make(map[string]*deletedFolder),
			cleanupQueue:          NewCleanupQueue(1000, 10*time.Minute),
			stopCh:                make(chan struct{}),
		}
	}

	const folder = "/buckets/mybucket/folder"

	t.Run("a create event for the deleted folder puts it back", func(t *testing.T) {
		var restored []string
		mock := &mockFilerOps{
			countFn:     func(util.FullPath) (int, error) { return 0, nil },
			ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "file.txt")

		// nothing has been written to it, so it is only observed
		cleaner.processCleanupQueue()
		if len(restored) != 0 {
			t.Fatalf("a folder nothing was written to should not be restored, got %v", restored)
		}
		if len(cleaner.deleted) != 1 {
			t.Fatalf("folder should still be observed, got %d", len(cleaner.deleted))
		}

		// the write that raced the delete arrives as a create event
		mock.countFn = func(util.FullPath) (int, error) { return 1, nil }
		cleaner.OnCreateEvent(folder, "obj", false)
		cleaner.processCleanupQueue()
		if len(restored) != 1 || restored[0] != folder {
			t.Fatalf("a folder written to while being deleted should be restored, got %v", restored)
		}
		if len(cleaner.deleted) != 0 {
			t.Fatalf("a restored folder should stop being observed, got %d", len(cleaner.deleted))
		}
	})

	t.Run("a create event elsewhere is ignored", func(t *testing.T) {
		var restored []string
		mock := &mockFilerOps{
			countFn:     func(util.FullPath) (int, error) { return 0, nil },
			ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "file.txt")
		cleaner.OnCreateEvent("/buckets/mybucket/other", "obj", false)
		cleaner.processCleanupQueue()
		if len(restored) != 0 {
			t.Fatalf("only the folder written to should be restored, got %v", restored)
		}
	})

	t.Run("an entry that went away again is not restored", func(t *testing.T) {
		var restored []string
		mock := &mockFilerOps{
			countFn:     func(util.FullPath) (int, error) { return 0, nil },
			ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "file.txt")
		cleaner.OnCreateEvent(folder, "obj", false)
		cleaner.processCleanupQueue()
		if len(restored) != 0 {
			t.Fatalf("an empty folder should not be restored, got %v", restored)
		}
	})

	t.Run("observation expires", func(t *testing.T) {
		mock := &mockFilerOps{countFn: func(util.FullPath) (int, error) { return 0, nil }}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "file.txt")
		cleaner.processCleanupQueue()
		if len(cleaner.deleted) != 1 {
			t.Fatalf("folder should still be observed inside the window, got %d", len(cleaner.deleted))
		}

		cleaner.deleted[folder].deletedAt = time.Now().Add(-DefaultObservationWindow - time.Second)
		cleaner.processCleanupQueue()
		if len(cleaner.deleted) != 0 {
			t.Fatalf("folder should stop being observed once the window has passed, got %d", len(cleaner.deleted))
		}
	})

	t.Run("a failed restore is retried", func(t *testing.T) {
		attempts := 0
		mock := &mockFilerOps{
			countFn: func(util.FullPath) (int, error) { return 1, nil },
			ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error {
				attempts++
				if attempts == 1 {
					return errors.New("store unavailable")
				}
				return nil
			},
		}

		cleaner := newCleaner(mock)
		// deleted while empty, then written to
		mock.countFn = func(util.FullPath) (int, error) { return 0, nil }
		cleaner.executeCleanup(folder, "file.txt")
		mock.countFn = func(util.FullPath) (int, error) { return 1, nil }
		cleaner.OnCreateEvent(folder, "obj", false)

		cleaner.processCleanupQueue()
		if len(cleaner.deleted) != 1 {
			t.Fatalf("a folder whose restore failed should be kept, got %d", len(cleaner.deleted))
		}

		cleaner.processCleanupQueue()
		if attempts != 2 {
			t.Fatalf("a failed restore should be retried, got %d attempts", attempts)
		}
		if len(cleaner.deleted) != 0 {
			t.Fatalf("a restored folder should stop being observed, got %d", len(cleaner.deleted))
		}
	})

	t.Run("a folder whose restore keeps failing stops being observed", func(t *testing.T) {
		mock := &mockFilerOps{
			countFn:     func(util.FullPath) (int, error) { return 0, nil },
			ensureDirFn: func(util.FullPath, DirectoryAttributes) error { return errors.New("store unavailable") },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "file.txt")
		mock.countFn = func(util.FullPath) (int, error) { return 1, nil }
		cleaner.OnCreateEvent(folder, "obj", false)

		cleaner.processCleanupQueue()
		if len(cleaner.deleted) != 1 {
			t.Fatalf("a folder whose restore failed should be kept, got %d", len(cleaner.deleted))
		}

		// otherwise it would be retried on every pass for the rest of the process
		cleaner.deleted[folder].deletedAt = time.Now().Add(-DefaultObservationWindow - time.Second)
		cleaner.processCleanupQueue()
		if len(cleaner.deleted) != 0 {
			t.Fatalf("the window should apply to a failing folder too, got %d", len(cleaner.deleted))
		}
	})

	t.Run("an ancestor taken by the cascade is restored with its own attributes", func(t *testing.T) {
		const ancestor = "/buckets/mybucket/data"
		const nested = ancestor + "/abc"

		restored := map[string]DirectoryAttributes{}
		var order []string
		mock := &mockFilerOps{
			countFn: func(util.FullPath) (int, error) { return 0, nil },
			dirAttrsFn: func(p util.FullPath) (DirectoryAttributes, error) {
				if string(p) == ancestor {
					return DirectoryAttributes{Mode: 0700, Uid: 4242}, nil
				}
				return DirectoryAttributes{Mode: 0755, Uid: 7}, nil
			},
			ensureDirFn: func(p util.FullPath, attrs DirectoryAttributes) error {
				restored[string(p)] = attrs
				order = append(order, string(p))
				return nil
			},
		}

		cleaner := newCleaner(mock)
		// the cascade takes the nested folder and then its parent
		cleaner.executeCleanup(nested, "file.txt")
		cleaner.executeCleanup(ancestor, "abc")

		// only the nested folder is written to
		mock.countFn = func(util.FullPath) (int, error) { return 1, nil }
		cleaner.OnCreateEvent(nested, "obj", false)
		cleaner.processCleanupQueue()

		if len(order) != 2 || order[0] != ancestor {
			t.Fatalf("the ancestor should be rebuilt first, got %v", order)
		}
		if got := restored[ancestor]; got.Mode != 0700 || got.Uid != 4242 {
			t.Errorf("ancestor should keep its own attributes, got mode %o uid %d", got.Mode, got.Uid)
		}
	})

	t.Run("a delete that reports failure is still observed", func(t *testing.T) {
		// the redis stores drop the folder before its parent-list member, so a failure
		// return is not proof that the folder survived
		mock := &mockFilerOps{
			countFn:  func(util.FullPath) (int, error) { return 0, nil },
			deleteFn: func(util.FullPath) error { return errors.New("partial delete") },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "file.txt")
		if len(cleaner.deleted) != 1 {
			t.Fatalf("a failed delete should still leave the folder observed, got %d", len(cleaner.deleted))
		}
	})
}

func Test_autoRemoveEmptyFoldersEnabled(t *testing.T) {
	tests := []struct {
		name      string
		attrs     map[string][]byte
		enabled   bool
		attrValue string
	}{
		{
			name:      "no attrs defaults enabled",
			attrs:     nil,
			enabled:   true,
			attrValue: "<no_attrs>",
		},
		{
			name:      "missing key defaults enabled",
			attrs:     map[string][]byte{},
			enabled:   true,
			attrValue: "<missing>",
		},
		{
			name: "allow-empty disables cleanup",
			attrs: map[string][]byte{
				s3_constants.ExtAllowEmptyFolders: []byte("true"),
			},
			enabled:   false,
			attrValue: "true",
		},
		{
			name: "explicit false keeps cleanup enabled",
			attrs: map[string][]byte{
				s3_constants.ExtAllowEmptyFolders: []byte("false"),
			},
			enabled:   true,
			attrValue: "false",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			enabled, attrValue := autoRemoveEmptyFoldersEnabled(tt.attrs)
			if enabled != tt.enabled {
				t.Fatalf("expected enabled=%v, got %v", tt.enabled, enabled)
			}
			if attrValue != tt.attrValue {
				t.Fatalf("expected attrValue=%q, got %q", tt.attrValue, attrValue)
			}
		})
	}
}

func TestEmptyFolderCleaner_ownsFolder(t *testing.T) {
	// Create a LockRing with multiple servers
	lockRing := lock_manager.NewLockRing(5 * time.Second)

	servers := []pb.ServerAddress{
		"filer1:8888",
		"filer2:8888",
		"filer3:8888",
	}
	lockRing.SetSnapshot(servers, 0)

	// Create cleaner for filer1
	cleaner1 := &EmptyFolderCleaner{
		lockRing: lockRing,
		host:     "filer1:8888",
	}

	// Create cleaner for filer2
	cleaner2 := &EmptyFolderCleaner{
		lockRing: lockRing,
		host:     "filer2:8888",
	}

	// Create cleaner for filer3
	cleaner3 := &EmptyFolderCleaner{
		lockRing: lockRing,
		host:     "filer3:8888",
	}

	// Test that exactly one filer owns each folder
	testFolders := []string{
		"/buckets/mybucket/folder1",
		"/buckets/mybucket/folder2",
		"/buckets/mybucket/folder3",
		"/buckets/mybucket/a/b/c",
		"/buckets/otherbucket/x",
	}

	for _, folder := range testFolders {
		ownCount := 0
		if cleaner1.ownsFolder(folder) {
			ownCount++
		}
		if cleaner2.ownsFolder(folder) {
			ownCount++
		}
		if cleaner3.ownsFolder(folder) {
			ownCount++
		}

		if ownCount != 1 {
			t.Errorf("folder %q owned by %d filers, expected exactly 1", folder, ownCount)
		}
	}
}

func TestEmptyFolderCleaner_ownsFolder_singleServer(t *testing.T) {
	// Create a LockRing with a single server
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing: lockRing,
		host:     "filer1:8888",
	}

	// Single filer should own all folders
	testFolders := []string{
		"/buckets/mybucket/folder1",
		"/buckets/mybucket/folder2",
		"/buckets/otherbucket/x",
	}

	for _, folder := range testFolders {
		if !cleaner.ownsFolder(folder) {
			t.Errorf("single filer should own folder %q", folder)
		}
	}
}

func TestEmptyFolderCleaner_ownsFolder_emptyRing(t *testing.T) {
	// Create an empty LockRing
	lockRing := lock_manager.NewLockRing(5 * time.Second)

	cleaner := &EmptyFolderCleaner{
		lockRing: lockRing,
		host:     "filer1:8888",
	}

	// With empty ring, should own all folders
	if !cleaner.ownsFolder("/buckets/mybucket/folder") {
		t.Error("should own folder with empty ring")
	}
}

func TestEmptyFolderCleaner_OnCreateEvent_cancelsCleanup(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	folder := "/buckets/mybucket/testfolder"
	now := time.Now()

	// Simulate delete event
	cleaner.OnDeleteEvent(folder, "file.txt", false, now)

	// Check that cleanup is queued
	if cleaner.GetPendingCleanupCount() != 1 {
		t.Errorf("expected 1 pending cleanup, got %d", cleaner.GetPendingCleanupCount())
	}

	// Simulate create event
	cleaner.OnCreateEvent(folder, "newfile.txt", false)

	// Check that cleanup is cancelled
	if cleaner.GetPendingCleanupCount() != 0 {
		t.Errorf("expected 0 pending cleanups after create, got %d", cleaner.GetPendingCleanupCount())
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_OnDeleteEvent_deduplication(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	folder := "/buckets/mybucket/testfolder"
	now := time.Now()

	// Simulate multiple delete events for same folder
	for i := 0; i < 5; i++ {
		cleaner.OnDeleteEvent(folder, "file"+string(rune('0'+i))+".txt", false, now.Add(time.Duration(i)*time.Second))
	}

	// Check that only 1 cleanup is queued (deduplicated)
	if cleaner.GetPendingCleanupCount() != 1 {
		t.Errorf("expected 1 pending cleanup after deduplication, got %d", cleaner.GetPendingCleanupCount())
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_OnDeleteEvent_multipleFolders(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	now := time.Now()

	// Delete files in different folders
	cleaner.OnDeleteEvent("/buckets/mybucket/folder1", "file.txt", false, now)
	cleaner.OnDeleteEvent("/buckets/mybucket/folder2", "file.txt", false, now.Add(1*time.Second))
	cleaner.OnDeleteEvent("/buckets/mybucket/folder3", "file.txt", false, now.Add(2*time.Second))

	// Each folder should be queued
	if cleaner.GetPendingCleanupCount() != 3 {
		t.Errorf("expected 3 pending cleanups, got %d", cleaner.GetPendingCleanupCount())
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_OnDeleteEvent_notOwner(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888", "filer2:8888"}, 0)

	// Create cleaner for filer that doesn't own the folder
	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	now := time.Now()

	// Try many folders, looking for one that filer1 doesn't own
	foundNonOwned := false
	for i := 0; i < 100; i++ {
		folder := "/buckets/mybucket/folder" + string(rune('0'+i%10)) + string(rune('0'+i/10))
		if !cleaner.ownsFolder(folder) {
			// This folder is not owned by filer1
			cleaner.OnDeleteEvent(folder, "file.txt", false, now)
			if cleaner.GetPendingCleanupCount() != 0 {
				t.Errorf("non-owner should not queue cleanup for folder %s", folder)
			}
			foundNonOwned = true
			break
		}
	}

	if !foundNonOwned {
		t.Skip("could not find a folder not owned by filer1")
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_OnDeleteEvent_disabled(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      false, // Disabled
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	folder := "/buckets/mybucket/testfolder"
	now := time.Now()

	// Simulate delete event
	cleaner.OnDeleteEvent(folder, "file.txt", false, now)

	// Check that no cleanup is queued when disabled
	if cleaner.GetPendingCleanupCount() != 0 {
		t.Errorf("disabled cleaner should not queue cleanup, got %d", cleaner.GetPendingCleanupCount())
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_OnDeleteEvent_directoryDeletion(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	folder := "/buckets/mybucket/testfolder"
	now := time.Now()

	// Simulate directory delete event - should trigger cleanup
	// because subdirectory deletion also makes parent potentially empty
	cleaner.OnDeleteEvent(folder, "subdir", true, now)

	// Check that cleanup IS queued for directory deletion
	if cleaner.GetPendingCleanupCount() != 1 {
		t.Errorf("directory deletion should trigger cleanup, got %d", cleaner.GetPendingCleanupCount())
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_cachedCounts(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	folder := "/buckets/mybucket/testfolder"

	// Initialize cached count
	cleaner.folderCounts[folder] = &folderState{roughCount: 5}

	// Simulate create events
	cleaner.OnCreateEvent(folder, "newfile1.txt", false)
	cleaner.OnCreateEvent(folder, "newfile2.txt", false)

	// Check cached count increased
	count, exists := cleaner.GetCachedFolderCount(folder)
	if !exists {
		t.Error("cached folder count should exist")
	}
	if count != 7 {
		t.Errorf("expected cached count 7, got %d", count)
	}

	// Simulate delete events
	now := time.Now()
	cleaner.OnDeleteEvent(folder, "file1.txt", false, now)
	cleaner.OnDeleteEvent(folder, "file2.txt", false, now.Add(1*time.Second))

	// Check cached count decreased
	count, exists = cleaner.GetCachedFolderCount(folder)
	if !exists {
		t.Error("cached folder count should exist")
	}
	if count != 5 {
		t.Errorf("expected cached count 5, got %d", count)
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_Stop(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	now := time.Now()

	// Queue some cleanups
	cleaner.OnDeleteEvent("/buckets/mybucket/folder1", "file1.txt", false, now)
	cleaner.OnDeleteEvent("/buckets/mybucket/folder2", "file2.txt", false, now.Add(1*time.Second))
	cleaner.OnDeleteEvent("/buckets/mybucket/folder3", "file3.txt", false, now.Add(2*time.Second))

	// Verify cleanups are queued
	if cleaner.GetPendingCleanupCount() < 1 {
		t.Error("expected at least 1 pending cleanup before stop")
	}

	// Stop the cleaner
	cleaner.Stop()

	// Verify all cleanups are cancelled
	if cleaner.GetPendingCleanupCount() != 0 {
		t.Errorf("expected 0 pending cleanups after stop, got %d", cleaner.GetPendingCleanupCount())
	}
}

func TestEmptyFolderCleaner_cacheEviction(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		cacheExpiry:  100 * time.Millisecond, // Short expiry for testing
		stopCh:       make(chan struct{}),
	}

	folder1 := "/buckets/mybucket/folder1"
	folder2 := "/buckets/mybucket/folder2"
	folder3 := "/buckets/mybucket/folder3"

	// Add some cache entries with old timestamps
	oldTime := time.Now().Add(-1 * time.Hour)
	cleaner.folderCounts[folder1] = &folderState{roughCount: 5, lastCheck: oldTime}
	cleaner.folderCounts[folder2] = &folderState{roughCount: 3, lastCheck: oldTime}
	// folder3 has recent activity
	cleaner.folderCounts[folder3] = &folderState{roughCount: 2, lastCheck: time.Now()}

	// Verify all entries exist
	if len(cleaner.folderCounts) != 3 {
		t.Errorf("expected 3 cache entries, got %d", len(cleaner.folderCounts))
	}

	// Run eviction
	cleaner.evictStaleCacheEntries()

	// Verify stale entries are evicted
	if len(cleaner.folderCounts) != 1 {
		t.Errorf("expected 1 cache entry after eviction, got %d", len(cleaner.folderCounts))
	}

	// Verify the recent entry still exists
	if _, exists := cleaner.folderCounts[folder3]; !exists {
		t.Error("expected folder3 to still exist in cache")
	}

	// Verify stale entries are removed
	if _, exists := cleaner.folderCounts[folder1]; exists {
		t.Error("expected folder1 to be evicted")
	}
	if _, exists := cleaner.folderCounts[folder2]; exists {
		t.Error("expected folder2 to be evicted")
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_cacheEviction_skipsEntriesInQueue(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		cacheExpiry:  100 * time.Millisecond,
		stopCh:       make(chan struct{}),
	}

	folder := "/buckets/mybucket/folder"
	oldTime := time.Now().Add(-1 * time.Hour)

	// Add a stale cache entry
	cleaner.folderCounts[folder] = &folderState{roughCount: 0, lastCheck: oldTime}
	// Also add to cleanup queue
	cleaner.cleanupQueue.Add(folder, "item", time.Now())

	// Run eviction
	cleaner.evictStaleCacheEntries()

	// Verify entry is NOT evicted because it's in cleanup queue
	if _, exists := cleaner.folderCounts[folder]; !exists {
		t.Error("expected folder to still exist in cache (is in cleanup queue)")
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_queueFIFOOrder(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	now := time.Now()

	// Add folders in order
	folders := []string{
		"/buckets/mybucket/folder1",
		"/buckets/mybucket/folder2",
		"/buckets/mybucket/folder3",
	}
	for i, folder := range folders {
		cleaner.OnDeleteEvent(folder, "file.txt", false, now.Add(time.Duration(i)*time.Second))
	}

	// Verify queue length
	if cleaner.GetPendingCleanupCount() != 3 {
		t.Errorf("expected 3 queued folders, got %d", cleaner.GetPendingCleanupCount())
	}

	// Verify time-sorted order by popping
	for i, expected := range folders {
		folder, _, ok := cleaner.cleanupQueue.Pop()
		if !ok || folder != expected {
			t.Errorf("expected folder %s at index %d, got %s", expected, i, folder)
		}
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_processCleanupQueue_onlyProcessesAgedItems(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	var deleted []string
	mock := &mockFilerOps{
		countFn: func(_ util.FullPath) (int, error) {
			return 0, nil
		},
		deleteFn: func(path util.FullPath) error {
			deleted = append(deleted, string(path))
			return nil
		},
	}

	maxAge := 100 * time.Millisecond
	cleaner := &EmptyFolderCleaner{
		filer:          mock,
		lockRing:       lockRing,
		host:           "filer1:8888",
		bucketPath:     "/buckets",
		enabled:        true,
		folderCounts:   make(map[string]*folderState),
		cleanupQueue:   NewCleanupQueue(1000, maxAge),
		maxCountCheck:  1000,
		cacheExpiry:    time.Minute,
		processorSleep: time.Second,
		stopCh:         make(chan struct{}),
	}

	// Add old items (well past maxAge) and a fresh item
	old := time.Now().Add(-time.Second)
	cleaner.cleanupQueue.Add("/buckets/test/folder1", "i1", old)
	cleaner.cleanupQueue.Add("/buckets/test/folder2", "i2", old.Add(time.Millisecond))
	cleaner.cleanupQueue.Add("/buckets/test/folder3", "i3", time.Now()) // fresh, should NOT be processed

	cleaner.processCleanupQueue()

	// Only the two old items should have been processed
	if len(deleted) != 2 {
		t.Fatalf("expected 2 deleted folders (aged items only), got %d: %v", len(deleted), deleted)
	}
	if got := cleaner.cleanupQueue.Len(); got != 1 {
		t.Fatalf("expected 1 item remaining in queue, got %d", got)
	}
}

func TestEmptyFolderCleaner_executeCleanup_bucketPolicyDisabledSkips(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	var deleted []string
	mock := &mockFilerOps{
		countFn: func(_ util.FullPath) (int, error) {
			return 0, nil
		},
		deleteFn: func(path util.FullPath) error {
			deleted = append(deleted, string(path))
			return nil
		},
		attrsFn: func(path util.FullPath) (map[string][]byte, error) {
			if string(path) == "/buckets/test" {
				return map[string][]byte{s3_constants.ExtAllowEmptyFolders: []byte("true")}, nil
			}
			return nil, nil
		},
	}

	cleaner := &EmptyFolderCleaner{
		filer:          mock,
		lockRing:       lockRing,
		host:           "filer1:8888",
		bucketPath:     "/buckets",
		enabled:        true,
		folderCounts:   make(map[string]*folderState),
		cleanupQueue:   NewCleanupQueue(1000, time.Minute),
		maxCountCheck:  1000,
		cacheExpiry:    time.Minute,
		processorSleep: time.Second,
		stopCh:         make(chan struct{}),
	}

	folder := "/buckets/test/folder"
	cleaner.executeCleanup(folder, "triggered_item")

	if len(deleted) != 0 {
		t.Fatalf("expected folder %s to be skipped, got deletions %v", folder, deleted)
	}
}

func TestEmptyFolderCleaner_executeCleanup_directoryMarker(t *testing.T) {
	testCases := []struct {
		name           string
		isDirKeyObj    bool
		expectDeletion bool
	}{
		{
			name:           "skips explicit directory marker",
			isDirKeyObj:    true,
			expectDeletion: false,
		},
		{
			name:           "deletes implicit empty folder",
			isDirKeyObj:    false,
			expectDeletion: true,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			lockRing := lock_manager.NewLockRing(5 * time.Second)
			lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

			var deleted []string
			mock := &mockFilerOps{
				countFn: func(_ util.FullPath) (int, error) {
					return 0, nil
				},
				deleteFn: func(path util.FullPath) error {
					deleted = append(deleted, string(path))
					return nil
				},
				isDirKeyObjFn: func(path util.FullPath) (bool, error) {
					return tc.isDirKeyObj, nil
				},
			}

			cleaner := &EmptyFolderCleaner{
				filer:          mock,
				lockRing:       lockRing,
				host:           "filer1:8888",
				bucketPath:     "/buckets",
				enabled:        true,
				folderCounts:   make(map[string]*folderState),
				cleanupQueue:   NewCleanupQueue(1000, time.Minute),
				maxCountCheck:  1000,
				cacheExpiry:    time.Minute,
				processorSleep: time.Second,
				stopCh:         make(chan struct{}),
			}

			folder := "/buckets/test/folder"
			cleaner.executeCleanup(folder, "triggered_item")

			if tc.expectDeletion {
				if len(deleted) != 1 || deleted[0] != folder {
					t.Fatalf("expected implicit empty folder %s to be deleted, got deletions %v", folder, deleted)
				}
			} else {
				if len(deleted) != 0 {
					t.Fatalf("expected explicit directory marker %s to be preserved, got deletions %v", folder, deleted)
				}
			}
		})
	}
}

func TestEmptyFolderCleaner_OnCreateEvent_cancelsCleanupForRecreatedDirectory(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		filer:          &mockFilerOps{},
		lockRing:       lockRing,
		host:           "filer1:8888",
		bucketPath:     "/buckets",
		enabled:        true,
		folderCounts:   make(map[string]*folderState),
		deleted:        make(map[string]*deletedFolder),
		cleanupQueue:   NewCleanupQueue(1000, time.Minute),
		maxCountCheck:  1000,
		cacheExpiry:    time.Minute,
		processorSleep: time.Second,
		stopCh:         make(chan struct{}),
	}

	// Dropping the table queues the folder its children were removed from, then a
	// rename puts a new table back under the same name.
	cleaner.OnDeleteEvent("/buckets/warehouse/pedsnet/person", "metadata", true, time.Now())
	if cleaner.GetPendingCleanupCount() != 1 {
		t.Fatalf("expected the emptied folder to be queued, got %d", cleaner.GetPendingCleanupCount())
	}

	cleaner.OnCreateEvent("/buckets/warehouse/pedsnet", "person", true)
	if cleaner.GetPendingCleanupCount() != 0 {
		t.Fatalf("expected the recreated folder to leave the queue, got %d", cleaner.GetPendingCleanupCount())
	}
}

func TestEmptyFolderCleaner_executeCleanup_skipsCatalogEntry(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	const namespace = "/buckets/warehouse/pedsnet"
	const table = namespace + "/person"

	var deleted []string
	mock := &mockFilerOps{
		countFn: func(_ util.FullPath) (int, error) {
			return 0, nil
		},
		deleteFn: func(path util.FullPath) error {
			deleted = append(deleted, string(path))
			return nil
		},
		attrsFn: func(path util.FullPath) (map[string][]byte, error) {
			if path == namespace || path == table {
				return map[string][]byte{s3_constants.ExtS3TablesPrefix + "metadata": []byte("{}")}, nil
			}
			return nil, nil
		},
	}

	cleaner := &EmptyFolderCleaner{
		filer:          mock,
		lockRing:       lockRing,
		host:           "filer1:8888",
		bucketPath:     "/buckets",
		enabled:        true,
		folderCounts:   make(map[string]*folderState),
		cleanupQueue:   NewCleanupQueue(1000, time.Minute),
		maxCountCheck:  1000,
		cacheExpiry:    time.Minute,
		processorSleep: time.Second,
		stopCh:         make(chan struct{}),
	}

	// The dropped table left its metadata folder empty; the cascade up from it must
	// stop at the table the rename put back, and at the namespace above it.
	cleaner.executeCleanup(table+"/metadata", "v2.metadata.json")

	if len(deleted) != 1 || deleted[0] != table+"/metadata" {
		t.Fatalf("expected only %s to be deleted, got %v", table+"/metadata", deleted)
	}
}

func TestEmptyFolderCleaner_restoreIfWrittenTo(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	newCleaner := func(mock *mockFilerOps) *EmptyFolderCleaner {
		return &EmptyFolderCleaner{
			filer:                 mock,
			lockRing:              lockRing,
			host:                  "filer1:8888",
			bucketPath:            "/buckets",
			enabled:               true,
			maxCountCheck:         DefaultMaxCountCheck,
			cacheExpiry:           DefaultCacheExpiry,
			folderCounts:          make(map[string]*folderState),
			bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
			deleted:               make(map[string]*deletedFolder),
			cleanupQueue:          NewCleanupQueue(1000, 10*time.Minute),
			stopCh:                make(chan struct{}),
		}
	}

	const folder = "/buckets/mybucket/a/b"
	const parent = "/buckets/mybucket/a"

	t.Run("an entry that landed during the delete is restored without waiting for its event", func(t *testing.T) {
		counts := map[string]int{}
		var deleted, restored []string
		mock := &mockFilerOps{
			countFn: func(p util.FullPath) (int, error) { return counts[string(p)], nil },
			deleteFn: func(p util.FullPath) error {
				deleted = append(deleted, string(p))
				if string(p) == folder {
					// the racing write lands between the delete's own emptiness
					// check and the removal of the folder entry
					counts[folder] = 1
				}
				return nil
			},
			ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "obj")

		if len(restored) != 1 || restored[0] != folder {
			t.Fatalf("folder written to during the delete should be restored, got %v", restored)
		}
		if len(deleted) != 1 || deleted[0] != folder {
			t.Fatalf("a restored folder leaves the parent non-empty, so it must not cascade, got %v", deleted)
		}
		if len(cleaner.deleted) != 0 {
			t.Fatalf("a restored folder should stop being observed, got %d", len(cleaner.deleted))
		}
	})

	t.Run("a folder still empty after the delete is not restored and the parent is cascaded", func(t *testing.T) {
		var deleted, restored []string
		mock := &mockFilerOps{
			countFn:     func(util.FullPath) (int, error) { return 0, nil },
			deleteFn:    func(p util.FullPath) error { deleted = append(deleted, string(p)); return nil },
			ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "obj")

		if len(restored) != 0 {
			t.Fatalf("an empty folder should not be restored, got %v", restored)
		}
		if len(deleted) != 2 || deleted[0] != folder || deleted[1] != parent {
			t.Fatalf("the parent should still be cascaded, got %v", deleted)
		}
	})

	t.Run("a restore that fails is left marked for the deferred pass", func(t *testing.T) {
		counts := map[string]int{}
		mock := &mockFilerOps{
			countFn: func(p util.FullPath) (int, error) { return counts[string(p)], nil },
			deleteFn: func(p util.FullPath) error {
				if string(p) == folder {
					counts[folder] = 1
				}
				return nil
			},
			ensureDirFn: func(util.FullPath, DirectoryAttributes) error { return errors.New("store unavailable") },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "obj")

		observed, found := cleaner.deleted[folder]
		if !found {
			t.Fatal("a folder whose restore failed must stay under observation")
		}
		if !observed.writtenTo {
			t.Fatal("a folder seen to hold an entry must be marked, or the deferred pass skips it")
		}
	})

	t.Run("a re-check that fails leaves the folder to the event path", func(t *testing.T) {
		var restored []string
		deletedFolder := false
		mock := &mockFilerOps{
			countFn: func(util.FullPath) (int, error) {
				if deletedFolder {
					return 0, errors.New("store unavailable")
				}
				return 0, nil
			},
			deleteFn:    func(p util.FullPath) error { deletedFolder = string(p) == folder; return nil },
			ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "obj")

		if len(restored) != 0 {
			t.Fatalf("a failed re-check should not restore, got %v", restored)
		}
		if _, found := cleaner.deleted[folder]; !found {
			t.Fatal("a folder whose re-check failed must stay under observation")
		}
	})
}