package empty_folder_cleanup import ( "context" "errors" "testing" "time" "github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/util" ) type mockFilerOps struct { countFn func(path util.FullPath) (int, error) deleteFn func(path util.FullPath) error attrsFn func(path util.FullPath) (map[string][]byte, error) isDirKeyObjFn func(path util.FullPath) (bool, error) ensureDirFn func(path util.FullPath, attrs DirectoryAttributes) error dirAttrsFn func(path util.FullPath) (DirectoryAttributes, error) } func (m *mockFilerOps) EnsureDirectoryEntry(_ context.Context, p util.FullPath, attrs DirectoryAttributes) error { if m.ensureDirFn == nil { return nil } return m.ensureDirFn(p, attrs) } func (m *mockFilerOps) DirectoryAttributes(_ context.Context, p util.FullPath) (DirectoryAttributes, error) { if m.dirAttrsFn == nil { return DirectoryAttributes{Mode: 0755}, nil } return m.dirAttrsFn(p) } func (m *mockFilerOps) CountDirectoryEntries(_ context.Context, dirPath util.FullPath, _ int) (int, error) { if m.countFn == nil { return 0, nil } return m.countFn(dirPath) } func (m *mockFilerOps) DeleteEntryMetaAndData(_ context.Context, p util.FullPath, _, _, _, _ bool, _ []int32, _ int64) error { if m.deleteFn == nil { return nil } return m.deleteFn(p) } func (m *mockFilerOps) GetEntryAttributes(_ context.Context, p util.FullPath) (map[string][]byte, error) { if m.attrsFn == nil { return nil, nil } return m.attrsFn(p) } func (m *mockFilerOps) IsDirectoryKeyObject(_ context.Context, p util.FullPath) (bool, error) { if m.isDirKeyObjFn == nil { return false, nil } return m.isDirKeyObjFn(p) } func Test_isUnderPath(t *testing.T) { tests := []struct { name string child string parent string expected bool }{ {"child under parent", "/buckets/mybucket/folder/file.txt", "/buckets", true}, {"child is parent", "/buckets", "/buckets", true}, {"child not under parent", "/other/path", "/buckets", false}, {"empty parent", "/any/path", "", true}, {"root parent", "/any/path", "/", true}, {"parent with trailing slash", "/buckets/mybucket", "/buckets/", true}, {"similar prefix but not under", "/buckets-other/file", "/buckets", false}, {"deeply nested", "/buckets/a/b/c/d/e/f", "/buckets/a/b", true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := isUnderPath(tt.child, tt.parent) if result != tt.expected { t.Errorf("isUnderPath(%q, %q) = %v, want %v", tt.child, tt.parent, result, tt.expected) } }) } } func Test_isUnderBucketPath(t *testing.T) { tests := []struct { name string directory string bucketPath string expected bool }{ // Should NOT process - bucket path itself {"bucket path itself", "/buckets", "/buckets", false}, // Should NOT process - bucket directory (immediate child) {"bucket directory", "/buckets/mybucket", "/buckets", false}, // Should process - folder inside bucket {"folder in bucket", "/buckets/mybucket/folder", "/buckets", true}, // Should process - nested folder {"nested folder", "/buckets/mybucket/a/b/c", "/buckets", true}, // Should NOT process - outside buckets {"outside buckets", "/other/path", "/buckets", false}, // Empty bucket path allows all {"empty bucket path", "/any/path", "", true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := isUnderBucketPath(tt.directory, tt.bucketPath) if result != tt.expected { t.Errorf("isUnderBucketPath(%q, %q) = %v, want %v", tt.directory, tt.bucketPath, result, tt.expected) } }) } } func Test_isMultipartUploadsPath(t *testing.T) { uploads := s3_constants.MultipartUploadsFolder tests := []struct { name string directory string bucketPath string expected bool }{ {"staging root", "/buckets/mybucket/" + uploads, "/buckets", true}, {"upload marker dir", "/buckets/mybucket/" + uploads + "/abc123", "/buckets", true}, {"nested under staging", "/buckets/mybucket/" + uploads + "/abc123/hashstates", "/buckets", true}, {"normal folder", "/buckets/mybucket/folder", "/buckets", false}, {"registry own uploads dir", "/buckets/mybucket/docker/registry/v2/_uploads/x", "/buckets", false}, {"bucket itself", "/buckets/mybucket", "/buckets", false}, {"outside buckets", "/other/" + uploads, "/buckets", false}, {"empty bucket path", "/buckets/mybucket/" + uploads, "", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := isMultipartUploadsPath(tt.bucketPath, tt.directory); got != tt.expected { t.Errorf("isMultipartUploadsPath(%q, %q) = %v, want %v", tt.bucketPath, tt.directory, got, tt.expected) } }) } } func TestEmptyFolderCleaner_OnDeleteEvent_skipsMultipartUploads(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) cleaner := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, folderCounts: make(map[string]*folderState), cleanupQueue: NewCleanupQueue(1000, 10*time.Minute), stopCh: make(chan struct{}), } now := time.Now() uploads := "/buckets/mybucket/" + s3_constants.MultipartUploadsFolder cleaner.OnDeleteEvent(uploads, "abc123", true, now) cleaner.OnDeleteEvent(uploads+"/abc123", "0001.part", false, now) if cleaner.GetPendingCleanupCount() != 0 { t.Fatalf("multipart staging paths must not be queued, got %d pending", cleaner.GetPendingCleanupCount()) } // A normal folder is still queued, proving the guard is scoped to .uploads. cleaner.OnDeleteEvent("/buckets/mybucket/folder", "file.txt", false, now) if cleaner.GetPendingCleanupCount() != 1 { t.Fatalf("normal folder should be queued, got %d pending", cleaner.GetPendingCleanupCount()) } cleaner.Stop() } func TestEmptyFolderCleaner_executeCleanup_skipsMultipartUploads(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) var deleted []string mock := &mockFilerOps{ countFn: func(util.FullPath) (int, error) { return 0, nil }, deleteFn: func(p util.FullPath) error { deleted = append(deleted, string(p)); return nil }, } cleaner := &EmptyFolderCleaner{ filer: mock, lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, maxCountCheck: DefaultMaxCountCheck, cacheExpiry: DefaultCacheExpiry, folderCounts: make(map[string]*folderState), bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState), cleanupQueue: NewCleanupQueue(1000, 10*time.Minute), stopCh: make(chan struct{}), } uploads := "/buckets/mybucket/" + s3_constants.MultipartUploadsFolder cleaner.executeCleanup(uploads, "abc123") cleaner.executeCleanup(uploads+"/abc123", "0001.part") if len(deleted) != 0 { t.Fatalf("multipart staging paths must not be deleted, got %v", deleted) } // An empty normal folder is deleted, proving the guard did not disable cleanup. cleaner.executeCleanup("/buckets/mybucket/folder", "file.txt") if len(deleted) != 1 || deleted[0] != "/buckets/mybucket/folder" { t.Fatalf("normal empty folder should be deleted, got %v", deleted) } } func TestEmptyFolderCleaner_restoreFoldersWrittenDuringDelete(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) newCleaner := func(mock *mockFilerOps) *EmptyFolderCleaner { return &EmptyFolderCleaner{ filer: mock, lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, maxCountCheck: DefaultMaxCountCheck, cacheExpiry: DefaultCacheExpiry, folderCounts: make(map[string]*folderState), bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState), deleted: make(map[string]*deletedFolder), cleanupQueue: NewCleanupQueue(1000, 10*time.Minute), stopCh: make(chan struct{}), } } const folder = "/buckets/mybucket/folder" t.Run("a create event for the deleted folder puts it back", func(t *testing.T) { var restored []string mock := &mockFilerOps{ countFn: func(util.FullPath) (int, error) { return 0, nil }, ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil }, } cleaner := newCleaner(mock) cleaner.executeCleanup(folder, "file.txt") // nothing has been written to it, so it is only observed cleaner.processCleanupQueue() if len(restored) != 0 { t.Fatalf("a folder nothing was written to should not be restored, got %v", restored) } if len(cleaner.deleted) != 1 { t.Fatalf("folder should still be observed, got %d", len(cleaner.deleted)) } // the write that raced the delete arrives as a create event mock.countFn = func(util.FullPath) (int, error) { return 1, nil } cleaner.OnCreateEvent(folder, "obj", false) cleaner.processCleanupQueue() if len(restored) != 1 || restored[0] != folder { t.Fatalf("a folder written to while being deleted should be restored, got %v", restored) } if len(cleaner.deleted) != 0 { t.Fatalf("a restored folder should stop being observed, got %d", len(cleaner.deleted)) } }) t.Run("a create event elsewhere is ignored", func(t *testing.T) { var restored []string mock := &mockFilerOps{ countFn: func(util.FullPath) (int, error) { return 0, nil }, ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil }, } cleaner := newCleaner(mock) cleaner.executeCleanup(folder, "file.txt") cleaner.OnCreateEvent("/buckets/mybucket/other", "obj", false) cleaner.processCleanupQueue() if len(restored) != 0 { t.Fatalf("only the folder written to should be restored, got %v", restored) } }) t.Run("an entry that went away again is not restored", func(t *testing.T) { var restored []string mock := &mockFilerOps{ countFn: func(util.FullPath) (int, error) { return 0, nil }, ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil }, } cleaner := newCleaner(mock) cleaner.executeCleanup(folder, "file.txt") cleaner.OnCreateEvent(folder, "obj", false) cleaner.processCleanupQueue() if len(restored) != 0 { t.Fatalf("an empty folder should not be restored, got %v", restored) } }) t.Run("observation expires", func(t *testing.T) { mock := &mockFilerOps{countFn: func(util.FullPath) (int, error) { return 0, nil }} cleaner := newCleaner(mock) cleaner.executeCleanup(folder, "file.txt") cleaner.processCleanupQueue() if len(cleaner.deleted) != 1 { t.Fatalf("folder should still be observed inside the window, got %d", len(cleaner.deleted)) } cleaner.deleted[folder].deletedAt = time.Now().Add(-DefaultObservationWindow - time.Second) cleaner.processCleanupQueue() if len(cleaner.deleted) != 0 { t.Fatalf("folder should stop being observed once the window has passed, got %d", len(cleaner.deleted)) } }) t.Run("a failed restore is retried", func(t *testing.T) { attempts := 0 mock := &mockFilerOps{ countFn: func(util.FullPath) (int, error) { return 1, nil }, ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { attempts++ if attempts == 1 { return errors.New("store unavailable") } return nil }, } cleaner := newCleaner(mock) // deleted while empty, then written to mock.countFn = func(util.FullPath) (int, error) { return 0, nil } cleaner.executeCleanup(folder, "file.txt") mock.countFn = func(util.FullPath) (int, error) { return 1, nil } cleaner.OnCreateEvent(folder, "obj", false) cleaner.processCleanupQueue() if len(cleaner.deleted) != 1 { t.Fatalf("a folder whose restore failed should be kept, got %d", len(cleaner.deleted)) } cleaner.processCleanupQueue() if attempts != 2 { t.Fatalf("a failed restore should be retried, got %d attempts", attempts) } if len(cleaner.deleted) != 0 { t.Fatalf("a restored folder should stop being observed, got %d", len(cleaner.deleted)) } }) t.Run("a folder whose restore keeps failing stops being observed", func(t *testing.T) { mock := &mockFilerOps{ countFn: func(util.FullPath) (int, error) { return 0, nil }, ensureDirFn: func(util.FullPath, DirectoryAttributes) error { return errors.New("store unavailable") }, } cleaner := newCleaner(mock) cleaner.executeCleanup(folder, "file.txt") mock.countFn = func(util.FullPath) (int, error) { return 1, nil } cleaner.OnCreateEvent(folder, "obj", false) cleaner.processCleanupQueue() if len(cleaner.deleted) != 1 { t.Fatalf("a folder whose restore failed should be kept, got %d", len(cleaner.deleted)) } // otherwise it would be retried on every pass for the rest of the process cleaner.deleted[folder].deletedAt = time.Now().Add(-DefaultObservationWindow - time.Second) cleaner.processCleanupQueue() if len(cleaner.deleted) != 0 { t.Fatalf("the window should apply to a failing folder too, got %d", len(cleaner.deleted)) } }) t.Run("an ancestor taken by the cascade is restored with its own attributes", func(t *testing.T) { const ancestor = "/buckets/mybucket/data" const nested = ancestor + "/abc" restored := map[string]DirectoryAttributes{} var order []string mock := &mockFilerOps{ countFn: func(util.FullPath) (int, error) { return 0, nil }, dirAttrsFn: func(p util.FullPath) (DirectoryAttributes, error) { if string(p) == ancestor { return DirectoryAttributes{Mode: 0700, Uid: 4242}, nil } return DirectoryAttributes{Mode: 0755, Uid: 7}, nil }, ensureDirFn: func(p util.FullPath, attrs DirectoryAttributes) error { restored[string(p)] = attrs order = append(order, string(p)) return nil }, } cleaner := newCleaner(mock) // the cascade takes the nested folder and then its parent cleaner.executeCleanup(nested, "file.txt") cleaner.executeCleanup(ancestor, "abc") // only the nested folder is written to mock.countFn = func(util.FullPath) (int, error) { return 1, nil } cleaner.OnCreateEvent(nested, "obj", false) cleaner.processCleanupQueue() if len(order) != 2 || order[0] != ancestor { t.Fatalf("the ancestor should be rebuilt first, got %v", order) } if got := restored[ancestor]; got.Mode != 0700 || got.Uid != 4242 { t.Errorf("ancestor should keep its own attributes, got mode %o uid %d", got.Mode, got.Uid) } }) t.Run("a delete that reports failure is still observed", func(t *testing.T) { // the redis stores drop the folder before its parent-list member, so a failure // return is not proof that the folder survived mock := &mockFilerOps{ countFn: func(util.FullPath) (int, error) { return 0, nil }, deleteFn: func(util.FullPath) error { return errors.New("partial delete") }, } cleaner := newCleaner(mock) cleaner.executeCleanup(folder, "file.txt") if len(cleaner.deleted) != 1 { t.Fatalf("a failed delete should still leave the folder observed, got %d", len(cleaner.deleted)) } }) } func Test_autoRemoveEmptyFoldersEnabled(t *testing.T) { tests := []struct { name string attrs map[string][]byte enabled bool attrValue string }{ { name: "no attrs defaults enabled", attrs: nil, enabled: true, attrValue: "", }, { name: "missing key defaults enabled", attrs: map[string][]byte{}, enabled: true, attrValue: "", }, { name: "allow-empty disables cleanup", attrs: map[string][]byte{ s3_constants.ExtAllowEmptyFolders: []byte("true"), }, enabled: false, attrValue: "true", }, { name: "explicit false keeps cleanup enabled", attrs: map[string][]byte{ s3_constants.ExtAllowEmptyFolders: []byte("false"), }, enabled: true, attrValue: "false", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { enabled, attrValue := autoRemoveEmptyFoldersEnabled(tt.attrs) if enabled != tt.enabled { t.Fatalf("expected enabled=%v, got %v", tt.enabled, enabled) } if attrValue != tt.attrValue { t.Fatalf("expected attrValue=%q, got %q", tt.attrValue, attrValue) } }) } } func TestEmptyFolderCleaner_ownsFolder(t *testing.T) { // Create a LockRing with multiple servers lockRing := lock_manager.NewLockRing(5 * time.Second) servers := []pb.ServerAddress{ "filer1:8888", "filer2:8888", "filer3:8888", } lockRing.SetSnapshot(servers, 0) // Create cleaner for filer1 cleaner1 := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer1:8888", } // Create cleaner for filer2 cleaner2 := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer2:8888", } // Create cleaner for filer3 cleaner3 := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer3:8888", } // Test that exactly one filer owns each folder testFolders := []string{ "/buckets/mybucket/folder1", "/buckets/mybucket/folder2", "/buckets/mybucket/folder3", "/buckets/mybucket/a/b/c", "/buckets/otherbucket/x", } for _, folder := range testFolders { ownCount := 0 if cleaner1.ownsFolder(folder) { ownCount++ } if cleaner2.ownsFolder(folder) { ownCount++ } if cleaner3.ownsFolder(folder) { ownCount++ } if ownCount != 1 { t.Errorf("folder %q owned by %d filers, expected exactly 1", folder, ownCount) } } } func TestEmptyFolderCleaner_ownsFolder_singleServer(t *testing.T) { // Create a LockRing with a single server lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) cleaner := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer1:8888", } // Single filer should own all folders testFolders := []string{ "/buckets/mybucket/folder1", "/buckets/mybucket/folder2", "/buckets/otherbucket/x", } for _, folder := range testFolders { if !cleaner.ownsFolder(folder) { t.Errorf("single filer should own folder %q", folder) } } } func TestEmptyFolderCleaner_ownsFolder_emptyRing(t *testing.T) { // Create an empty LockRing lockRing := lock_manager.NewLockRing(5 * time.Second) cleaner := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer1:8888", } // With empty ring, should own all folders if !cleaner.ownsFolder("/buckets/mybucket/folder") { t.Error("should own folder with empty ring") } } func TestEmptyFolderCleaner_OnCreateEvent_cancelsCleanup(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) cleaner := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, folderCounts: make(map[string]*folderState), cleanupQueue: NewCleanupQueue(1000, 10*time.Minute), stopCh: make(chan struct{}), } folder := "/buckets/mybucket/testfolder" now := time.Now() // Simulate delete event cleaner.OnDeleteEvent(folder, "file.txt", false, now) // Check that cleanup is queued if cleaner.GetPendingCleanupCount() != 1 { t.Errorf("expected 1 pending cleanup, got %d", cleaner.GetPendingCleanupCount()) } // Simulate create event cleaner.OnCreateEvent(folder, "newfile.txt", false) // Check that cleanup is cancelled if cleaner.GetPendingCleanupCount() != 0 { t.Errorf("expected 0 pending cleanups after create, got %d", cleaner.GetPendingCleanupCount()) } cleaner.Stop() } func TestEmptyFolderCleaner_OnDeleteEvent_deduplication(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) cleaner := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, folderCounts: make(map[string]*folderState), cleanupQueue: NewCleanupQueue(1000, 10*time.Minute), stopCh: make(chan struct{}), } folder := "/buckets/mybucket/testfolder" now := time.Now() // Simulate multiple delete events for same folder for i := 0; i < 5; i++ { cleaner.OnDeleteEvent(folder, "file"+string(rune('0'+i))+".txt", false, now.Add(time.Duration(i)*time.Second)) } // Check that only 1 cleanup is queued (deduplicated) if cleaner.GetPendingCleanupCount() != 1 { t.Errorf("expected 1 pending cleanup after deduplication, got %d", cleaner.GetPendingCleanupCount()) } cleaner.Stop() } func TestEmptyFolderCleaner_OnDeleteEvent_multipleFolders(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) cleaner := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, folderCounts: make(map[string]*folderState), cleanupQueue: NewCleanupQueue(1000, 10*time.Minute), stopCh: make(chan struct{}), } now := time.Now() // Delete files in different folders cleaner.OnDeleteEvent("/buckets/mybucket/folder1", "file.txt", false, now) cleaner.OnDeleteEvent("/buckets/mybucket/folder2", "file.txt", false, now.Add(1*time.Second)) cleaner.OnDeleteEvent("/buckets/mybucket/folder3", "file.txt", false, now.Add(2*time.Second)) // Each folder should be queued if cleaner.GetPendingCleanupCount() != 3 { t.Errorf("expected 3 pending cleanups, got %d", cleaner.GetPendingCleanupCount()) } cleaner.Stop() } func TestEmptyFolderCleaner_OnDeleteEvent_notOwner(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888", "filer2:8888"}, 0) // Create cleaner for filer that doesn't own the folder cleaner := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, folderCounts: make(map[string]*folderState), cleanupQueue: NewCleanupQueue(1000, 10*time.Minute), stopCh: make(chan struct{}), } now := time.Now() // Try many folders, looking for one that filer1 doesn't own foundNonOwned := false for i := 0; i < 100; i++ { folder := "/buckets/mybucket/folder" + string(rune('0'+i%10)) + string(rune('0'+i/10)) if !cleaner.ownsFolder(folder) { // This folder is not owned by filer1 cleaner.OnDeleteEvent(folder, "file.txt", false, now) if cleaner.GetPendingCleanupCount() != 0 { t.Errorf("non-owner should not queue cleanup for folder %s", folder) } foundNonOwned = true break } } if !foundNonOwned { t.Skip("could not find a folder not owned by filer1") } cleaner.Stop() } func TestEmptyFolderCleaner_OnDeleteEvent_disabled(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) cleaner := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: false, // Disabled folderCounts: make(map[string]*folderState), cleanupQueue: NewCleanupQueue(1000, 10*time.Minute), stopCh: make(chan struct{}), } folder := "/buckets/mybucket/testfolder" now := time.Now() // Simulate delete event cleaner.OnDeleteEvent(folder, "file.txt", false, now) // Check that no cleanup is queued when disabled if cleaner.GetPendingCleanupCount() != 0 { t.Errorf("disabled cleaner should not queue cleanup, got %d", cleaner.GetPendingCleanupCount()) } cleaner.Stop() } func TestEmptyFolderCleaner_OnDeleteEvent_directoryDeletion(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) cleaner := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, folderCounts: make(map[string]*folderState), cleanupQueue: NewCleanupQueue(1000, 10*time.Minute), stopCh: make(chan struct{}), } folder := "/buckets/mybucket/testfolder" now := time.Now() // Simulate directory delete event - should trigger cleanup // because subdirectory deletion also makes parent potentially empty cleaner.OnDeleteEvent(folder, "subdir", true, now) // Check that cleanup IS queued for directory deletion if cleaner.GetPendingCleanupCount() != 1 { t.Errorf("directory deletion should trigger cleanup, got %d", cleaner.GetPendingCleanupCount()) } cleaner.Stop() } func TestEmptyFolderCleaner_cachedCounts(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) cleaner := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, folderCounts: make(map[string]*folderState), cleanupQueue: NewCleanupQueue(1000, 10*time.Minute), stopCh: make(chan struct{}), } folder := "/buckets/mybucket/testfolder" // Initialize cached count cleaner.folderCounts[folder] = &folderState{roughCount: 5} // Simulate create events cleaner.OnCreateEvent(folder, "newfile1.txt", false) cleaner.OnCreateEvent(folder, "newfile2.txt", false) // Check cached count increased count, exists := cleaner.GetCachedFolderCount(folder) if !exists { t.Error("cached folder count should exist") } if count != 7 { t.Errorf("expected cached count 7, got %d", count) } // Simulate delete events now := time.Now() cleaner.OnDeleteEvent(folder, "file1.txt", false, now) cleaner.OnDeleteEvent(folder, "file2.txt", false, now.Add(1*time.Second)) // Check cached count decreased count, exists = cleaner.GetCachedFolderCount(folder) if !exists { t.Error("cached folder count should exist") } if count != 5 { t.Errorf("expected cached count 5, got %d", count) } cleaner.Stop() } func TestEmptyFolderCleaner_Stop(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) cleaner := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, folderCounts: make(map[string]*folderState), cleanupQueue: NewCleanupQueue(1000, 10*time.Minute), stopCh: make(chan struct{}), } now := time.Now() // Queue some cleanups cleaner.OnDeleteEvent("/buckets/mybucket/folder1", "file1.txt", false, now) cleaner.OnDeleteEvent("/buckets/mybucket/folder2", "file2.txt", false, now.Add(1*time.Second)) cleaner.OnDeleteEvent("/buckets/mybucket/folder3", "file3.txt", false, now.Add(2*time.Second)) // Verify cleanups are queued if cleaner.GetPendingCleanupCount() < 1 { t.Error("expected at least 1 pending cleanup before stop") } // Stop the cleaner cleaner.Stop() // Verify all cleanups are cancelled if cleaner.GetPendingCleanupCount() != 0 { t.Errorf("expected 0 pending cleanups after stop, got %d", cleaner.GetPendingCleanupCount()) } } func TestEmptyFolderCleaner_cacheEviction(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) cleaner := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, folderCounts: make(map[string]*folderState), cleanupQueue: NewCleanupQueue(1000, 10*time.Minute), cacheExpiry: 100 * time.Millisecond, // Short expiry for testing stopCh: make(chan struct{}), } folder1 := "/buckets/mybucket/folder1" folder2 := "/buckets/mybucket/folder2" folder3 := "/buckets/mybucket/folder3" // Add some cache entries with old timestamps oldTime := time.Now().Add(-1 * time.Hour) cleaner.folderCounts[folder1] = &folderState{roughCount: 5, lastCheck: oldTime} cleaner.folderCounts[folder2] = &folderState{roughCount: 3, lastCheck: oldTime} // folder3 has recent activity cleaner.folderCounts[folder3] = &folderState{roughCount: 2, lastCheck: time.Now()} // Verify all entries exist if len(cleaner.folderCounts) != 3 { t.Errorf("expected 3 cache entries, got %d", len(cleaner.folderCounts)) } // Run eviction cleaner.evictStaleCacheEntries() // Verify stale entries are evicted if len(cleaner.folderCounts) != 1 { t.Errorf("expected 1 cache entry after eviction, got %d", len(cleaner.folderCounts)) } // Verify the recent entry still exists if _, exists := cleaner.folderCounts[folder3]; !exists { t.Error("expected folder3 to still exist in cache") } // Verify stale entries are removed if _, exists := cleaner.folderCounts[folder1]; exists { t.Error("expected folder1 to be evicted") } if _, exists := cleaner.folderCounts[folder2]; exists { t.Error("expected folder2 to be evicted") } cleaner.Stop() } func TestEmptyFolderCleaner_cacheEviction_skipsEntriesInQueue(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) cleaner := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, folderCounts: make(map[string]*folderState), cleanupQueue: NewCleanupQueue(1000, 10*time.Minute), cacheExpiry: 100 * time.Millisecond, stopCh: make(chan struct{}), } folder := "/buckets/mybucket/folder" oldTime := time.Now().Add(-1 * time.Hour) // Add a stale cache entry cleaner.folderCounts[folder] = &folderState{roughCount: 0, lastCheck: oldTime} // Also add to cleanup queue cleaner.cleanupQueue.Add(folder, "item", time.Now()) // Run eviction cleaner.evictStaleCacheEntries() // Verify entry is NOT evicted because it's in cleanup queue if _, exists := cleaner.folderCounts[folder]; !exists { t.Error("expected folder to still exist in cache (is in cleanup queue)") } cleaner.Stop() } func TestEmptyFolderCleaner_queueFIFOOrder(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) cleaner := &EmptyFolderCleaner{ lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, folderCounts: make(map[string]*folderState), cleanupQueue: NewCleanupQueue(1000, 10*time.Minute), stopCh: make(chan struct{}), } now := time.Now() // Add folders in order folders := []string{ "/buckets/mybucket/folder1", "/buckets/mybucket/folder2", "/buckets/mybucket/folder3", } for i, folder := range folders { cleaner.OnDeleteEvent(folder, "file.txt", false, now.Add(time.Duration(i)*time.Second)) } // Verify queue length if cleaner.GetPendingCleanupCount() != 3 { t.Errorf("expected 3 queued folders, got %d", cleaner.GetPendingCleanupCount()) } // Verify time-sorted order by popping for i, expected := range folders { folder, _, ok := cleaner.cleanupQueue.Pop() if !ok || folder != expected { t.Errorf("expected folder %s at index %d, got %s", expected, i, folder) } } cleaner.Stop() } func TestEmptyFolderCleaner_processCleanupQueue_onlyProcessesAgedItems(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) var deleted []string mock := &mockFilerOps{ countFn: func(_ util.FullPath) (int, error) { return 0, nil }, deleteFn: func(path util.FullPath) error { deleted = append(deleted, string(path)) return nil }, } maxAge := 100 * time.Millisecond cleaner := &EmptyFolderCleaner{ filer: mock, lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, folderCounts: make(map[string]*folderState), cleanupQueue: NewCleanupQueue(1000, maxAge), maxCountCheck: 1000, cacheExpiry: time.Minute, processorSleep: time.Second, stopCh: make(chan struct{}), } // Add old items (well past maxAge) and a fresh item old := time.Now().Add(-time.Second) cleaner.cleanupQueue.Add("/buckets/test/folder1", "i1", old) cleaner.cleanupQueue.Add("/buckets/test/folder2", "i2", old.Add(time.Millisecond)) cleaner.cleanupQueue.Add("/buckets/test/folder3", "i3", time.Now()) // fresh, should NOT be processed cleaner.processCleanupQueue() // Only the two old items should have been processed if len(deleted) != 2 { t.Fatalf("expected 2 deleted folders (aged items only), got %d: %v", len(deleted), deleted) } if got := cleaner.cleanupQueue.Len(); got != 1 { t.Fatalf("expected 1 item remaining in queue, got %d", got) } } func TestEmptyFolderCleaner_executeCleanup_bucketPolicyDisabledSkips(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) var deleted []string mock := &mockFilerOps{ countFn: func(_ util.FullPath) (int, error) { return 0, nil }, deleteFn: func(path util.FullPath) error { deleted = append(deleted, string(path)) return nil }, attrsFn: func(path util.FullPath) (map[string][]byte, error) { if string(path) == "/buckets/test" { return map[string][]byte{s3_constants.ExtAllowEmptyFolders: []byte("true")}, nil } return nil, nil }, } cleaner := &EmptyFolderCleaner{ filer: mock, lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, folderCounts: make(map[string]*folderState), cleanupQueue: NewCleanupQueue(1000, time.Minute), maxCountCheck: 1000, cacheExpiry: time.Minute, processorSleep: time.Second, stopCh: make(chan struct{}), } folder := "/buckets/test/folder" cleaner.executeCleanup(folder, "triggered_item") if len(deleted) != 0 { t.Fatalf("expected folder %s to be skipped, got deletions %v", folder, deleted) } } func TestEmptyFolderCleaner_executeCleanup_directoryMarker(t *testing.T) { testCases := []struct { name string isDirKeyObj bool expectDeletion bool }{ { name: "skips explicit directory marker", isDirKeyObj: true, expectDeletion: false, }, { name: "deletes implicit empty folder", isDirKeyObj: false, expectDeletion: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) var deleted []string mock := &mockFilerOps{ countFn: func(_ util.FullPath) (int, error) { return 0, nil }, deleteFn: func(path util.FullPath) error { deleted = append(deleted, string(path)) return nil }, isDirKeyObjFn: func(path util.FullPath) (bool, error) { return tc.isDirKeyObj, nil }, } cleaner := &EmptyFolderCleaner{ filer: mock, lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, folderCounts: make(map[string]*folderState), cleanupQueue: NewCleanupQueue(1000, time.Minute), maxCountCheck: 1000, cacheExpiry: time.Minute, processorSleep: time.Second, stopCh: make(chan struct{}), } folder := "/buckets/test/folder" cleaner.executeCleanup(folder, "triggered_item") if tc.expectDeletion { if len(deleted) != 1 || deleted[0] != folder { t.Fatalf("expected implicit empty folder %s to be deleted, got deletions %v", folder, deleted) } } else { if len(deleted) != 0 { t.Fatalf("expected explicit directory marker %s to be preserved, got deletions %v", folder, deleted) } } }) } } func TestEmptyFolderCleaner_OnCreateEvent_cancelsCleanupForRecreatedDirectory(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) cleaner := &EmptyFolderCleaner{ filer: &mockFilerOps{}, lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, folderCounts: make(map[string]*folderState), deleted: make(map[string]*deletedFolder), cleanupQueue: NewCleanupQueue(1000, time.Minute), maxCountCheck: 1000, cacheExpiry: time.Minute, processorSleep: time.Second, stopCh: make(chan struct{}), } // Dropping the table queues the folder its children were removed from, then a // rename puts a new table back under the same name. cleaner.OnDeleteEvent("/buckets/warehouse/pedsnet/person", "metadata", true, time.Now()) if cleaner.GetPendingCleanupCount() != 1 { t.Fatalf("expected the emptied folder to be queued, got %d", cleaner.GetPendingCleanupCount()) } cleaner.OnCreateEvent("/buckets/warehouse/pedsnet", "person", true) if cleaner.GetPendingCleanupCount() != 0 { t.Fatalf("expected the recreated folder to leave the queue, got %d", cleaner.GetPendingCleanupCount()) } } func TestEmptyFolderCleaner_executeCleanup_skipsCatalogEntry(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) const namespace = "/buckets/warehouse/pedsnet" const table = namespace + "/person" var deleted []string mock := &mockFilerOps{ countFn: func(_ util.FullPath) (int, error) { return 0, nil }, deleteFn: func(path util.FullPath) error { deleted = append(deleted, string(path)) return nil }, attrsFn: func(path util.FullPath) (map[string][]byte, error) { if path == namespace || path == table { return map[string][]byte{s3_constants.ExtS3TablesPrefix + "metadata": []byte("{}")}, nil } return nil, nil }, } cleaner := &EmptyFolderCleaner{ filer: mock, lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, folderCounts: make(map[string]*folderState), cleanupQueue: NewCleanupQueue(1000, time.Minute), maxCountCheck: 1000, cacheExpiry: time.Minute, processorSleep: time.Second, stopCh: make(chan struct{}), } // The dropped table left its metadata folder empty; the cascade up from it must // stop at the table the rename put back, and at the namespace above it. cleaner.executeCleanup(table+"/metadata", "v2.metadata.json") if len(deleted) != 1 || deleted[0] != table+"/metadata" { t.Fatalf("expected only %s to be deleted, got %v", table+"/metadata", deleted) } } func TestEmptyFolderCleaner_restoreIfWrittenTo(t *testing.T) { lockRing := lock_manager.NewLockRing(5 * time.Second) lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0) newCleaner := func(mock *mockFilerOps) *EmptyFolderCleaner { return &EmptyFolderCleaner{ filer: mock, lockRing: lockRing, host: "filer1:8888", bucketPath: "/buckets", enabled: true, maxCountCheck: DefaultMaxCountCheck, cacheExpiry: DefaultCacheExpiry, folderCounts: make(map[string]*folderState), bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState), deleted: make(map[string]*deletedFolder), cleanupQueue: NewCleanupQueue(1000, 10*time.Minute), stopCh: make(chan struct{}), } } const folder = "/buckets/mybucket/a/b" const parent = "/buckets/mybucket/a" t.Run("an entry that landed during the delete is restored without waiting for its event", func(t *testing.T) { counts := map[string]int{} var deleted, restored []string mock := &mockFilerOps{ countFn: func(p util.FullPath) (int, error) { return counts[string(p)], nil }, deleteFn: func(p util.FullPath) error { deleted = append(deleted, string(p)) if string(p) == folder { // the racing write lands between the delete's own emptiness // check and the removal of the folder entry counts[folder] = 1 } return nil }, ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil }, } cleaner := newCleaner(mock) cleaner.executeCleanup(folder, "obj") if len(restored) != 1 || restored[0] != folder { t.Fatalf("folder written to during the delete should be restored, got %v", restored) } if len(deleted) != 1 || deleted[0] != folder { t.Fatalf("a restored folder leaves the parent non-empty, so it must not cascade, got %v", deleted) } if len(cleaner.deleted) != 0 { t.Fatalf("a restored folder should stop being observed, got %d", len(cleaner.deleted)) } }) t.Run("a folder still empty after the delete is not restored and the parent is cascaded", func(t *testing.T) { var deleted, restored []string mock := &mockFilerOps{ countFn: func(util.FullPath) (int, error) { return 0, nil }, deleteFn: func(p util.FullPath) error { deleted = append(deleted, string(p)); return nil }, ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil }, } cleaner := newCleaner(mock) cleaner.executeCleanup(folder, "obj") if len(restored) != 0 { t.Fatalf("an empty folder should not be restored, got %v", restored) } if len(deleted) != 2 || deleted[0] != folder || deleted[1] != parent { t.Fatalf("the parent should still be cascaded, got %v", deleted) } }) t.Run("a restore that fails is left marked for the deferred pass", func(t *testing.T) { counts := map[string]int{} mock := &mockFilerOps{ countFn: func(p util.FullPath) (int, error) { return counts[string(p)], nil }, deleteFn: func(p util.FullPath) error { if string(p) == folder { counts[folder] = 1 } return nil }, ensureDirFn: func(util.FullPath, DirectoryAttributes) error { return errors.New("store unavailable") }, } cleaner := newCleaner(mock) cleaner.executeCleanup(folder, "obj") observed, found := cleaner.deleted[folder] if !found { t.Fatal("a folder whose restore failed must stay under observation") } if !observed.writtenTo { t.Fatal("a folder seen to hold an entry must be marked, or the deferred pass skips it") } }) t.Run("a re-check that fails leaves the folder to the event path", func(t *testing.T) { var restored []string deletedFolder := false mock := &mockFilerOps{ countFn: func(util.FullPath) (int, error) { if deletedFolder { return 0, errors.New("store unavailable") } return 0, nil }, deleteFn: func(p util.FullPath) error { deletedFolder = string(p) == folder; return nil }, ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil }, } cleaner := newCleaner(mock) cleaner.executeCleanup(folder, "obj") if len(restored) != 0 { t.Fatalf("a failed re-check should not restore, got %v", restored) } if _, found := cleaner.deleted[folder]; !found { t.Fatal("a folder whose re-check failed must stay under observation") } }) }