snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/mount/meta_cache/meta_cache_deadlock_test.go

@@ -0,0 +1,161 @@
+package meta_cache
+
+import (
+	"context"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// TestApplyLoopInvalidateDoesNotDeadlockWithLockHoldingEnqueuer reproduces the
+// mount hang seen in production (all readdirs stuck, FUSE waiting>0):
+//
+//   - A FUSE flush handler (flushMetadataToFiler) holds the file handle's
+//     exclusive lock in wfs.fhLockTable and then enqueues a local metadata
+//     event, waiting for the apply loop to process it.
+//   - Concurrently the apply loop processes a subscriber event whose
+//     invalidateFunc (weedfs.go) acquires the same file handle lock.
+//
+// The single-goroutine apply loop blocks on the fh lock while the fh lock
+// holder blocks on the apply loop: a permanent ABBA deadlock. Every later
+// EnsureVisited/BeginDirectoryBuild then backs up behind the dead loop, FUSE
+// handler goroutines exhaust, and the kernel queue (waiting) grows.
+//
+// The contract under test: the apply loop must never block on locks that can
+// be held by goroutines waiting on the apply loop.
+func TestApplyLoopInvalidateDoesNotDeadlockWithLockHoldingEnqueuer(t *testing.T) {
+	mapper, err := NewUidGidMapper("", "")
+	if err != nil {
+		t.Fatalf("uid/gid mapper: %v", err)
+	}
+
+	// Stand-in for wfs.fhLockTable, keyed like FileHandleId.
+	fhLockTable := util.NewLockTable[uint64]()
+	const fhKey uint64 = 42
+
+	invalidateEntered := make(chan struct{})
+	var enteredOnce sync.Once
+
+	var cachedMu sync.Mutex
+	cached := map[util.FullPath]bool{"/": true, "/dir": true}
+
+	mc := NewMetaCache(
+		filepath.Join(t.TempDir(), "meta"),
+		mapper,
+		util.FullPath("/"),
+		false,
+		func(path util.FullPath) {
+			cachedMu.Lock()
+			defer cachedMu.Unlock()
+			cached[path] = true
+		},
+		func(path util.FullPath) bool {
+			cachedMu.Lock()
+			defer cachedMu.Unlock()
+			return cached[path]
+		},
+		func(inv EntryInvalidation) {
+			// Mirrors the wfs invalidateFunc: it takes the open file
+			// handle's exclusive lock before refreshing the handle entry.
+			enteredOnce.Do(func() { close(invalidateEntered) })
+			lock := fhLockTable.AcquireLock("invalidateFunc", fhKey, util.ExclusiveLock)
+			fhLockTable.ReleaseLock(fhKey, lock)
+		},
+		func(dir util.FullPath) {},
+	)
+	defer func() {
+		// Only safe to shut down once the apply loop is unwedged.
+		mc.Shutdown()
+	}()
+
+	subscriberEvent := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name: "file.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    1,
+					FileMode: 0100644,
+				},
+			},
+		},
+	}
+	localEvent := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/dir",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{Name: "file.txt"},
+			NewEntry: &filer_pb.Entry{
+				Name: "file.txt",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    2,
+					FileMode: 0100644,
+				},
+			},
+			NewParentPath: "/dir",
+		},
+	}
+
+	// Step 1: the "flush handler" takes the fh exclusive lock
+	// (flushMetadataToFiler does this before talking to the filer).
+	flushLock := fhLockTable.AcquireLock("doFlush", fhKey, util.ExclusiveLock)
+	flushLockReleased := false
+	releaseFlushLock := func() {
+		if !flushLockReleased {
+			flushLockReleased = true
+			fhLockTable.ReleaseLock(fhKey, flushLock)
+		}
+	}
+	defer releaseFlushLock()
+
+	// Step 2: a subscriber event arrives; the apply loop will run
+	// invalidateFunc, which wants the fh lock held above.
+	subscriberDone := make(chan error, 1)
+	go func() {
+		subscriberDone <- mc.ApplyMetadataResponse(context.Background(), subscriberEvent, SubscriberMetadataResponseApplyOptions)
+	}()
+
+	select {
+	case <-invalidateEntered:
+	case <-time.After(5 * time.Second):
+		t.Fatal("invalidateFunc was never invoked for the subscriber event")
+	}
+	// Give the invalidate goroutine a moment to actually block on AcquireLock.
+	time.Sleep(50 * time.Millisecond)
+
+	// Step 3: still holding the fh lock, the flush handler applies its local
+	// metadata event and waits for the apply loop — exactly what
+	// flushMetadataToFiler does after streamCreateEntry succeeds.
+	flushApplyDone := make(chan error, 1)
+	go func() {
+		flushApplyDone <- mc.ApplyMetadataResponseOwned(context.Background(), localEvent, LocalMetadataResponseApplyOptions)
+	}()
+
+	select {
+	case err := <-flushApplyDone:
+		if err != nil {
+			t.Fatalf("local metadata apply: %v", err)
+		}
+	case <-time.After(3 * time.Second):
+		// Unwedge the apply loop so Shutdown in the deferred cleanup
+		// doesn't hang the whole test binary.
+		releaseFlushLock()
+		t.Fatal("deadlock: apply loop blocked on fh lock while fh lock holder waits on apply loop")
+	}
+
+	releaseFlushLock()
+
+	select {
+	case err := <-subscriberDone:
+		if err != nil {
+			t.Fatalf("subscriber apply: %v", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("subscriber apply never completed")
+	}
+}