upstream-seaweedfs/weed/server/filer_grpc_server_stream_mutate_scheduler_test.go

package weed_server

import (
	"sync/atomic"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

// TestMutatePathAncestors mirrors TestPathAncestors in filer_sync_jobs_test.go.
func TestMutatePathAncestors(t *testing.T) {
	tests := []struct {
		path util.FullPath
		want []util.FullPath
	}{
		{"/a/b/c/file.txt", []util.FullPath{"/a/b/c", "/a/b", "/a", "/"}},
		{"/a/b", []util.FullPath{"/a", "/"}},
		{"/a", []util.FullPath{"/"}},
		{"/", nil},
	}
	for _, tt := range tests {
		got := mutatePathAncestors(tt.path)
		if len(got) != len(tt.want) {
			t.Errorf("mutatePathAncestors(%q) = %v, want %v", tt.path, got, tt.want)
			continue
		}
		for i := range got {
			if got[i] != tt.want[i] {
				t.Errorf("mutatePathAncestors(%q)[%d] = %q, want %q", tt.path, i, got[i], tt.want[i])
			}
		}
	}
}

// TestClassifyMutation covers the four request shapes and the delete-recursive
// barrier upgrade.
func TestClassifyMutation(t *testing.T) {
	tests := []struct {
		name          string
		req           *filer_pb.StreamMutateEntryRequest
		wantPrimary   util.FullPath
		wantSecondary util.FullPath
		wantKind      mutateJobKind
	}{
		{
			name: "create file",
			req: &filer_pb.StreamMutateEntryRequest{
				Request: &filer_pb.StreamMutateEntryRequest_CreateRequest{
					CreateRequest: &filer_pb.CreateEntryRequest{
						Directory: "/a",
						Entry:     &filer_pb.Entry{Name: "f", IsDirectory: false},
					},
				},
			},
			wantPrimary: "/a/f",
			wantKind:    kindMutateFile,
		},
		{
			name: "create directory (barrier)",
			req: &filer_pb.StreamMutateEntryRequest{
				Request: &filer_pb.StreamMutateEntryRequest_CreateRequest{
					CreateRequest: &filer_pb.CreateEntryRequest{
						Directory: "/a",
						Entry:     &filer_pb.Entry{Name: "d", IsDirectory: true},
					},
				},
			},
			wantPrimary: "/a/d",
			wantKind:    kindMutateBarrierDir,
		},
		{
			name: "update file",
			req: &filer_pb.StreamMutateEntryRequest{
				Request: &filer_pb.StreamMutateEntryRequest_UpdateRequest{
					UpdateRequest: &filer_pb.UpdateEntryRequest{
						Directory: "/a",
						Entry:     &filer_pb.Entry{Name: "f", IsDirectory: false},
					},
				},
			},
			wantPrimary: "/a/f",
			wantKind:    kindMutateFile,
		},
		{
			name: "update directory (non-barrier attr bump)",
			req: &filer_pb.StreamMutateEntryRequest{
				Request: &filer_pb.StreamMutateEntryRequest_UpdateRequest{
					UpdateRequest: &filer_pb.UpdateEntryRequest{
						Directory: "/",
						Entry:     &filer_pb.Entry{Name: "a", IsDirectory: true},
					},
				},
			},
			wantPrimary: "/a",
			wantKind:    kindMutateNonBarrierDir,
		},
		{
			name: "delete non-recursive (barrier: target type unknown)",
			req: &filer_pb.StreamMutateEntryRequest{
				Request: &filer_pb.StreamMutateEntryRequest_DeleteRequest{
					DeleteRequest: &filer_pb.DeleteEntryRequest{
						Directory: "/a", Name: "f",
					},
				},
			},
			wantPrimary: "/a/f",
			wantKind:    kindMutateBarrierDir,
		},
		{
			name: "delete recursive (barrier)",
			req: &filer_pb.StreamMutateEntryRequest{
				Request: &filer_pb.StreamMutateEntryRequest_DeleteRequest{
					DeleteRequest: &filer_pb.DeleteEntryRequest{
						Directory: "/a", Name: "d", IsRecursive: true,
					},
				},
			},
			wantPrimary: "/a/d",
			wantKind:    kindMutateBarrierDir,
		},
		{
			name: "malformed create (nil Entry)",
			req: &filer_pb.StreamMutateEntryRequest{
				Request: &filer_pb.StreamMutateEntryRequest_CreateRequest{
					CreateRequest: &filer_pb.CreateEntryRequest{Directory: "/a"},
				},
			},
			wantPrimary: "/",
			wantKind:    kindMutateBarrierDir,
		},
		{
			name: "malformed update (nil Entry)",
			req: &filer_pb.StreamMutateEntryRequest{
				Request: &filer_pb.StreamMutateEntryRequest_UpdateRequest{
					UpdateRequest: &filer_pb.UpdateEntryRequest{Directory: "/a"},
				},
			},
			wantPrimary: "/",
			wantKind:    kindMutateBarrierDir,
		},
		{
			name:        "empty oneof",
			req:         &filer_pb.StreamMutateEntryRequest{},
			wantPrimary: "/",
			wantKind:    kindMutateBarrierDir,
		},
		{
			name: "rename",
			req: &filer_pb.StreamMutateEntryRequest{
				Request: &filer_pb.StreamMutateEntryRequest_RenameRequest{
					RenameRequest: &filer_pb.StreamRenameEntryRequest{
						OldDirectory: "/src", OldName: "a",
						NewDirectory: "/dst", NewName: "b",
					},
				},
			},
			wantPrimary:   "/src/a",
			wantSecondary: "/dst/b",
			wantKind:      kindMutateBarrierDir,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			p, s, k := classifyMutation(tt.req)
			if p != tt.wantPrimary {
				t.Errorf("primary = %q, want %q", p, tt.wantPrimary)
			}
			if s != tt.wantSecondary {
				t.Errorf("secondary = %q, want %q", s, tt.wantSecondary)
			}
			if k != tt.wantKind {
				t.Errorf("kind = %v, want %v", k, tt.wantKind)
			}
		})
	}
}

// TestPathConflictSamePathFile: two file ops on the same path conflict; on
// different paths they do not.
func TestPathConflictSamePathFile(t *testing.T) {
	s := newMutateScheduler(100)
	s.addPathLocked("/a/f", kindMutateFile)

	if !s.pathConflictsLocked("/a/f", kindMutateFile) {
		t.Error("same-path file+file should conflict")
	}
	if s.pathConflictsLocked("/a/g", kindMutateFile) {
		t.Error("different-path file+file should not conflict")
	}
}

// TestPathConflictBarrierBlocksSamePath: a barrier dir in flight at /a blocks
// any new job (file, barrier, non-barrier) at /a.
func TestPathConflictBarrierBlocksSamePath(t *testing.T) {
	s := newMutateScheduler(100)
	s.addPathLocked("/a", kindMutateBarrierDir)

	for _, k := range []mutateJobKind{kindMutateFile, kindMutateBarrierDir, kindMutateNonBarrierDir} {
		if !s.pathConflictsLocked("/a", k) {
			t.Errorf("barrier at /a should block kind %v at /a", k)
		}
	}
}

// TestPathConflictBarrierBlocksDescendants: a barrier at /a blocks any job
// strictly under /a, regardless of kind.
func TestPathConflictBarrierBlocksDescendants(t *testing.T) {
	s := newMutateScheduler(100)
	s.addPathLocked("/a", kindMutateBarrierDir)

	for _, child := range []util.FullPath{"/a/f", "/a/b", "/a/b/c"} {
		for _, k := range []mutateJobKind{kindMutateFile, kindMutateBarrierDir, kindMutateNonBarrierDir} {
			if !s.pathConflictsLocked(child, k) {
				t.Errorf("barrier at /a should block %v at %q", k, child)
			}
		}
	}
}

// TestPathConflictIncomingBarrierWaitsForDescendants: a new barrier at /a must
// wait for any active descendant (even a file) to drain.
func TestPathConflictIncomingBarrierWaitsForDescendants(t *testing.T) {
	s := newMutateScheduler(100)
	s.addPathLocked("/a/f", kindMutateFile)

	if !s.pathConflictsLocked("/a", kindMutateBarrierDir) {
		t.Error("incoming barrier at /a should wait for active file at /a/f")
	}
}

// TestPathConflictNonBarrierDirAllowsDescendants: an in-flight attribute-only
// dir update at /a does NOT block descendants. This is the filer.sync
// optimization that prevents mtime bumps from serializing file writes.
func TestPathConflictNonBarrierDirAllowsDescendants(t *testing.T) {
	s := newMutateScheduler(100)
	s.addPathLocked("/a", kindMutateNonBarrierDir)

	if s.pathConflictsLocked("/a/f", kindMutateFile) {
		t.Error("non-barrier dir update at /a should not block file at /a/f")
	}
	if s.pathConflictsLocked("/a/b", kindMutateNonBarrierDir) {
		t.Error("non-barrier dir update at /a should not block another non-barrier at /a/b")
	}
}

// TestPathConflictIncomingBarrierWaitsForSamePathNonBarrier: a delete/rename on
// a dir must wait for an in-flight chmod/xattr/mtime update at the same dir.
func TestPathConflictIncomingBarrierWaitsForSamePathNonBarrier(t *testing.T) {
	s := newMutateScheduler(100)
	s.addPathLocked("/a", kindMutateNonBarrierDir)

	if !s.pathConflictsLocked("/a", kindMutateBarrierDir) {
		t.Error("incoming barrier at /a should wait for non-barrier update at /a")
	}
	// But non-barrier vs non-barrier at the same path may overlap (attr bumps).
	if s.pathConflictsLocked("/a", kindMutateNonBarrierDir) {
		t.Error("non-barrier vs non-barrier at /a should not conflict")
	}
}

// TestAdmitDoneLifecycle: Admit marks state; Done unmarks; a subsequent
// conflicting Admit that was blocked must unblock after Done.
func TestAdmitDoneLifecycle(t *testing.T) {
	s := newMutateScheduler(100)
	s.Admit("/a/f", "", kindMutateFile)
	// State is registered.
	s.mu.Lock()
	if s.activeFilePaths["/a/f"] != 1 {
		s.mu.Unlock()
		t.Fatalf("activeFilePaths[/a/f] = %d, want 1", s.activeFilePaths["/a/f"])
	}
	s.mu.Unlock()

	// Start a blocked admit on the same path; it must not complete until Done.
	admitted := make(chan struct{})
	go func() {
		s.Admit("/a/f", "", kindMutateFile)
		close(admitted)
	}()
	select {
	case <-admitted:
		t.Fatal("second Admit should block while first is active")
	case <-time.After(50 * time.Millisecond):
	}

	s.Done("/a/f", "", kindMutateFile)

	select {
	case <-admitted:
	case <-time.After(time.Second):
		t.Fatal("second Admit did not unblock after Done")
	}

	// Clean up.
	s.Done("/a/f", "", kindMutateFile)
}

// TestAdmitParallelDistinctPaths: different paths do not block each other.
func TestAdmitParallelDistinctPaths(t *testing.T) {
	s := newMutateScheduler(100)
	done := make(chan struct{}, 3)
	for i, p := range []util.FullPath{"/a", "/b", "/c"} {
		go func(p util.FullPath, i int) {
			s.Admit(p, "", kindMutateFile)
			done <- struct{}{}
		}(p, i)
	}
	for i := 0; i < 3; i++ {
		select {
		case <-done:
		case <-time.After(time.Second):
			t.Fatal("distinct-path Admits blocked each other")
		}
	}
	for _, p := range []util.FullPath{"/a", "/b", "/c"} {
		s.Done(p, "", kindMutateFile)
	}
}

// TestAdmitConcurrencyCap: with concurrencyLimit=2 and three Admits, the
// third must block until one of the first two calls Done.
func TestAdmitConcurrencyCap(t *testing.T) {
	s := newMutateScheduler(2)
	s.Admit("/a", "", kindMutateFile)
	s.Admit("/b", "", kindMutateFile)

	admittedThird := make(chan struct{})
	go func() {
		s.Admit("/c", "", kindMutateFile)
		close(admittedThird)
	}()
	select {
	case <-admittedThird:
		t.Fatal("third Admit should block when cap is reached")
	case <-time.After(50 * time.Millisecond):
	}

	s.Done("/a", "", kindMutateFile)
	select {
	case <-admittedThird:
	case <-time.After(time.Second):
		t.Fatal("third Admit did not unblock after Done")
	}

	s.Done("/b", "", kindMutateFile)
	s.Done("/c", "", kindMutateFile)
}

// TestAdmitRenameTwoPathConflict: a rename holds both src and dst. A later
// single-path admit on either must block until Done clears both.
func TestAdmitRenameTwoPathConflict(t *testing.T) {
	s := newMutateScheduler(100)
	s.Admit("/src/a", "/dst/b", kindMutateBarrierDir)

	for _, blocked := range []util.FullPath{"/src/a", "/dst/b"} {
		admitted := make(chan struct{})
		go func(p util.FullPath) {
			s.Admit(p, "", kindMutateFile)
			close(admitted)
		}(blocked)
		select {
		case <-admitted:
			t.Fatalf("Admit at %q should block while rename holds both paths", blocked)
		case <-time.After(20 * time.Millisecond):
		}
	}

	s.Done("/src/a", "/dst/b", kindMutateBarrierDir)

	// Two waiters now become admissible; drain them.
	deadline := time.After(time.Second)
	drained := 0
	for drained < 2 {
		select {
		case <-deadline:
			t.Fatalf("only %d waiters unblocked after Done", drained)
		default:
			// Attempt to Done both waiters; they may still be racing to Admit.
			time.Sleep(10 * time.Millisecond)
			s.mu.Lock()
			active := s.totalActive
			s.mu.Unlock()
			if active == 2 {
				s.Done("/src/a", "", kindMutateFile)
				s.Done("/dst/b", "", kindMutateFile)
				drained = 2
			}
		}
	}
}

// waitQueueLen blocks until pathQueue[p] has exactly n entries, or fails the
// test on timeout. Uses the scheduler's own mutex to observe state so the
// caller never needs a sleep to "let things settle".
func waitQueueLen(t *testing.T, s *mutateScheduler, p util.FullPath, n int) {
	t.Helper()
	deadline := time.Now().Add(time.Second)
	for {
		s.mu.Lock()
		got := len(s.pathQueue[p])
		s.mu.Unlock()
		if got == n {
			return
		}
		if time.Now().After(deadline) {
			t.Fatalf("pathQueue[%q] len = %d, want %d (timeout)", p, got, n)
		}
		time.Sleep(100 * time.Microsecond)
	}
}

// TestAdmitSamePathFIFO verifies arrival order is preserved for same-path
// admits. Regression test for the Cond.Broadcast race where later admits
// could be woken and registered before earlier ones.
//
// The arrival ordering is made deterministic by observing the scheduler's
// per-path queue length between spawns — no sleep-based fudging.
func TestAdmitSamePathFIFO(t *testing.T) {
	s := newMutateScheduler(100)

	// A barrier on /a holds the whole path while we enqueue N waiters.
	// After this call pathQueue["/a"] has 1 entry (the barrier holder).
	s.Admit("/a", "", kindMutateBarrierDir)

	const N = 20
	order := make(chan int, N)
	for i := 0; i < N; i++ {
		i := i
		go func() {
			s.Admit("/a", "", kindMutateFile)
			order <- i
			s.Done("/a", "", kindMutateFile)
		}()
		// Before spawning the next goroutine, wait until this one has
		// observably enqueued itself. The barrier holder plus i+1 file
		// waiters should be present on /a.
		waitQueueLen(t, s, "/a", 1+i+1)
	}

	// Still held — nothing admitted yet.
	select {
	case got := <-order:
		t.Fatalf("unexpected early admit %d while /a is held", got)
	case <-time.After(20 * time.Millisecond):
	}

	s.Done("/a", "", kindMutateBarrierDir)

	for i := 0; i < N; i++ {
		select {
		case got := <-order:
			if got != i {
				t.Fatalf("admit order[%d] = %d, want %d (FIFO violated)", i, got, i)
			}
		case <-time.After(time.Second):
			t.Fatalf("only %d/%d admits completed", i, N)
		}
	}
}

// TestAdmitSamePathNonBarrierSerializes verifies that two non-barrier dir
// updates at the same path no longer overlap (filer.sync's last-writer-wins
// optimization is intentionally dropped for streamed mutations, which carry
// client-submitted operations whose order matters).
func TestAdmitSamePathNonBarrierSerializes(t *testing.T) {
	s := newMutateScheduler(100)
	s.Admit("/a", "", kindMutateNonBarrierDir)

	second := make(chan struct{})
	go func() {
		s.Admit("/a", "", kindMutateNonBarrierDir)
		close(second)
	}()
	select {
	case <-second:
		t.Fatal("second non-barrier dir update should not run concurrently with first at same path")
	case <-time.After(30 * time.Millisecond):
	}
	s.Done("/a", "", kindMutateNonBarrierDir)
	select {
	case <-second:
	case <-time.After(time.Second):
		t.Fatal("second non-barrier admit did not unblock after first Done")
	}
	s.Done("/a", "", kindMutateNonBarrierDir)
}

// TestAdmitPressureFromManyWaiters: 100 goroutines all want /a; exactly one at
// a time is active; all 100 eventually complete. This is both a smoke test for
// the broadcast/signal wake-up and a guard against lost wake-ups.
func TestAdmitPressureFromManyWaiters(t *testing.T) {
	s := newMutateScheduler(10) // cap higher than peak needed — path conflict should be the bottleneck
	const N = 100

	var activeAtOnce atomic.Int32
	var peak atomic.Int32
	done := make(chan struct{}, N)

	for i := 0; i < N; i++ {
		go func() {
			s.Admit("/a", "", kindMutateFile)
			a := activeAtOnce.Add(1)
			for {
				p := peak.Load()
				if a <= p || peak.CompareAndSwap(p, a) {
					break
				}
			}
			time.Sleep(time.Millisecond) // keep admission held so overlap is observable
			activeAtOnce.Add(-1)
			s.Done("/a", "", kindMutateFile)
			done <- struct{}{}
		}()
	}

	for i := 0; i < N; i++ {
		select {
		case <-done:
		case <-time.After(10 * time.Second):
			t.Fatalf("only %d/%d admits completed — possible lost wake-up", i, N)
		}
	}
	if peak.Load() != 1 {
		t.Errorf("peak concurrent same-path admits = %d, want 1", peak.Load())
	}
}