snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/server/filer_grpc_server_stream_mutate_bench_test.go

@@ -0,0 +1,792 @@
+package weed_server
+
+import (
+	"context"
+	"io"
+	"net"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// fakeFilerServer simulates a filer whose CreateEntry takes a fixed amount of
+// wall-clock time (meant to model the filer-store commit latency). It implements
+// only what these benchmarks need: unary CreateEntry and bidi StreamMutateEntry.
+//
+// The StreamMutateEntry handler mirrors the production handler's structure:
+// a single goroutine per stream, processing one request at a time.
+type fakeFilerServer struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+	serviceDelay time.Duration
+	createCalls  atomic.Int64
+}
+
+func (s *fakeFilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
+	s.createCalls.Add(1)
+	if s.serviceDelay > 0 {
+		time.Sleep(s.serviceDelay)
+	}
+	return &filer_pb.CreateEntryResponse{}, nil
+}
+
+// StreamMutateEntry mirrors the OLD serial handler: one goroutine, strictly
+// one request at a time.
+func (s *fakeFilerServer) StreamMutateEntry(stream grpc.BidiStreamingServer[filer_pb.StreamMutateEntryRequest, filer_pb.StreamMutateEntryResponse]) error {
+	for {
+		req, err := stream.Recv()
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		switch r := req.Request.(type) {
+		case *filer_pb.StreamMutateEntryRequest_CreateRequest:
+			resp, createErr := s.CreateEntry(stream.Context(), r.CreateRequest)
+			if createErr != nil {
+				resp = &filer_pb.CreateEntryResponse{Error: createErr.Error()}
+			}
+			out := &filer_pb.StreamMutateEntryResponse{
+				RequestId: req.RequestId,
+				IsLast:    true,
+				Response:  &filer_pb.StreamMutateEntryResponse_CreateResponse{CreateResponse: resp},
+			}
+			if resp.Error != "" {
+				out.Error = resp.Error
+				out.Errno = int32(syscall.EIO)
+			}
+			if sendErr := stream.Send(out); sendErr != nil {
+				return sendErr
+			}
+		}
+	}
+}
+
+// fakeConcurrentFilerServer mirrors the NEW concurrent handler: per-request
+// goroutine, bounded by a semaphore, with a Send mutex for stream safety.
+// Structurally identical to weed/server/filer_grpc_server_stream_mutate.go
+// after the parallelization patch.
+type fakeConcurrentFilerServer struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+	serviceDelay time.Duration
+	concurrency  int
+	createCalls  atomic.Int64
+}
+
+func (s *fakeConcurrentFilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
+	s.createCalls.Add(1)
+	if s.serviceDelay > 0 {
+		time.Sleep(s.serviceDelay)
+	}
+	return &filer_pb.CreateEntryResponse{}, nil
+}
+
+func (s *fakeConcurrentFilerServer) StreamMutateEntry(stream grpc.BidiStreamingServer[filer_pb.StreamMutateEntryRequest, filer_pb.StreamMutateEntryResponse]) error {
+	var sendMu sync.Mutex
+	send := func(r *filer_pb.StreamMutateEntryResponse) error {
+		sendMu.Lock()
+		defer sendMu.Unlock()
+		return stream.Send(r)
+	}
+	sem := make(chan struct{}, s.concurrency)
+	var wg sync.WaitGroup
+	for {
+		req, err := stream.Recv()
+		if err == io.EOF {
+			wg.Wait()
+			return nil
+		}
+		if err != nil {
+			wg.Wait()
+			return err
+		}
+		sem <- struct{}{}
+		wg.Add(1)
+		go func(req *filer_pb.StreamMutateEntryRequest) {
+			defer wg.Done()
+			defer func() { <-sem }()
+			switch r := req.Request.(type) {
+			case *filer_pb.StreamMutateEntryRequest_CreateRequest:
+				resp, _ := s.CreateEntry(stream.Context(), r.CreateRequest)
+				_ = send(&filer_pb.StreamMutateEntryResponse{
+					RequestId: req.RequestId,
+					IsLast:    true,
+					Response:  &filer_pb.StreamMutateEntryResponse_CreateResponse{CreateResponse: resp},
+				})
+			}
+		}(req)
+	}
+}
+
+// startFakeConcurrentFilerServer spins up the concurrent-handler variant.
+func startFakeConcurrentFilerServer(t testing.TB, serviceDelay time.Duration, concurrency int) (string, *fakeConcurrentFilerServer, func()) {
+	t.Helper()
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	srv := grpc.NewServer()
+	fake := &fakeConcurrentFilerServer{serviceDelay: serviceDelay, concurrency: concurrency}
+	filer_pb.RegisterSeaweedFilerServer(srv, fake)
+	go srv.Serve(lis)
+	return lis.Addr().String(), fake, func() {
+		srv.GracefulStop()
+		_ = lis.Close()
+	}
+}
+
+// fakeSchedulerFilerServer uses the real mutateScheduler from the production
+// handler to admit requests by (path, kind). Per-path operations serialize;
+// cross-path operations run in parallel.
+type fakeSchedulerFilerServer struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+	serviceDelay time.Duration
+	concurrency  int
+	createCalls  atomic.Int64
+	maxInFlight  atomic.Int64 // sampled peak of in-flight create goroutines
+	curInFlight  atomic.Int64
+}
+
+func (s *fakeSchedulerFilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
+	s.createCalls.Add(1)
+	cur := s.curInFlight.Add(1)
+	for {
+		peak := s.maxInFlight.Load()
+		if cur <= peak || s.maxInFlight.CompareAndSwap(peak, cur) {
+			break
+		}
+	}
+	if s.serviceDelay > 0 {
+		time.Sleep(s.serviceDelay)
+	}
+	s.curInFlight.Add(-1)
+	return &filer_pb.CreateEntryResponse{}, nil
+}
+
+func (s *fakeSchedulerFilerServer) StreamMutateEntry(stream grpc.BidiStreamingServer[filer_pb.StreamMutateEntryRequest, filer_pb.StreamMutateEntryResponse]) error {
+	var sendMu sync.Mutex
+	send := func(r *filer_pb.StreamMutateEntryResponse) error {
+		sendMu.Lock()
+		defer sendMu.Unlock()
+		return stream.Send(r)
+	}
+	sched := newMutateScheduler(s.concurrency)
+	var wg sync.WaitGroup
+	for {
+		req, err := stream.Recv()
+		if err == io.EOF {
+			wg.Wait()
+			return nil
+		}
+		if err != nil {
+			wg.Wait()
+			return err
+		}
+		primary, secondary, kind := classifyMutation(req)
+		sched.Admit(primary, secondary, kind)
+		wg.Add(1)
+		go func(req *filer_pb.StreamMutateEntryRequest) {
+			defer wg.Done()
+			defer sched.Done(primary, secondary, kind)
+			switch r := req.Request.(type) {
+			case *filer_pb.StreamMutateEntryRequest_CreateRequest:
+				resp, _ := s.CreateEntry(stream.Context(), r.CreateRequest)
+				_ = send(&filer_pb.StreamMutateEntryResponse{
+					RequestId: req.RequestId,
+					IsLast:    true,
+					Response:  &filer_pb.StreamMutateEntryResponse_CreateResponse{CreateResponse: resp},
+				})
+			}
+		}(req)
+	}
+}
+
+func startFakeSchedulerFilerServer(t testing.TB, serviceDelay time.Duration, concurrency int) (string, *fakeSchedulerFilerServer, func()) {
+	t.Helper()
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	srv := grpc.NewServer()
+	fake := &fakeSchedulerFilerServer{serviceDelay: serviceDelay, concurrency: concurrency}
+	filer_pb.RegisterSeaweedFilerServer(srv, fake)
+	go srv.Serve(lis)
+	return lis.Addr().String(), fake, func() {
+		srv.GracefulStop()
+		_ = lis.Close()
+	}
+}
+
+// startFakeFilerServer spins up an in-process gRPC filer with the given per-
+// request service delay, returns the dial target and a shutdown func.
+func startFakeFilerServer(t testing.TB, serviceDelay time.Duration) (string, *fakeFilerServer, func()) {
+	t.Helper()
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	srv := grpc.NewServer()
+	fake := &fakeFilerServer{serviceDelay: serviceDelay}
+	filer_pb.RegisterSeaweedFilerServer(srv, fake)
+	go srv.Serve(lis)
+	return lis.Addr().String(), fake, func() {
+		srv.GracefulStop()
+		_ = lis.Close()
+	}
+}
+
+// dialFakeFiler returns a client connection to the in-process filer.
+func dialFakeFiler(t testing.TB, addr string) *grpc.ClientConn {
+	t.Helper()
+	conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("dial: %v", err)
+	}
+	return conn
+}
+
+// opsForWorker splits `ops` across `concurrency` workers, handing the
+// remainder to the first `remainder` workers so the total is always exactly
+// `ops`. Returns the count for worker index g.
+func opsForWorker(g, concurrency, ops int) int {
+	per := ops / concurrency
+	if g < ops%concurrency {
+		return per + 1
+	}
+	return per
+}
+
+// runUnaryCreateWorkload fires `ops` CreateEntry unary RPCs across `concurrency`
+// goroutines, all sharing a single client connection. Returns total duration.
+func runUnaryCreateWorkload(t testing.TB, conn *grpc.ClientConn, concurrency, ops int) time.Duration {
+	t.Helper()
+	client := filer_pb.NewSeaweedFilerClient(conn)
+	var wg sync.WaitGroup
+	start := time.Now()
+	for g := 0; g < concurrency; g++ {
+		wg.Add(1)
+		count := opsForWorker(g, concurrency, ops)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < count; i++ {
+				_, err := client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
+					Directory: "/",
+					Entry: &filer_pb.Entry{
+						Name:        "f",
+						IsDirectory: false,
+					},
+				})
+				if err != nil {
+					t.Errorf("unary CreateEntry: %v", err)
+					return
+				}
+			}
+		}()
+	}
+	wg.Wait()
+	return time.Since(start)
+}
+
+// pathFn chooses the (directory, name) of the i'th op. The default used by the
+// existing callers is samePath — all ops hit "/f". distinctPath gives every
+// op its own name so cross-path parallelism can kick in.
+type pathFn func(i int) (dir, name string)
+
+func samePath(_ int) (string, string)     { return "/", "f" }
+func distinctPath(i int) (string, string) { return "/", "f" + itoa(i) }
+
+// itoa is a tight local decimal formatter — avoids a strconv import inside a
+// hot benchmark loop.
+func itoa(n int) string {
+	if n == 0 {
+		return "0"
+	}
+	buf := make([]byte, 0, 12)
+	for n > 0 {
+		buf = append(buf, byte('0'+n%10))
+		n /= 10
+	}
+	for i, j := 0, len(buf)-1; i < j; i, j = i+1, j-1 {
+		buf[i], buf[j] = buf[j], buf[i]
+	}
+	return string(buf)
+}
+
+// runStreamCreateWorkload fires `ops` CreateEntry ops through a SINGLE bidi
+// stream, with `concurrency` goroutines all submitting to the same stream via
+// a small multiplexer that mirrors weed/mount/weedfs_stream_mutate.go's
+// sendLoop+recvLoop structure. Returns total duration.
+func runStreamCreateWorkload(t testing.TB, conn *grpc.ClientConn, concurrency, ops int) time.Duration {
+	return runStreamCreateWorkloadAt(t, conn, concurrency, ops, samePath)
+}
+
+func runStreamCreateWorkloadAt(t testing.TB, conn *grpc.ClientConn, concurrency, ops int, path pathFn) time.Duration {
+	t.Helper()
+	client := filer_pb.NewSeaweedFilerClient(conn)
+	stream, err := client.StreamMutateEntry(context.Background())
+	if err != nil {
+		t.Fatalf("open stream: %v", err)
+	}
+
+	var nextID atomic.Uint64
+	var pending sync.Map // map[uint64]chan struct{}
+
+	// single recv goroutine, dispatching by request id
+	recvErr := make(chan error, 1)
+	go func() {
+		for {
+			resp, err := stream.Recv()
+			if err != nil {
+				recvErr <- err
+				return
+			}
+			if ch, ok := pending.LoadAndDelete(resp.RequestId); ok {
+				close(ch.(chan struct{}))
+			}
+		}
+	}()
+
+	// dedicated send goroutine (gRPC Send is not concurrent-safe)
+	sendCh := make(chan *filer_pb.StreamMutateEntryRequest, 512)
+	sendErr := make(chan error, 1)
+	go func() {
+		for req := range sendCh {
+			if err := stream.Send(req); err != nil {
+				sendErr <- err
+				return
+			}
+		}
+		sendErr <- nil
+	}()
+
+	var wg sync.WaitGroup
+	start := time.Now()
+	offset := 0
+	for g := 0; g < concurrency; g++ {
+		wg.Add(1)
+		count := opsForWorker(g, concurrency, ops)
+		startOff := offset
+		offset += count
+		go func(startOff, count int) {
+			defer wg.Done()
+			for i := 0; i < count; i++ {
+				id := nextID.Add(1)
+				done := make(chan struct{})
+				pending.Store(id, done)
+				dir, name := path(startOff + i)
+				sendCh <- &filer_pb.StreamMutateEntryRequest{
+					RequestId: id,
+					Request: &filer_pb.StreamMutateEntryRequest_CreateRequest{
+						CreateRequest: &filer_pb.CreateEntryRequest{
+							Directory: dir,
+							Entry:     &filer_pb.Entry{Name: name},
+						},
+					},
+				}
+				<-done
+			}
+		}(startOff, count)
+	}
+	wg.Wait()
+	elapsed := time.Since(start)
+
+	close(sendCh)
+	_ = stream.CloseSend()
+	select {
+	case <-sendErr:
+	case <-time.After(time.Second):
+	}
+	select {
+	case <-recvErr:
+	case <-time.After(time.Second):
+	}
+	return elapsed
+}
+
+// runStreamAsyncCreateWorkload fires `ops` CreateEntry ops through a single
+// bidi stream WITHOUT the client waiting for responses between sends. The
+// timer starts at the first send and stops when the last response is received,
+// so this measures true end-to-end throughput (not just client-send rate).
+//
+// This isolates the question: does relaxing client-side per-request waiting
+// help throughput when the server is a serial loop?
+func runStreamAsyncCreateWorkload(t testing.TB, conn *grpc.ClientConn, concurrency, ops int) time.Duration {
+	t.Helper()
+	client := filer_pb.NewSeaweedFilerClient(conn)
+	stream, err := client.StreamMutateEntry(context.Background())
+	if err != nil {
+		t.Fatalf("open stream: %v", err)
+	}
+
+	var nextID atomic.Uint64
+	var outstanding atomic.Int64
+	allDone := make(chan struct{})
+
+	// Single recv goroutine — signals allDone when the last response arrives.
+	recvErr := make(chan error, 1)
+	go func() {
+		for {
+			_, err := stream.Recv()
+			if err != nil {
+				recvErr <- err
+				return
+			}
+			if outstanding.Add(-1) == 0 {
+				select {
+				case <-allDone:
+				default:
+					close(allDone)
+				}
+				return
+			}
+		}
+	}()
+
+	// Dedicated send goroutine (gRPC Send is not concurrent-safe).
+	sendCh := make(chan *filer_pb.StreamMutateEntryRequest, 4096)
+	sendErr := make(chan error, 1)
+	go func() {
+		for req := range sendCh {
+			if err := stream.Send(req); err != nil {
+				sendErr <- err
+				return
+			}
+		}
+		sendErr <- nil
+	}()
+
+	outstanding.Store(int64(ops))
+
+	var wg sync.WaitGroup
+	start := time.Now()
+	for g := 0; g < concurrency; g++ {
+		wg.Add(1)
+		count := opsForWorker(g, concurrency, ops)
+		go func(count int) {
+			defer wg.Done()
+			for i := 0; i < count; i++ {
+				id := nextID.Add(1)
+				// Fire-and-forget: push to sendCh, do not wait for response.
+				sendCh <- &filer_pb.StreamMutateEntryRequest{
+					RequestId: id,
+					Request: &filer_pb.StreamMutateEntryRequest_CreateRequest{
+						CreateRequest: &filer_pb.CreateEntryRequest{
+							Directory: "/",
+							Entry:     &filer_pb.Entry{Name: "f"},
+						},
+					},
+				}
+			}
+		}(count)
+	}
+	wg.Wait()
+	// All requests queued; wait for server to drain all responses.
+	<-allDone
+	elapsed := time.Since(start)
+
+	close(sendCh)
+	_ = stream.CloseSend()
+	select {
+	case <-sendErr:
+	case <-time.After(time.Second):
+	}
+	select {
+	case <-recvErr:
+	case <-time.After(time.Second):
+	}
+	return elapsed
+}
+
+// TestReproStreamSerializationCeiling runs a head-to-head: same concurrency,
+// same op count, same per-request service delay. The only difference is the
+// transport — unary (one goroutine per RPC at the server) vs single bidi stream
+// (one goroutine for the whole stream at the server).
+//
+// With a 2ms per-request service delay, unary with N=12 should deliver ~6000/s
+// (12 concurrent handlers, each 2ms), while stream delivers ~500/s (serial).
+func TestReproStreamSerializationCeiling(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping repro in -short mode")
+	}
+
+	const (
+		concurrency  = 12
+		opsPerRun    = 1200
+		serviceDelay = 2 * time.Millisecond
+	)
+
+	addr, fake, stop := startFakeFilerServer(t, serviceDelay)
+	defer stop()
+
+	conn := dialFakeFiler(t, addr)
+	defer conn.Close()
+
+	// warm up one call so the gRPC HTTP/2 connection is fully established
+	warmupClient := filer_pb.NewSeaweedFilerClient(conn)
+	if _, err := warmupClient.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
+		Directory: "/",
+		Entry:     &filer_pb.Entry{Name: "warmup"},
+	}); err != nil {
+		t.Fatalf("warmup: %v", err)
+	}
+	fake.createCalls.Store(0)
+
+	unaryDur := runUnaryCreateWorkload(t, conn, concurrency, opsPerRun)
+	unaryCalls := fake.createCalls.Load()
+	fake.createCalls.Store(0)
+
+	streamDur := runStreamCreateWorkload(t, conn, concurrency, opsPerRun)
+	streamCalls := fake.createCalls.Load()
+
+	unaryQPS := float64(unaryCalls) / unaryDur.Seconds()
+	streamQPS := float64(streamCalls) / streamDur.Seconds()
+
+	t.Logf("service delay:       %v", serviceDelay)
+	t.Logf("concurrency:         %d", concurrency)
+	t.Logf("ops per run:         %d", opsPerRun)
+	t.Logf("unary  : %d calls in %v -> %.0f QPS", unaryCalls, unaryDur, unaryQPS)
+	t.Logf("stream : %d calls in %v -> %.0f QPS", streamCalls, streamDur, streamQPS)
+	t.Logf("unary / stream ratio: %.2fx", unaryQPS/streamQPS)
+
+	// The serial stream handler cannot exceed 1 / serviceDelay regardless of
+	// client concurrency. Assert that unary is meaningfully faster — if the
+	// server is ever parallelized, this assertion is safe to revisit.
+	if unaryQPS < streamQPS*2 {
+		t.Fatalf("expected unary to be at least 2x faster than stream, got unary=%.0f stream=%.0f",
+			unaryQPS, streamQPS)
+	}
+
+	// Stream QPS should cluster near the theoretical ceiling of 1/serviceDelay.
+	theoreticalMax := 1.0 / serviceDelay.Seconds()
+	if streamQPS > theoreticalMax*1.25 {
+		t.Fatalf("stream QPS %.0f exceeds theoretical serial ceiling %.0f — handler may have changed",
+			streamQPS, theoreticalMax)
+	}
+}
+
+// TestServerSerialVsConcurrentHandler measures the lift from changing the
+// server-side handler from serial (one goroutine per stream) to concurrent
+// (per-request goroutine, bounded by a semaphore, Send protected by a mutex).
+//
+// Client is identical in both runs (sync stream, N concurrent submitters).
+// Only the server handler differs.
+func TestServerSerialVsConcurrentHandler(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping repro in -short mode")
+	}
+
+	const (
+		opsPerRun         = 2400
+		serviceDelay      = 2 * time.Millisecond
+		serverConcurrency = 64
+	)
+
+	// Serial server (the OLD handler).
+	serialAddr, serialFake, stopSerial := startFakeFilerServer(t, serviceDelay)
+	defer stopSerial()
+	serialConn := dialFakeFiler(t, serialAddr)
+	defer serialConn.Close()
+
+	// Concurrent server (the NEW handler).
+	concAddr, concFake, stopConc := startFakeConcurrentFilerServer(t, serviceDelay, serverConcurrency)
+	defer stopConc()
+	concConn := dialFakeFiler(t, concAddr)
+	defer concConn.Close()
+
+	// Warm up both connections.
+	for _, c := range []*grpc.ClientConn{serialConn, concConn} {
+		client := filer_pb.NewSeaweedFilerClient(c)
+		if _, err := client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
+			Directory: "/",
+			Entry:     &filer_pb.Entry{Name: "warmup"},
+		}); err != nil {
+			t.Fatalf("warmup: %v", err)
+		}
+	}
+
+	clientConcurrencies := []int{1, 12, 64, 256}
+	theoreticalSerial := 1.0 / serviceDelay.Seconds()
+	theoreticalConcurrent := float64(serverConcurrency) / serviceDelay.Seconds()
+
+	t.Logf("service delay:               %v", serviceDelay)
+	t.Logf("server-side concurrency cap: %d", serverConcurrency)
+	t.Logf("theoretical serial ceiling:  %.0f QPS", theoreticalSerial)
+	t.Logf("theoretical concurrent cap:  %.0f QPS", theoreticalConcurrent)
+	t.Logf("ops per run:                 %d", opsPerRun)
+	t.Logf("")
+	t.Logf("%-16s %-14s %-14s %-10s",
+		"client workers", "serial QPS", "concurrent QPS", "lift")
+
+	for _, c := range clientConcurrencies {
+		serialFake.createCalls.Store(0)
+		serialDur := runStreamCreateWorkload(t, serialConn, c, opsPerRun)
+		serialCalls := serialFake.createCalls.Load()
+		serialQPS := float64(serialCalls) / serialDur.Seconds()
+
+		concFake.createCalls.Store(0)
+		concDur := runStreamCreateWorkload(t, concConn, c, opsPerRun)
+		concCalls := concFake.createCalls.Load()
+		concQPS := float64(concCalls) / concDur.Seconds()
+
+		t.Logf("%-16d %-14.0f %-14.0f %.2fx",
+			c, serialQPS, concQPS, concQPS/serialQPS)
+	}
+}
+
+// TestSchedulerOrderedParallelism compares three server-side handler shapes
+// under two workloads:
+//
+//	serial     : OLD handler — one goroutine per stream
+//	sem        : NEW handler v1 — per-request goroutine + semaphore, no ordering
+//	scheduler  : NEW handler v2 — per-path admission + subtree barriers
+//	              (mirrors filer.sync's MetadataProcessor)
+//
+// Workload "distinct" puts each op on its own path; workload "same" puts every
+// op on the same path.
+//
+// Expected shape:
+//   - serial     — both workloads ~1/serviceDelay (no parallelism)
+//   - sem        — both workloads lift ~serverConcurrency× (wrong: reorders
+//     same-path ops, acceptable for our filer store but undesired)
+//   - scheduler  — distinct workload lifts ~serverConcurrency×
+//     same workload falls back to ~1/serviceDelay (correctness)
+func TestSchedulerOrderedParallelism(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping repro in -short mode")
+	}
+
+	const (
+		opsPerRun         = 2400
+		serviceDelay      = 2 * time.Millisecond
+		serverConcurrency = 64
+		clientWorkers     = 12
+	)
+
+	// Serial, sem-only concurrent, scheduler-based concurrent.
+	serialAddr, serialFake, stopSerial := startFakeFilerServer(t, serviceDelay)
+	defer stopSerial()
+	serialConn := dialFakeFiler(t, serialAddr)
+	defer serialConn.Close()
+
+	semAddr, semFake, stopSem := startFakeConcurrentFilerServer(t, serviceDelay, serverConcurrency)
+	defer stopSem()
+	semConn := dialFakeFiler(t, semAddr)
+	defer semConn.Close()
+
+	schedAddr, schedFake, stopSched := startFakeSchedulerFilerServer(t, serviceDelay, serverConcurrency)
+	defer stopSched()
+	schedConn := dialFakeFiler(t, schedAddr)
+	defer schedConn.Close()
+
+	// Warm up all three connections.
+	for _, c := range []*grpc.ClientConn{serialConn, semConn, schedConn} {
+		client := filer_pb.NewSeaweedFilerClient(c)
+		if _, err := client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
+			Directory: "/",
+			Entry:     &filer_pb.Entry{Name: "warmup"},
+		}); err != nil {
+			t.Fatalf("warmup: %v", err)
+		}
+	}
+
+	t.Logf("service delay:       %v", serviceDelay)
+	t.Logf("server concurrency:  %d", serverConcurrency)
+	t.Logf("client workers:      %d", clientWorkers)
+	t.Logf("ops per run:         %d", opsPerRun)
+	t.Logf("serial ceiling:      %.0f QPS", 1.0/serviceDelay.Seconds())
+	t.Logf("concurrent ceiling:  %.0f QPS (%d x %.0f)", float64(serverConcurrency)/serviceDelay.Seconds(), serverConcurrency, 1.0/serviceDelay.Seconds())
+	t.Logf("")
+
+	workloads := []struct {
+		name string
+		fn   pathFn
+	}{
+		{"distinct paths", distinctPath},
+		{"same path    ", samePath},
+	}
+
+	t.Logf("%-16s %-14s %-14s %-14s %-14s",
+		"workload", "serial QPS", "sem QPS", "sched QPS", "sched peak-conc")
+
+	for _, w := range workloads {
+		serialFake.createCalls.Store(0)
+		serialDur := runStreamCreateWorkloadAt(t, serialConn, clientWorkers, opsPerRun, w.fn)
+		serialQPS := float64(serialFake.createCalls.Load()) / serialDur.Seconds()
+
+		semFake.createCalls.Store(0)
+		semDur := runStreamCreateWorkloadAt(t, semConn, clientWorkers, opsPerRun, w.fn)
+		semQPS := float64(semFake.createCalls.Load()) / semDur.Seconds()
+
+		schedFake.createCalls.Store(0)
+		schedFake.maxInFlight.Store(0)
+		schedDur := runStreamCreateWorkloadAt(t, schedConn, clientWorkers, opsPerRun, w.fn)
+		schedQPS := float64(schedFake.createCalls.Load()) / schedDur.Seconds()
+		schedPeak := schedFake.maxInFlight.Load()
+
+		t.Logf("%-16s %-14.0f %-14.0f %-14.0f %d",
+			w.name, serialQPS, semQPS, schedQPS, schedPeak)
+	}
+}
+
+// TestStreamSyncVsAsyncClient measures whether making the CLIENT side
+// asynchronous (fire-and-forget, no per-request waits) lifts the server-set
+// ceiling. It runs sync and async workloads against the same serial server
+// and compares end-to-end throughput.
+//
+// Expectation: at steady state, async cannot exceed 1/serviceDelay because the
+// server still processes one request at a time. Any "improvement" is transient
+// (in-flight pipelining up to the HTTP/2 window) and disappears as ops >> window.
+func TestStreamSyncVsAsyncClient(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping repro in -short mode")
+	}
+
+	const (
+		opsPerRun    = 2400
+		serviceDelay = 2 * time.Millisecond
+	)
+
+	addr, fake, stop := startFakeFilerServer(t, serviceDelay)
+	defer stop()
+
+	conn := dialFakeFiler(t, addr)
+	defer conn.Close()
+
+	warmupClient := filer_pb.NewSeaweedFilerClient(conn)
+	if _, err := warmupClient.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
+		Directory: "/",
+		Entry:     &filer_pb.Entry{Name: "warmup"},
+	}); err != nil {
+		t.Fatalf("warmup: %v", err)
+	}
+
+	// Sweep concurrency to show that async does not break the 1/serviceDelay
+	// ceiling no matter how much client-side parallelism we throw at it.
+	concurrencies := []int{1, 12, 64, 256}
+	t.Logf("service delay:    %v (theoretical serial ceiling = %.0f QPS)",
+		serviceDelay, 1.0/serviceDelay.Seconds())
+	t.Logf("ops per run:      %d", opsPerRun)
+	t.Logf("")
+	t.Logf("%-12s %-12s %-12s %-10s", "concurrency", "sync QPS", "async QPS", "ratio")
+
+	for _, c := range concurrencies {
+		fake.createCalls.Store(0)
+		syncDur := runStreamCreateWorkload(t, conn, c, opsPerRun)
+		syncCalls := fake.createCalls.Load()
+		syncQPS := float64(syncCalls) / syncDur.Seconds()
+
+		fake.createCalls.Store(0)
+		asyncDur := runStreamAsyncCreateWorkload(t, conn, c, opsPerRun)
+		asyncCalls := fake.createCalls.Load()
+		asyncQPS := float64(asyncCalls) / asyncDur.Seconds()
+
+		t.Logf("%-12d %-12.0f %-12.0f %-10.2fx",
+			c, syncQPS, asyncQPS, asyncQPS/syncQPS)
+	}
+}