snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/util/async_worker.go

@@ -0,0 +1,94 @@
+package util
+
+import "sync"
+
+// AsyncBatchWorker is an unbounded FIFO drained by one background goroutine,
+// handing it everything queued so far per process call. Unbounded on purpose:
+// a bound would let a stalled consumer block producers, which deadlocks when a
+// producer is itself a goroutine the consumer can end up waiting on.
+type AsyncBatchWorker[T any] struct {
+	mu        sync.Mutex
+	cond      *sync.Cond
+	queue     []T
+	enqueued  int64
+	processed int64
+	closed    bool
+	done      chan struct{}
+	process   func([]T)
+}
+
+// NewAsyncBatchWorker starts the worker. process runs outside the internal
+// lock, so it may take locks held by goroutines that are themselves enqueueing.
+func NewAsyncBatchWorker[T any](process func([]T)) *AsyncBatchWorker[T] {
+	w := &AsyncBatchWorker[T]{
+		process: process,
+		done:    make(chan struct{}),
+	}
+	w.cond = sync.NewCond(&w.mu)
+	go w.run()
+	return w
+}
+
+// Enqueue appends items for the worker without blocking. Items enqueued after
+// Shutdown are dropped.
+func (w *AsyncBatchWorker[T]) Enqueue(items ...T) {
+	if len(items) == 0 {
+		return
+	}
+	w.mu.Lock()
+	if w.closed {
+		w.mu.Unlock()
+		return
+	}
+	w.queue = append(w.queue, items...)
+	w.enqueued += int64(len(items))
+	w.mu.Unlock()
+	w.cond.Signal()
+}
+
+func (w *AsyncBatchWorker[T]) run() {
+	defer close(w.done)
+	for {
+		w.mu.Lock()
+		for len(w.queue) == 0 && !w.closed {
+			w.cond.Wait()
+		}
+		if len(w.queue) == 0 { // closed and drained
+			w.mu.Unlock()
+			return
+		}
+		batch := w.queue
+		w.queue = nil
+		w.mu.Unlock()
+
+		w.process(batch)
+
+		// Once per batch, not per item: Drain only needs the count to reach its target.
+		w.mu.Lock()
+		w.processed += int64(len(batch))
+		w.mu.Unlock()
+		w.cond.Broadcast()
+	}
+}
+
+// Drain blocks until every item enqueued so far has been processed, including
+// across a concurrent Shutdown — Shutdown drains all accepted items, so
+// processed always reaches the target and this cannot hang.
+func (w *AsyncBatchWorker[T]) Drain() {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	target := w.enqueued
+	for w.processed < target {
+		w.cond.Wait()
+	}
+}
+
+// Shutdown processes any outstanding items, then stops the worker and waits for
+// it to exit. Subsequent Enqueue calls are dropped.
+func (w *AsyncBatchWorker[T]) Shutdown() {
+	w.mu.Lock()
+	w.closed = true
+	w.mu.Unlock()
+	w.cond.Broadcast()
+	<-w.done
+}

weed/util/async_worker_test.go

@@ -0,0 +1,123 @@
+package util
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestAsyncBatchWorkerProcessesInOrder(t *testing.T) {
+	var mu sync.Mutex
+	var got []int
+	w := NewAsyncBatchWorker(func(batch []int) {
+		mu.Lock()
+		got = append(got, batch...)
+		mu.Unlock()
+	})
+	for i := 0; i < 100; i++ {
+		w.Enqueue(i)
+	}
+	w.Drain()
+	mu.Lock()
+	defer mu.Unlock()
+	if len(got) != 100 {
+		t.Fatalf("processed %d items, want 100", len(got))
+	}
+	for i, v := range got {
+		if v != i {
+			t.Fatalf("item %d = %d, want %d (FIFO order broken)", i, v, i)
+		}
+	}
+}
+
+func TestAsyncBatchWorkerDrainWaitsForEnqueued(t *testing.T) {
+	var processed int64
+	w := NewAsyncBatchWorker(func(batch []int) {
+		time.Sleep(time.Millisecond)
+		atomic.AddInt64(&processed, int64(len(batch)))
+	})
+	w.Enqueue(1, 2, 3)
+	w.Enqueue(4, 5)
+	w.Drain()
+	if n := atomic.LoadInt64(&processed); n != 5 {
+		t.Fatalf("processed %d, want 5 after Drain", n)
+	}
+}
+
+// Enqueue must not block even while the worker is busy processing under a lock
+// the producer also holds — the unbounded queue is what guarantees this.
+func TestAsyncBatchWorkerEnqueueNeverBlocks(t *testing.T) {
+	release := make(chan struct{})
+	entered := make(chan struct{})
+	var once sync.Once
+	w := NewAsyncBatchWorker(func(batch []int) {
+		once.Do(func() { close(entered) })
+		<-release
+	})
+	w.Enqueue(0) // worker picks this up and blocks in process
+	<-entered
+
+	done := make(chan struct{})
+	go func() {
+		for i := 1; i < 1000; i++ {
+			w.Enqueue(i)
+		}
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("Enqueue blocked while worker was busy")
+	}
+	close(release)
+	w.Shutdown()
+}
+
+// Drain must honor its contract even when Shutdown runs concurrently: it may
+// not return until the in-flight batch has actually been processed.
+func TestAsyncBatchWorkerDrainCompletesDuringShutdown(t *testing.T) {
+	var processed int64
+	release := make(chan struct{})
+	entered := make(chan struct{})
+	var once sync.Once
+	w := NewAsyncBatchWorker(func(batch []int) {
+		once.Do(func() { close(entered) })
+		<-release
+		atomic.AddInt64(&processed, int64(len(batch)))
+	})
+	w.Enqueue(1, 2, 3)
+	<-entered // worker has picked up the batch and is blocked in process
+
+	drained := make(chan struct{})
+	go func() { w.Drain(); close(drained) }()
+	go w.Shutdown() // sets closed and broadcasts while the batch is still in flight
+
+	select {
+	case <-drained:
+		t.Fatal("Drain returned before the in-flight batch finished")
+	case <-time.After(100 * time.Millisecond):
+	}
+	close(release)
+	<-drained
+	if n := atomic.LoadInt64(&processed); n != 3 {
+		t.Fatalf("processed %d after Drain, want 3", n)
+	}
+}
+
+func TestAsyncBatchWorkerShutdownDrainsThenStops(t *testing.T) {
+	var processed int64
+	w := NewAsyncBatchWorker(func(batch []int) {
+		atomic.AddInt64(&processed, int64(len(batch)))
+	})
+	w.Enqueue(1, 2, 3, 4)
+	w.Shutdown()
+	if n := atomic.LoadInt64(&processed); n != 4 {
+		t.Fatalf("processed %d, want 4 (Shutdown should drain)", n)
+	}
+	// Enqueue after Shutdown is dropped, not processed.
+	w.Enqueue(5, 6)
+	if n := atomic.LoadInt64(&processed); n != 4 {
+		t.Fatalf("processed %d after post-shutdown Enqueue, want 4", n)
+	}
+}

weed/util/buckets.go

@@ -0,0 +1,33 @@
+package util
+
+import "strings"
+
+// ExtractBucketPath returns the bucket path under basePath that contains target.
+// If requireChild is true, the target must include additional segments beyond the bucket itself.
+func ExtractBucketPath(basePath, target string, requireChild bool) (string, bool) {
+	cleanBase := strings.TrimSuffix(basePath, "/")
+	if cleanBase == "" {
+		return "", false
+	}
+
+	prefix := cleanBase + "/"
+	if !strings.HasPrefix(target, prefix) {
+		return "", false
+	}
+
+	rest := strings.TrimPrefix(target, prefix)
+	if rest == "" {
+		return "", false
+	}
+
+	bucketName, _, found := strings.Cut(rest, "/")
+	if bucketName == "" {
+		return "", false
+	}
+
+	if requireChild && !found {
+		return "", false
+	}
+
+	return prefix + bucketName, true
+}

weed/util/buckets_test.go

@@ -0,0 +1,63 @@
+package util
+
+import "testing"
+
+func TestExtractBucketPath(t *testing.T) {
+	for _, tt := range []struct {
+		name         string
+		base         string
+		target       string
+		requireChild bool
+		expected     string
+		ok           bool
+	}{
+		{
+			name:         "child paths return bucket",
+			base:         "/buckets",
+			target:       "/buckets/test/folder/file",
+			requireChild: true,
+			expected:     "/buckets/test",
+			ok:           true,
+		},
+		{
+			name:         "bucket root without child fails when required",
+			base:         "/buckets",
+			target:       "/buckets/test",
+			requireChild: true,
+			ok:           false,
+		},
+		{
+			name:         "bucket root allowed when not required",
+			base:         "/buckets",
+			target:       "/buckets/test",
+			requireChild: false,
+			expected:     "/buckets/test",
+			ok:           true,
+		},
+		{
+			name:         "path outside buckets fails",
+			base:         "/buckets",
+			target:       "/data/test/folder",
+			requireChild: true,
+			ok:           false,
+		},
+		{
+			name:         "trailing slash on base is normalized",
+			base:         "/buckets/",
+			target:       "/buckets/test/sub",
+			requireChild: true,
+			expected:     "/buckets/test",
+			ok:           true,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			got, ok := ExtractBucketPath(tt.base, tt.target, tt.requireChild)
+			if ok != tt.ok {
+				t.Fatalf("expected ok=%v, got %v", tt.ok, ok)
+			}
+			if got != tt.expected {
+				t.Fatalf("expected path %q, got %q", tt.expected, got)
+			}
+		})
+	}
+}

weed/util/buffer_pool/sync_pool.go

@@ -0,0 +1,38 @@
+package buffer_pool
+
+import (
+	"bytes"
+	"sync"
+)
+
+// maxRetainedBufferCap caps the capacity of buffers we hand back to the
+// sync.Pool. Buffers grown past this (e.g. by a 64 MiB chunk upload through
+// volume.PostHandler -> needle.ParseUpload -> bytes.Buffer.ReadFrom) are
+// dropped instead of pooled, so the underlying byte array becomes garbage
+// and is collected. Without this cap the pool effectively hoards every
+// high-water buffer for the process's lifetime — see #6541, where Harbor's
+// concurrent UploadPartCopy filled the pool with 64 MiB buffers and RSS
+// never receded.
+const maxRetainedBufferCap = 4 * 1024 * 1024
+
+var syncPool = sync.Pool{
+	New: func() interface{} {
+		return new(bytes.Buffer)
+	},
+}
+
+func SyncPoolGetBuffer() *bytes.Buffer {
+	return syncPool.Get().(*bytes.Buffer)
+}
+
+func SyncPoolPutBuffer(buffer *bytes.Buffer) {
+	if buffer == nil {
+		return
+	}
+	if buffer.Cap() > maxRetainedBufferCap {
+		// Drop the buffer; let GC reclaim the oversized backing array.
+		return
+	}
+	buffer.Reset()
+	syncPool.Put(buffer)
+}

weed/util/buffer_pool/sync_pool_test.go

@@ -0,0 +1,92 @@
+package buffer_pool
+
+import (
+	"bytes"
+	"testing"
+)
+
+// TestSyncPoolPutBuffer_DropsOversized is a regression guard for the volume-side
+// retention amplifier in https://github.com/seaweedfs/seaweedfs/issues/6541.
+//
+// volume.PostHandler -> needle.ParseUpload -> bytes.Buffer.ReadFrom grows the
+// buffer to chunk size. If we Put that buffer back as-is, sync.Pool keeps it
+// at the grown capacity for the rest of the process's lifetime. With
+// concurrent UploadPartCopy load that fills the pool with N × chunk-size
+// backing arrays that never shrink — exactly the "RSS never recedes" pattern
+// reported in the issue.
+//
+// We verify a Put + Get round trip can never round-trip a buffer larger than
+// maxRetainedBufferCap, regardless of how big it grew while in use.
+func TestSyncPoolPutBuffer_DropsOversized(t *testing.T) {
+	// Drain the pool so we start from a deterministic point. sync.Pool may
+	// still hold cached entries on other Ps, but for the small/big-cap
+	// distinction below that doesn't matter — we assert an invariant on
+	// every Get, not the identity of a specific buffer.
+	for i := 0; i < 64; i++ {
+		_ = SyncPoolGetBuffer()
+	}
+
+	big := &bytes.Buffer{}
+	big.Grow(maxRetainedBufferCap * 4) // simulate a large upload buffer
+	if got := big.Cap(); got <= maxRetainedBufferCap {
+		t.Fatalf("test setup: big.Cap=%d should exceed threshold %d",
+			got, maxRetainedBufferCap)
+	}
+	SyncPoolPutBuffer(big)
+
+	// Any number of Gets must not return a buffer with cap > threshold.
+	// (If big had been retained, sync.Pool's per-P cache would surface it
+	// on the very next Get on this goroutine.)
+	for i := 0; i < 16; i++ {
+		got := SyncPoolGetBuffer()
+		if cap := got.Cap(); cap > maxRetainedBufferCap {
+			t.Fatalf("Get %d returned buffer with cap=%d, exceeds threshold %d "+
+				"(regression: oversized buffers retained in pool?)",
+				i, cap, maxRetainedBufferCap)
+		}
+	}
+}
+
+// TestSyncPoolPutBuffer_KeepsRightSized verifies the cap is one-sided: we
+// still pool reasonably-sized buffers so the common case (small uploads,
+// header parsing) doesn't pay an alloc per request.
+func TestSyncPoolPutBuffer_KeepsRightSized(t *testing.T) {
+	for i := 0; i < 64; i++ {
+		_ = SyncPoolGetBuffer()
+	}
+
+	small := &bytes.Buffer{}
+	small.Grow(maxRetainedBufferCap / 2)
+	smallCap := small.Cap()
+	SyncPoolPutBuffer(small)
+
+	// We don't assert pointer identity (sync.Pool can hand back any cached
+	// buffer), but the previously-Put buffer should appear among Gets on
+	// the same goroutine in the absence of a GC. If our cap-policy ever
+	// regresses to dropping right-sized buffers, this test starts seeing
+	// only fresh (cap=0) buffers and fails.
+	sawPooled := false
+	for i := 0; i < 8; i++ {
+		got := SyncPoolGetBuffer()
+		if got.Cap() == smallCap {
+			sawPooled = true
+			break
+		}
+	}
+	if !sawPooled {
+		t.Fatalf("right-sized buffer (cap=%d) never came back from the pool "+
+			"(regression: cap policy too aggressive?)", smallCap)
+	}
+}
+
+// TestSyncPoolPutBuffer_NilSafe documents that Put tolerates a nil buffer.
+// The volume server defers Put on a buffer obtained via Get, but defensive
+// callers in other paths may Put(nil); we should not panic.
+func TestSyncPoolPutBuffer_NilSafe(t *testing.T) {
+	defer func() {
+		if r := recover(); r != nil {
+			t.Fatalf("SyncPoolPutBuffer(nil) panicked: %v", r)
+		}
+	}()
+	SyncPoolPutBuffer(nil)
+}

weed/util/buffered_queue/buffered_queue.go

@@ -0,0 +1,156 @@
+package buffered_queue
+
+import (
+	"fmt"
+	"sync"
+)
+
+// ItemChunkNode represents a node in the linked list of job chunks
+type ItemChunkNode[T any] struct {
+	items     []T
+	headIndex int
+	tailIndex int
+	next      *ItemChunkNode[T]
+	nodeId    int
+}
+
+// BufferedQueue implements a buffered queue using a linked list of job chunks
+type BufferedQueue[T any] struct {
+	chunkSize   int // Maximum number of items per chunk
+	head        *ItemChunkNode[T]
+	tail        *ItemChunkNode[T]
+	last        *ItemChunkNode[T] // Pointer to the last chunk, for reclaiming memory
+	count       int               // Total number of items in the queue
+	mutex       sync.Mutex
+	nodeCounter int
+	waitCond    *sync.Cond
+	isClosed    bool
+}
+
+// NewBufferedQueue creates a new buffered queue with the specified chunk size
+func NewBufferedQueue[T any](chunkSize int) *BufferedQueue[T] {
+	// Create an empty chunk to initialize head and tail
+	chunk := &ItemChunkNode[T]{items: make([]T, chunkSize), nodeId: 0}
+	bq := &BufferedQueue[T]{
+		chunkSize: chunkSize,
+		head:      chunk,
+		tail:      chunk,
+		last:      chunk,
+		count:     0,
+		mutex:     sync.Mutex{},
+	}
+	bq.waitCond = sync.NewCond(&bq.mutex)
+	return bq
+}
+
+// Enqueue adds a job to the queue
+func (q *BufferedQueue[T]) Enqueue(job T) error {
+
+	if q.isClosed {
+		return fmt.Errorf("queue is closed")
+	}
+
+	q.mutex.Lock()
+	defer q.mutex.Unlock()
+
+	// If the tail chunk is full, create a new chunk (reusing empty chunks if available)
+	if q.tail.tailIndex == q.chunkSize {
+		if q.tail == q.last {
+			// Create a new chunk
+			q.nodeCounter++
+			newChunk := &ItemChunkNode[T]{items: make([]T, q.chunkSize), nodeId: q.nodeCounter}
+			q.tail.next = newChunk
+			q.tail = newChunk
+			q.last = newChunk
+		} else {
+			// Reuse an empty chunk
+			q.tail = q.tail.next
+			q.tail.headIndex = 0
+			q.tail.tailIndex = 0
+			// println("tail moved to chunk", q.tail.nodeId)
+		}
+	}
+
+	// Add the job to the tail chunk
+	q.tail.items[q.tail.tailIndex] = job
+	q.tail.tailIndex++
+	q.count++
+	if q.count == 1 {
+		q.waitCond.Signal()
+	}
+
+	return nil
+}
+
+// Dequeue removes and returns a job from the queue
+func (q *BufferedQueue[T]) Dequeue() (T, bool) {
+	q.mutex.Lock()
+	defer q.mutex.Unlock()
+
+	for q.count <= 0 && !q.isClosed {
+		q.waitCond.Wait()
+	}
+	if q.count <= 0 && q.isClosed {
+		var a T
+		return a, false
+	}
+
+	q.maybeAdjustHeadIndex()
+
+	job := q.head.items[q.head.headIndex]
+	q.head.headIndex++
+	q.count--
+
+	return job, true
+}
+
+func (q *BufferedQueue[T]) maybeAdjustHeadIndex() {
+	if q.head.headIndex == q.chunkSize {
+		q.last.next = q.head
+		q.head = q.head.next
+		q.last = q.last.next
+		q.last.next = nil
+		//println("reusing chunk", q.last.nodeId)
+		//fmt.Printf("head: %+v\n", q.head)
+		//fmt.Printf("tail: %+v\n", q.tail)
+		//fmt.Printf("last: %+v\n", q.last)
+		//fmt.Printf("count: %d\n", q.count)
+		//for p := q.head; p != nil ; p = p.next {
+		//	fmt.Printf("Node: %+v\n", p)
+		//}
+	}
+}
+
+func (q *BufferedQueue[T]) PeekHead() (T, bool) {
+	q.mutex.Lock()
+	defer q.mutex.Unlock()
+
+	if q.count <= 0 {
+		var a T
+		return a, false
+	}
+
+	q.maybeAdjustHeadIndex()
+
+	job := q.head.items[q.head.headIndex]
+	return job, true
+}
+
+// Size returns the number of items in the queue
+func (q *BufferedQueue[T]) Size() int {
+	q.mutex.Lock()
+	defer q.mutex.Unlock()
+	return q.count
+}
+
+// IsEmpty returns true if the queue is empty
+func (q *BufferedQueue[T]) IsEmpty() bool {
+	return q.Size() == 0
+}
+
+func (q *BufferedQueue[T]) CloseInput() {
+	q.mutex.Lock()
+	defer q.mutex.Unlock()
+	q.isClosed = true
+	q.waitCond.Broadcast()
+}

weed/util/buffered_queue/buffered_queue_test.go

@@ -0,0 +1,128 @@
+package buffered_queue
+
+import (
+	"sync"
+	"testing"
+)
+
+func TestJobQueue(t *testing.T) {
+	type Job[T any] struct {
+		ID     int
+		Action string
+		Data   T
+	}
+
+	queue := NewBufferedQueue[Job[string]](2) // Chunk size of 5
+	queue.Enqueue(Job[string]{ID: 1, Action: "task1", Data: "hello"})
+	queue.Enqueue(Job[string]{ID: 2, Action: "task2", Data: "world"})
+
+	if queue.Size() != 2 {
+		t.Errorf("Expected queue size of 2, got %d", queue.Size())
+	}
+
+	queue.Enqueue(Job[string]{ID: 3, Action: "task3", Data: "3!"})
+	queue.Enqueue(Job[string]{ID: 4, Action: "task4", Data: "4!"})
+	queue.Enqueue(Job[string]{ID: 5, Action: "task5", Data: "5!"})
+
+	if queue.Size() != 5 {
+		t.Errorf("Expected queue size of 5, got %d", queue.Size())
+	}
+
+	println("enqueued 5 items")
+
+	println("dequeue", 1)
+	job, ok := queue.Dequeue()
+	if !ok {
+		t.Errorf("Expected dequeue to return true")
+	}
+	if job.ID != 1 {
+		t.Errorf("Expected job ID of 1, got %d", job.ID)
+	}
+
+	println("dequeue", 2)
+	job, ok = queue.Dequeue()
+	if !ok {
+		t.Errorf("Expected dequeue to return true")
+	}
+
+	println("enqueue", 6)
+	queue.Enqueue(Job[string]{ID: 6, Action: "task6", Data: "6!"})
+	println("enqueue", 7)
+	queue.Enqueue(Job[string]{ID: 7, Action: "task7", Data: "7!"})
+
+	for i := 0; i < 5; i++ {
+		println("dequeue ...")
+		job, ok = queue.Dequeue()
+		if !ok {
+			t.Errorf("Expected dequeue to return true")
+		}
+		println("dequeued", job.ID)
+	}
+
+	if queue.Size() != 0 {
+		t.Errorf("Expected queue size of 0, got %d", queue.Size())
+	}
+
+	for i := 0; i < 5; i++ {
+		println("enqueue", i+8)
+		queue.Enqueue(Job[string]{ID: i + 8, Action: "task", Data: "data"})
+	}
+	for i := 0; i < 5; i++ {
+		job, ok = queue.Dequeue()
+		if !ok {
+			t.Errorf("Expected dequeue to return true")
+		}
+		if job.ID != i+8 {
+			t.Errorf("Expected job ID of %d, got %d", i, job.ID)
+		}
+		println("dequeued", job.ID)
+	}
+
+}
+
+func TestJobQueueClose(t *testing.T) {
+	type Job[T any] struct {
+		ID     int
+		Action string
+		Data   T
+	}
+
+	queue := NewBufferedQueue[Job[string]](2)
+	queue.Enqueue(Job[string]{ID: 1, Action: "task1", Data: "hello"})
+	queue.Enqueue(Job[string]{ID: 2, Action: "task2", Data: "world"})
+
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for data, ok := queue.Dequeue(); ok; data, ok = queue.Dequeue() {
+			println("dequeued", data.ID)
+		}
+	}()
+
+	for i := 0; i < 5; i++ {
+		queue.Enqueue(Job[string]{ID: i + 3, Action: "task", Data: "data"})
+	}
+
+	queue.CloseInput()
+	wg.Wait()
+
+}
+
+func BenchmarkBufferedQueue(b *testing.B) {
+	type Job[T any] struct {
+		ID     int
+		Action string
+		Data   T
+	}
+
+	queue := NewBufferedQueue[Job[string]](1024)
+
+	for i := 0; i < b.N; i++ {
+		queue.Enqueue(Job[string]{ID: i, Action: "task", Data: "data"})
+	}
+	for i := 0; i < b.N; i++ {
+		_, _ = queue.Dequeue()
+	}
+
+}

weed/util/buffered_writer/buffered_writer.go

@@ -0,0 +1,52 @@
+package buffered_writer
+
+import (
+	"bytes"
+	"io"
+)
+
+var _ = io.WriteCloser(&BufferedWriteCloser{})
+
+type BufferedWriteCloser struct {
+	buffer          bytes.Buffer
+	bufferLimit     int
+	position        int64
+	nextFlushOffset int64
+	FlushFunc       func([]byte, int64) error
+	CloseFunc       func() error
+}
+
+func NewBufferedWriteCloser(bufferLimit int) *BufferedWriteCloser {
+	return &BufferedWriteCloser{
+		bufferLimit: bufferLimit,
+	}
+}
+
+func (b *BufferedWriteCloser) Write(p []byte) (n int, err error) {
+
+	if b.buffer.Len()+len(p) >= b.bufferLimit {
+		if err := b.FlushFunc(b.buffer.Bytes(), b.nextFlushOffset); err != nil {
+			return 0, err
+		}
+		b.nextFlushOffset += int64(b.buffer.Len())
+		b.buffer.Reset()
+	}
+
+	return b.buffer.Write(p)
+
+}
+
+func (b *BufferedWriteCloser) Close() error {
+	if b.buffer.Len() > 0 {
+		if err := b.FlushFunc(b.buffer.Bytes(), b.nextFlushOffset); err != nil {
+			return err
+		}
+	}
+	if b.CloseFunc != nil {
+		if err := b.CloseFunc(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

weed/util/bytes.go

@@ -0,0 +1,278 @@
+package util
+
+import (
+	"bytes"
+	"crypto/md5"
+	"crypto/rand"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"math"
+	"strconv"
+	"strings"
+	"unicode"
+)
+
+var randomRead = rand.Read
+
+// BytesToHumanReadable returns the converted human readable representation of the bytes.
+func BytesToHumanReadable(b uint64) string {
+	const unit = 1024
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+
+	div, exp := uint64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+
+	return fmt.Sprintf("%.2f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
+}
+
+// big endian
+
+func BytesToUint64(b []byte) (v uint64) {
+	length := uint(len(b))
+	if length == 0 {
+		return 0
+	}
+	for i := uint(0); i < length-1; i++ {
+		v += uint64(b[i])
+		v <<= 8
+	}
+	v += uint64(b[length-1])
+	return
+}
+func BytesToUint32(b []byte) (v uint32) {
+	length := uint(len(b))
+	if length == 0 {
+		return 0
+	}
+	for i := uint(0); i < length-1; i++ {
+		v += uint32(b[i])
+		v <<= 8
+	}
+	v += uint32(b[length-1])
+	return
+}
+func BytesToUint16(b []byte) (v uint16) {
+	length := uint(len(b))
+	if length == 0 {
+		return 0
+	}
+	for i := uint(0); i < length-1; i++ {
+		v += uint16(b[i])
+		v <<= 8
+	}
+	v += uint16(b[length-1])
+	return
+}
+func Uint64toBytes(b []byte, v uint64) {
+	for i := uint(0); i < 8; i++ {
+		b[7-i] = byte(v >> (i * 8))
+	}
+}
+func Uint32toBytes(b []byte, v uint32) {
+	for i := uint(0); i < 4; i++ {
+		b[3-i] = byte(v >> (i * 8))
+	}
+}
+func Uint16toBytes(b []byte, v uint16) {
+	b[0] = byte(v >> 8)
+	b[1] = byte(v)
+}
+func Uint8toBytes(b []byte, v uint8) {
+	b[0] = byte(v)
+}
+
+// returns a 64 bit big int
+func HashStringToLong(dir string) (v int64) {
+	h := md5.New()
+	io.WriteString(h, dir)
+
+	b := h.Sum(nil)
+
+	v += int64(b[0])
+	v <<= 8
+	v += int64(b[1])
+	v <<= 8
+	v += int64(b[2])
+	v <<= 8
+	v += int64(b[3])
+	v <<= 8
+	v += int64(b[4])
+	v <<= 8
+	v += int64(b[5])
+	v <<= 8
+	v += int64(b[6])
+	v <<= 8
+	v += int64(b[7])
+
+	return
+}
+
+func HashToInt32(data []byte) (v int32) {
+	h := md5.New()
+	h.Write(data)
+
+	b := h.Sum(nil)
+
+	v += int32(b[0])
+	v <<= 8
+	v += int32(b[1])
+	v <<= 8
+	v += int32(b[2])
+	v <<= 8
+	v += int32(b[3])
+
+	return
+}
+
+func Base64Encode(data []byte) string {
+	return base64.StdEncoding.EncodeToString(data)
+}
+
+func Md5(data []byte) []byte {
+	hash := md5.New()
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
+func Md5String(data []byte) string {
+	return fmt.Sprintf("%x", Md5(data))
+}
+
+func Base64Md5ToBytes(contentMd5 string) []byte {
+	data, err := base64.StdEncoding.DecodeString(contentMd5)
+	if err != nil {
+		return nil
+	}
+	return data
+}
+
+func RandomInt32() int32 {
+	buf := make([]byte, 4)
+	rand.Read(buf)
+	return int32(BytesToUint32(buf))
+}
+
+func RandomUint64() uint64 {
+	buf := make([]byte, 8)
+	if _, err := randomRead(buf); err != nil {
+		panic(err)
+	}
+	return BytesToUint64(buf)
+}
+
+func RandomBytes(byteCount int) []byte {
+	buf := make([]byte, byteCount)
+	rand.Read(buf)
+	return buf
+}
+
+type BytesReader struct {
+	Bytes []byte
+	*bytes.Reader
+}
+
+func NewBytesReader(b []byte) *BytesReader {
+	return &BytesReader{
+		Bytes:  b,
+		Reader: bytes.NewReader(b),
+	}
+}
+
+// EmptyTo returns to if s is empty.
+func EmptyTo(s, to string) string {
+	if s == "" {
+		return to
+	}
+
+	return s
+}
+
+// IfElse works like b ? this : that.
+func IfElse(b bool, this, that string) string {
+	if b {
+		return this
+	}
+	return that
+}
+
+// ParseBytes parses a string representation of bytes into the number
+// of bytes it represents.
+//
+// See Also: Bytes, IBytes.
+//
+// ParseBytes("42MB") -> 42000000, nil
+// ParseBytes("42 MB") -> 42000000, nil
+// ParseBytes("42 mib") -> 44040192, nil
+func ParseBytes(s string) (uint64, error) {
+	lastDigit := 0
+	hasComma := false
+	for _, r := range s {
+		if !(unicode.IsDigit(r) || r == '.' || r == ',') {
+			break
+		}
+		if r == ',' {
+			hasComma = true
+		}
+		lastDigit++
+	}
+
+	num := s[:lastDigit]
+	if hasComma {
+		num = strings.Replace(num, ",", "", -1)
+	}
+
+	f, err := strconv.ParseFloat(num, 64)
+	if err != nil {
+		return 0, err
+	}
+
+	extra := strings.ToLower(strings.TrimSpace(s[lastDigit:]))
+	if m, ok := bytesSizeTable[extra]; ok {
+		f *= float64(m)
+		if f >= math.MaxUint64 {
+			return 0, fmt.Errorf("too large: %v", s)
+		}
+		return uint64(f), nil
+	}
+
+	return 0, fmt.Errorf("unhandled size name: %v", extra)
+}
+
+var bytesSizeTable = map[string]uint64{
+	"b": Byte, "kib": KiByte, "kb": KByte, "mib": MiByte, "mb": MByte, "gib": GiByte, "gb": GByte,
+	"tib": TiByte, "tb": TByte, "pib": PiByte, "pb": PByte, "eib": EiByte, "eb": EByte,
+	// Without suffix
+	"": Byte, "ki": KiByte, "k": KByte, "mi": MiByte, "m": MByte, "gi": GiByte, "g": GByte,
+	"ti": TiByte, "t": TByte, "pi": PiByte, "p": PByte, "ei": EiByte, "e": EByte,
+}
+
+// IEC Sizes.
+// kibis of bits
+const (
+	Byte = 1 << (iota * 10)
+	KiByte
+	MiByte
+	GiByte
+	TiByte
+	PiByte
+	EiByte
+)
+
+// SI Sizes.
+const (
+	IByte = 1
+	KByte = IByte * 1000
+	MByte = KByte * 1000
+	GByte = MByte * 1000
+	TByte = GByte * 1000
+	PByte = TByte * 1000
+	EByte = PByte * 1000
+)
+
+const ErrVolumeNoSpaceLeft = "no space left for disk type"

weed/util/bytes_test.go

@@ -0,0 +1,98 @@
+package util
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestRandomUint64ReturnsFullRandomValue(t *testing.T) {
+	useRandomRead(t, func(p []byte) (int, error) {
+		copy(p, []byte{0x80, 0, 0, 0, 0, 0, 0, 1})
+		return len(p), nil
+	})
+
+	const want uint64 = 0x8000000000000001
+	if got := RandomUint64(); got != want {
+		t.Fatalf("RandomUint64() = %d, want %d", got, want)
+	}
+}
+
+func TestRandomUint64PanicsOnRandomReadError(t *testing.T) {
+	useRandomRead(t, func([]byte) (int, error) {
+		return 0, errors.New("random read failed")
+	})
+
+	defer func() {
+		if recover() == nil {
+			t.Fatal("RandomUint64() did not panic on random read error")
+		}
+	}()
+
+	RandomUint64()
+}
+
+func useRandomRead(t *testing.T, read func([]byte) (int, error)) {
+	t.Helper()
+
+	original := randomRead
+	randomRead = read
+	t.Cleanup(func() {
+		randomRead = original
+	})
+}
+
+func TestByteParsing(t *testing.T) {
+	tests := []struct {
+		in  string
+		exp uint64
+	}{
+		{"42", 42},
+		{"42MB", 42000000},
+		{"42MiB", 44040192},
+		{"42mb", 42000000},
+		{"42mib", 44040192},
+		{"42MIB", 44040192},
+		{"42 MB", 42000000},
+		{"42 MiB", 44040192},
+		{"42 mb", 42000000},
+		{"42 mib", 44040192},
+		{"42 MIB", 44040192},
+		{"42.5MB", 42500000},
+		{"42.5MiB", 44564480},
+		{"42.5 MB", 42500000},
+		{"42.5 MiB", 44564480},
+		// No need to say B
+		{"42M", 42000000},
+		{"42Mi", 44040192},
+		{"42m", 42000000},
+		{"42mi", 44040192},
+		{"42MI", 44040192},
+		{"42 M", 42000000},
+		{"42 Mi", 44040192},
+		{"42 m", 42000000},
+		{"42 mi", 44040192},
+		{"42 MI", 44040192},
+		{"42.5M", 42500000},
+		{"42.5Mi", 44564480},
+		{"42.5 M", 42500000},
+		{"42.5 Mi", 44564480},
+		// Bug #42
+		{"1,005.03 MB", 1005030000},
+		// Large testing, breaks when too much larger than
+		// this.
+		{"12.5 EB", uint64(12.5 * float64(EByte))},
+		{"12.5 E", uint64(12.5 * float64(EByte))},
+		{"12.5 EiB", uint64(12.5 * float64(EiByte))},
+	}
+
+	for _, p := range tests {
+		got, err := ParseBytes(p.in)
+		if err != nil {
+			t.Errorf("Couldn't parse %v: %v", p.in, err)
+		}
+		if got != p.exp {
+			t.Errorf("Expected %v for %v, got %v",
+				p.exp, p.in, got)
+		}
+	}
+}

weed/util/chunk_cache/chunk_cache.go

@@ -0,0 +1,272 @@
+package chunk_cache
+
+import (
+	"encoding/binary"
+	"errors"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+var ErrorOutOfBounds = errors.New("attempt to read out of bounds")
+
+const cacheHeaderSize = 8 // 4 bytes volumeId + 4 bytes cookie
+
+// parseCacheHeader extracts volume ID and cookie from the 8-byte cache header
+func parseCacheHeader(header []byte) (needle.VolumeId, types.Cookie) {
+	volumeId := needle.VolumeId(binary.BigEndian.Uint32(header[0:4]))
+	cookie := types.BytesToCookie(header[4:8])
+	return volumeId, cookie
+}
+
+type ChunkCache interface {
+	ReadChunkAt(data []byte, fileId string, offset uint64) (n int, err error)
+	SetChunk(fileId string, data []byte)
+	IsInCache(fileId string, lockNeeded bool) (answer bool)
+	GetMaxFilePartSizeInCache() (answer uint64)
+}
+
+// a global cache for recently accessed file chunks
+type TieredChunkCache struct {
+	memCache   *ChunkCacheInMemory
+	diskCaches []*OnDiskCacheLayer
+	sync.RWMutex
+	onDiskCacheSizeLimit0  uint64
+	onDiskCacheSizeLimit1  uint64
+	onDiskCacheSizeLimit2  uint64
+	maxFilePartSizeInCache uint64
+}
+
+var _ ChunkCache = &TieredChunkCache{}
+
+func NewTieredChunkCache(maxEntries int64, dir string, diskSizeInUnit int64, unitSize int64) *TieredChunkCache {
+
+	c := &TieredChunkCache{
+		memCache: NewChunkCacheInMemory(maxEntries),
+	}
+	c.diskCaches = make([]*OnDiskCacheLayer, 3)
+	c.onDiskCacheSizeLimit0 = uint64(unitSize)
+	c.onDiskCacheSizeLimit1 = 4 * c.onDiskCacheSizeLimit0
+	c.onDiskCacheSizeLimit2 = 2 * c.onDiskCacheSizeLimit1
+	c.diskCaches[0] = NewOnDiskCacheLayer(dir, "c0_2", diskSizeInUnit*unitSize/8, 2)
+	c.diskCaches[1] = NewOnDiskCacheLayer(dir, "c1_3", diskSizeInUnit*unitSize/4+diskSizeInUnit*unitSize/8, 3)
+	c.diskCaches[2] = NewOnDiskCacheLayer(dir, "c2_2", diskSizeInUnit*unitSize/2, 2)
+	c.maxFilePartSizeInCache = uint64(unitSize*diskSizeInUnit) / 4
+
+	return c
+}
+
+func (c *TieredChunkCache) GetMaxFilePartSizeInCache() (answer uint64) {
+	if c == nil {
+		return 0
+	}
+	return c.maxFilePartSizeInCache
+}
+
+func (c *TieredChunkCache) IsInCache(fileId string, lockNeeded bool) (answer bool) {
+	if c == nil {
+		return false
+	}
+
+	if lockNeeded {
+		c.RLock()
+		defer c.RUnlock()
+	}
+
+	item := c.memCache.cache.Get(fileId)
+	if item != nil {
+		glog.V(4).Infof("fileId %s is in memcache", fileId)
+		return true
+	}
+
+	fid, err := needle.ParseFileIdFromString(fileId)
+	if err != nil {
+		glog.V(4).Infof("failed to parse file id %s", fileId)
+		return false
+	}
+
+	// Check disk cache with volume ID and cookie validation
+	for i, diskCacheLayer := range c.diskCaches {
+		for k, v := range diskCacheLayer.diskCaches {
+			if nv, ok := v.nm.Get(fid.Key); ok {
+				// Read cache header to check volume ID and cookie
+				headerBytes := make([]byte, cacheHeaderSize)
+				if readN, readErr := v.DataBackend.ReadAt(headerBytes, nv.Offset.ToActualOffset()); readErr == nil && readN == cacheHeaderSize {
+					// Parse volume ID and cookie from header
+					storedVolumeId, storedCookie := parseCacheHeader(headerBytes)
+
+					if storedVolumeId == fid.VolumeId && storedCookie == fid.Cookie {
+						glog.V(4).Infof("fileId %s is in diskCaches[%d].volume[%d]", fileId, i, k)
+						return true
+					}
+					glog.V(4).Infof("fileId %s header mismatch in diskCaches[%d].volume[%d]: stored volume %d cookie %x, expected volume %d cookie %x",
+						fileId, i, k, storedVolumeId, storedCookie, fid.VolumeId, fid.Cookie)
+				}
+			}
+		}
+	}
+	return false
+}
+
+func (c *TieredChunkCache) ReadChunkAt(data []byte, fileId string, offset uint64) (n int, err error) {
+	if c == nil {
+		return 0, nil
+	}
+
+	c.RLock()
+	defer c.RUnlock()
+
+	minSize := offset + uint64(len(data))
+	if minSize <= c.onDiskCacheSizeLimit0 {
+		n, err = c.memCache.readChunkAt(data, fileId, offset)
+		if err != nil {
+			glog.Errorf("failed to read from memcache: %s", err)
+		}
+		if n == int(len(data)) {
+			return n, nil
+		}
+	}
+
+	fid, err := needle.ParseFileIdFromString(fileId)
+	if err != nil {
+		glog.Errorf("failed to parse file id %s", fileId)
+		return 0, nil
+	}
+
+	// Try disk caches with volume ID and cookie validation
+	if minSize <= c.onDiskCacheSizeLimit0 {
+		n, err = c.readChunkAtWithHeaderValidation(data, fid, offset, 0)
+		if n == int(len(data)) {
+			return
+		}
+	}
+	if minSize <= c.onDiskCacheSizeLimit1 {
+		n, err = c.readChunkAtWithHeaderValidation(data, fid, offset, 1)
+		if n == int(len(data)) {
+			return
+		}
+	}
+	{
+		n, err = c.readChunkAtWithHeaderValidation(data, fid, offset, 2)
+		if n == int(len(data)) {
+			return
+		}
+	}
+
+	return 0, nil
+
+}
+
+func (c *TieredChunkCache) SetChunk(fileId string, data []byte) {
+	if c == nil {
+		return
+	}
+	c.Lock()
+	defer c.Unlock()
+
+	glog.V(4).Infof("SetChunk %s size %d\n", fileId, len(data))
+	if c.IsInCache(fileId, false) {
+		glog.V(4).Infof("fileId %s is already in cache", fileId)
+		return
+	}
+
+	c.doSetChunk(fileId, data)
+}
+
+func (c *TieredChunkCache) doSetChunk(fileId string, data []byte) {
+	// Disk cache format: [4-byte volumeId][4-byte cookie][chunk data]
+	// Memory cache format: full fileId as key -> raw data (unchanged)
+
+	// Memory cache unchanged - uses full fileId
+	if len(data) <= int(c.onDiskCacheSizeLimit0) {
+		c.memCache.SetChunk(fileId, data)
+	}
+
+	fid, err := needle.ParseFileIdFromString(fileId)
+	if err != nil {
+		glog.Errorf("failed to parse file id %s", fileId)
+		return
+	}
+
+	// Prepend volume ID and cookie to data for disk cache
+	// Format: [4-byte volumeId][4-byte cookie][chunk data]
+	headerBytes := make([]byte, cacheHeaderSize)
+	// Store volume ID in first 4 bytes using big-endian
+	binary.BigEndian.PutUint32(headerBytes[0:4], uint32(fid.VolumeId))
+	// Store cookie in next 4 bytes
+	types.CookieToBytes(headerBytes[4:8], fid.Cookie)
+	dataWithHeader := append(headerBytes, data...)
+
+	// Store with volume ID and cookie header in disk cache
+	if len(data) <= int(c.onDiskCacheSizeLimit0) {
+		c.diskCaches[0].setChunk(fid.Key, dataWithHeader)
+	} else if len(data) <= int(c.onDiskCacheSizeLimit1) {
+		c.diskCaches[1].setChunk(fid.Key, dataWithHeader)
+	} else {
+		c.diskCaches[2].setChunk(fid.Key, dataWithHeader)
+	}
+
+}
+
+func (c *TieredChunkCache) Shutdown() {
+	if c == nil {
+		return
+	}
+	c.Lock()
+	defer c.Unlock()
+	for _, diskCache := range c.diskCaches {
+		diskCache.shutdown()
+	}
+}
+
+// readChunkAtWithHeaderValidation reads from disk cache with volume ID and cookie validation
+func (c *TieredChunkCache) readChunkAtWithHeaderValidation(data []byte, fid *needle.FileId, offset uint64, cacheLevel int) (n int, err error) {
+	// Step 1: Read and validate header (volume ID + cookie)
+	headerBuffer := make([]byte, cacheHeaderSize)
+	headerRead, err := c.diskCaches[cacheLevel].readChunkAt(headerBuffer, fid.Key, 0)
+
+	if err != nil {
+		glog.V(4).Infof("failed to read header for %s from cache level %d: %v",
+			fid.String(), cacheLevel, err)
+		return 0, err
+	}
+
+	if headerRead < cacheHeaderSize {
+		glog.V(4).Infof("insufficient data for header validation for %s from cache level %d: read %d bytes",
+			fid.String(), cacheLevel, headerRead)
+		return 0, nil // Not enough data for header - likely old format, treat as cache miss
+	}
+
+	// Parse volume ID and cookie from header
+	storedVolumeId, storedCookie := parseCacheHeader(headerBuffer)
+
+	// Validate both volume ID and cookie
+	if storedVolumeId != fid.VolumeId || storedCookie != fid.Cookie {
+		glog.V(4).Infof("header mismatch for %s in cache level %d: stored volume %d cookie %x, expected volume %d cookie %x (possible old format)",
+			fid.String(), cacheLevel, storedVolumeId, storedCookie, fid.VolumeId, fid.Cookie)
+		return 0, nil // Treat as cache miss - could be old format or actual mismatch
+	}
+
+	// Step 2: Read actual data from the offset position (after header)
+	// The disk cache has format: [4-byte volumeId][4-byte cookie][actual chunk data]
+	// We want to read from position: cacheHeaderSize + offset
+	dataOffset := cacheHeaderSize + offset
+	n, err = c.diskCaches[cacheLevel].readChunkAt(data, fid.Key, dataOffset)
+
+	if err != nil {
+		glog.V(4).Infof("failed to read data at offset %d for %s from cache level %d: %v",
+			offset, fid.String(), cacheLevel, err)
+		return 0, err
+	}
+
+	return n, nil
+}
+
+func min(x, y int) int {
+	if x < y {
+		return x
+	}
+	return y
+}

weed/util/chunk_cache/chunk_cache_in_memory.go

@@ -0,0 +1,73 @@
+package chunk_cache
+
+import (
+	"time"
+
+	"github.com/karlseguin/ccache/v2"
+)
+
+var (
+	_ ChunkCache = &ChunkCacheInMemory{}
+)
+
+// a global cache for recently accessed file chunks
+type ChunkCacheInMemory struct {
+	cache *ccache.Cache
+}
+
+func (c *ChunkCacheInMemory) ReadChunkAt(data []byte, fileId string, offset uint64) (n int, err error) {
+	return c.readChunkAt(data, fileId, offset)
+}
+
+func (c *ChunkCacheInMemory) IsInCache(fileId string, lockNeeded bool) (answer bool) {
+	item := c.cache.Get(fileId)
+	if item == nil {
+		return false
+	}
+	return true
+}
+
+func (c *ChunkCacheInMemory) GetMaxFilePartSizeInCache() (answer uint64) {
+	return 8 * 1024 * 1024
+}
+
+func NewChunkCacheInMemory(maxEntries int64) *ChunkCacheInMemory {
+	pruneCount := maxEntries >> 3
+	if pruneCount <= 0 {
+		pruneCount = 500
+	}
+	return &ChunkCacheInMemory{
+		cache: ccache.New(ccache.Configure().MaxSize(maxEntries).ItemsToPrune(uint32(pruneCount))),
+	}
+}
+
+func (c *ChunkCacheInMemory) GetChunk(fileId string) []byte {
+	item := c.cache.Get(fileId)
+	if item == nil {
+		return nil
+	}
+	data := item.Value().([]byte)
+	item.Extend(time.Hour)
+	return data
+}
+
+func (c *ChunkCacheInMemory) readChunkAt(buffer []byte, fileId string, offset uint64) (int, error) {
+	item := c.cache.Get(fileId)
+	if item == nil {
+		return 0, nil
+	}
+	data := item.Value().([]byte)
+	item.Extend(time.Hour)
+	wanted := min(len(buffer), len(data)-int(offset))
+	if wanted < 0 {
+		return 0, ErrorOutOfBounds
+	}
+	n := copy(buffer, data[offset:int(offset)+wanted])
+	return n, nil
+}
+
+func (c *ChunkCacheInMemory) SetChunk(fileId string, data []byte) {
+	localCopy := make([]byte, len(data))
+	copy(localCopy, data)
+	c.cache.Set(fileId, localCopy, time.Hour)
+}

weed/util/chunk_cache/chunk_cache_on_disk.go

@@ -0,0 +1,235 @@
+package chunk_cache
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb/opt"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// This implements an on disk cache
+// The entries are an FIFO with a size limit
+
+type ChunkCacheVolume struct {
+	DataBackend backend.BackendStorageFile
+	nm          storage.NeedleMapper
+	fileName    string
+	smallBuffer []byte
+	sizeLimit   int64
+	lastModTime time.Time
+	fileSize    int64
+}
+
+func LoadOrCreateChunkCacheVolume(fileName string, preallocate int64) (*ChunkCacheVolume, error) {
+
+	v := &ChunkCacheVolume{
+		smallBuffer: make([]byte, types.NeedlePaddingSize),
+		fileName:    fileName,
+		sizeLimit:   preallocate,
+	}
+
+	var err error
+	var indexFile *os.File
+
+	defer func() {
+		if err != nil {
+			if v.DataBackend != nil {
+				v.DataBackend.Close()
+			}
+			if indexFile != nil {
+				indexFile.Close()
+			}
+		}
+	}()
+
+	if exists, canRead, canWrite, modTime, fileSize := util.CheckFile(v.fileName + ".dat"); exists {
+		if !canRead {
+			return nil, fmt.Errorf("cannot read cache file %s.dat", v.fileName)
+		}
+		if !canWrite {
+			return nil, fmt.Errorf("cannot write cache file %s.dat", v.fileName)
+		}
+		var dataFile *os.File
+		if dataFile, err = os.OpenFile(v.fileName+".dat", os.O_RDWR|os.O_CREATE, 0644); err != nil {
+			return nil, fmt.Errorf("cannot create cache file %s.dat: %v", v.fileName, err)
+		}
+		v.DataBackend = backend.NewDiskFile(dataFile)
+		v.lastModTime = modTime
+		v.fileSize = fileSize
+	} else {
+		if v.DataBackend, err = backend.CreateVolumeFile(v.fileName+".dat", preallocate, 0); err != nil {
+			return nil, fmt.Errorf("cannot create cache file %s.dat: %v", v.fileName, err)
+		}
+		v.lastModTime = time.Now()
+	}
+
+	if indexFile, err = os.OpenFile(v.fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); err != nil {
+		return nil, fmt.Errorf("cannot write cache index %s.idx: %v", v.fileName, err)
+	}
+
+	glog.V(1).Infoln("loading leveldb", v.fileName+".ldb")
+	opts := &opt.Options{
+		BlockCacheCapacity:            2 * 1024 * 1024, // default value is 8MiB
+		WriteBuffer:                   1 * 1024 * 1024, // default value is 4MiB
+		CompactionTableSizeMultiplier: 10,              // default value is 1
+	}
+	if v.nm, err = storage.NewLevelDbNeedleMap(v.fileName+".ldb", indexFile, opts, 0, needle.GetCurrentVersion()); err != nil {
+		return nil, fmt.Errorf("loading leveldb %s error: %v", v.fileName+".ldb", err)
+	}
+
+	return v, nil
+
+}
+
+func (v *ChunkCacheVolume) Shutdown() {
+	if v.DataBackend != nil {
+		v.DataBackend.Close()
+		v.DataBackend = nil
+	}
+	if v.nm != nil {
+		v.nm.Close()
+		v.nm = nil
+	}
+}
+
+func (v *ChunkCacheVolume) doReset() {
+	v.Shutdown()
+	fn := v.fileName + ".dat"
+	err := os.Truncate(fn, 0)
+	if err != nil {
+		glog.Errorf("ChunkCacheVolume.doReset: truncate %q failed: %s", fn, err)
+	}
+	fn = v.fileName + ".idx"
+	err = os.Truncate(fn, 0)
+	if err != nil {
+		glog.Errorf("ChunkCacheVolume.doReset: truncate %q failed: %s", fn, err)
+	}
+	fn = v.fileName + ".ldb"
+	err = os.RemoveAll(fn)
+	if err != nil {
+		glog.Errorf("ChunkCacheVolume.doReset: remove %q failed: %s", fn, err)
+	} else {
+		glog.V(4).Infof("cache removed %s", fn)
+	}
+}
+
+func (v *ChunkCacheVolume) Reset() (*ChunkCacheVolume, error) {
+	v.doReset()
+	return LoadOrCreateChunkCacheVolume(v.fileName, v.sizeLimit)
+}
+
+// minFadviseSize is the minimum read size (in bytes) before we call fadvise
+// DONTNEED. For small reads the syscall overhead outweighs the benefit, and
+// the kernel's page cache may serve the data again sooner than we think.
+const minFadviseSize = 1 << 20 // 1 MiB
+
+// dropReadCache advises the kernel to drop page cache for the byte range
+// just read. This is best-effort; failures are logged at V(4).
+// Only applied for reads >= minFadviseSize to avoid syscall overhead on
+// small needle reads where the kernel page cache is more beneficial.
+func (v *ChunkCacheVolume) dropReadCache(offset int64, length int64) {
+	if length < minFadviseSize {
+		return
+	}
+	type fdProvider interface {
+		Fd() uintptr
+	}
+	if fp, ok := v.DataBackend.(fdProvider); ok {
+		fd := int(fp.Fd())
+		if fd < 0 {
+			return
+		}
+		if err := util.DropOSPageCache(fd, offset, length); err != nil {
+			glog.V(4).Infof("fadvise DONTNEED %s offset %d len %d: %v", v.fileName, offset, length, err)
+		}
+	}
+}
+
+func (v *ChunkCacheVolume) GetNeedle(key types.NeedleId) ([]byte, error) {
+
+	nv, ok := v.nm.Get(key)
+	if !ok {
+		return nil, storage.ErrorNotFound
+	}
+	data := make([]byte, nv.Size)
+	readOffset := nv.Offset.ToActualOffset()
+	if readSize, readErr := v.DataBackend.ReadAt(data, readOffset); readErr != nil {
+		if readSize != int(nv.Size) {
+			return nil, fmt.Errorf("read %s.dat [%d,%d): %v",
+				v.fileName, readOffset, readOffset+int64(nv.Size), readErr)
+		}
+	} else {
+		if readSize != int(nv.Size) {
+			return nil, fmt.Errorf("read %d, expected %d", readSize, nv.Size)
+		}
+	}
+
+	v.dropReadCache(readOffset, int64(nv.Size))
+	return data, nil
+}
+
+func (v *ChunkCacheVolume) readNeedleSliceAt(data []byte, key types.NeedleId, offset uint64) (n int, err error) {
+	nv, ok := v.nm.Get(key)
+	if !ok {
+		return 0, storage.ErrorNotFound
+	}
+	wanted := min(len(data), int(nv.Size)-int(offset))
+	if wanted < 0 {
+		// should never happen, but better than panicking
+		return 0, ErrorOutOfBounds
+	}
+	readOffset := nv.Offset.ToActualOffset() + int64(offset)
+	if n, err = v.DataBackend.ReadAt(data[:wanted], readOffset); err != nil {
+		if n != wanted {
+			return n, fmt.Errorf("read %s.dat [%d,%d): %v",
+				v.fileName, readOffset, int64(readOffset)+int64(wanted), err)
+		}
+	} else {
+		if n != wanted {
+			return n, fmt.Errorf("read %d, expected %d", n, wanted)
+		}
+	}
+	if err != nil && n == wanted {
+		err = nil
+	}
+	if n > 0 {
+		v.dropReadCache(readOffset, int64(n))
+	}
+	return n, err
+}
+
+func (v *ChunkCacheVolume) WriteNeedle(key types.NeedleId, data []byte) error {
+
+	offset := v.fileSize
+
+	written, err := v.DataBackend.WriteAt(data, offset)
+	if err != nil {
+		return err
+	} else if written != len(data) {
+		return fmt.Errorf("partial written %d, expected %d", written, len(data))
+	}
+
+	v.fileSize += int64(written)
+	extraSize := written % types.NeedlePaddingSize
+	if extraSize != 0 {
+		_, err = v.DataBackend.WriteAt(v.smallBuffer[:types.NeedlePaddingSize-extraSize], offset+int64(written))
+		if err != nil {
+			return err
+		}
+		v.fileSize += int64(types.NeedlePaddingSize - extraSize)
+	}
+
+	if err := v.nm.Put(key, types.ToOffset(offset), types.Size(len(data))); err != nil {
+		return err
+	}
+
+	return nil
+}

weed/util/chunk_cache/chunk_cache_on_disk_test.go

@@ -0,0 +1,201 @@
+package chunk_cache
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/util/mem"
+)
+
+func TestOnDisk(t *testing.T) {
+	tmpDir := t.TempDir()
+
+	totalDiskSizeInKB := int64(32)
+
+	cache := NewTieredChunkCache(2, tmpDir, totalDiskSizeInKB, 1024)
+
+	writeCount := 5
+	type test_data struct {
+		data   []byte
+		fileId string
+		size   int
+	}
+	testData := make([]*test_data, writeCount)
+	for i := 0; i < writeCount; i++ {
+		buff := make([]byte, 1024)
+		rand.Read(buff)
+		testData[i] = &test_data{
+			data:   buff,
+			fileId: fmt.Sprintf("1,%daabbccdd", i+1),
+			size:   len(buff),
+		}
+		cache.SetChunk(testData[i].fileId, testData[i].data)
+
+		// read back right after write
+		data := mem.Allocate(testData[i].size)
+		cache.ReadChunkAt(data, testData[i].fileId, 0)
+		if !bytes.Equal(data, testData[i].data) {
+			t.Errorf("failed to write to and read from cache: %d", i)
+		}
+		mem.Free(data)
+	}
+
+	// With the new validation system, evicted entries correctly return cache misses (0 bytes)
+	// instead of corrupt data. This is the desired behavior for data integrity.
+	for i := 0; i < 2; i++ {
+		data := mem.Allocate(testData[i].size)
+		n, _ := cache.ReadChunkAt(data, testData[i].fileId, 0)
+		// Entries may be evicted due to cache size constraints - this is acceptable
+		// The important thing is we don't get corrupt data
+		if n > 0 {
+			// If we get data back, it should be correct (not corrupted)
+			if !bytes.Equal(data[:n], testData[i].data[:n]) {
+				t.Errorf("cache returned corrupted data for entry %d", i)
+			}
+		}
+		// Cache miss (n == 0) is acceptable and safe behavior
+		mem.Free(data)
+	}
+
+	for i := 2; i < writeCount; i++ {
+		data := mem.Allocate(testData[i].size)
+		n, _ := cache.ReadChunkAt(data, testData[i].fileId, 0)
+		if n > 0 {
+			// If we get data back, it should be correct
+			if !bytes.Equal(data[:n], testData[i].data[:n]) {
+				t.Errorf("failed to write to and read from cache: %d", i)
+			}
+		} else {
+			// With enhanced validation and cache size limits, cache misses are acceptable
+			// This is safer than returning potentially corrupt data
+			t.Logf("cache miss for entry %d (acceptable with size constraints)", i)
+		}
+		mem.Free(data)
+	}
+
+	cache.Shutdown()
+
+	cache = NewTieredChunkCache(2, tmpDir, totalDiskSizeInKB, 1024)
+
+	// After cache restart, entries may or may not be persisted depending on eviction
+	// With new validation system, we should get either correct data or cache misses
+	for i := 0; i < 2; i++ {
+		data := mem.Allocate(testData[i].size)
+		n, _ := cache.ReadChunkAt(data, testData[i].fileId, 0)
+		if n > 0 {
+			// If we get data back, it should be correct (not corrupted)
+			if !bytes.Equal(data[:n], testData[i].data[:n]) {
+				t.Errorf("cache returned corrupted data for entry %d after restart", i)
+			}
+		}
+		// Cache miss (n == 0) is acceptable and safe behavior after restart
+		mem.Free(data)
+	}
+
+	for i := 2; i < writeCount; i++ {
+		if i == 4 {
+			// FIXME this failed many times on build machines
+			/*
+				I0928 06:04:12 10979 volume_create_linux.go:19] Preallocated 2048 bytes disk space for /tmp/c578652251/c0_2_0.dat
+				I0928 06:04:12 10979 volume_create_linux.go:19] Preallocated 2048 bytes disk space for /tmp/c578652251/c0_2_1.dat
+				I0928 06:04:12 10979 volume_create_linux.go:19] Preallocated 4096 bytes disk space for /tmp/c578652251/c1_3_0.dat
+				I0928 06:04:12 10979 volume_create_linux.go:19] Preallocated 4096 bytes disk space for /tmp/c578652251/c1_3_1.dat
+				I0928 06:04:12 10979 volume_create_linux.go:19] Preallocated 4096 bytes disk space for /tmp/c578652251/c1_3_2.dat
+				I0928 06:04:12 10979 volume_create_linux.go:19] Preallocated 8192 bytes disk space for /tmp/c578652251/c2_2_0.dat
+				I0928 06:04:12 10979 volume_create_linux.go:19] Preallocated 8192 bytes disk space for /tmp/c578652251/c2_2_1.dat
+				I0928 06:04:12 10979 volume_create_linux.go:19] Preallocated 2048 bytes disk space for /tmp/c578652251/c0_2_0.dat
+				I0928 06:04:12 10979 volume_create_linux.go:19] Preallocated 2048 bytes disk space for /tmp/c578652251/c0_2_1.dat
+				--- FAIL: TestOnDisk (0.19s)
+				    chunk_cache_on_disk_test.go:73: failed to write to and read from cache: 4
+				FAIL
+				FAIL	github.com/seaweedfs/seaweedfs/weed/util/chunk_cache	0.199s
+			*/
+			continue
+		}
+		data := mem.Allocate(testData[i].size)
+		n, _ := cache.ReadChunkAt(data, testData[i].fileId, 0)
+		if n > 0 {
+			// If we get data back, it should be correct
+			if !bytes.Equal(data[:n], testData[i].data[:n]) {
+				t.Errorf("failed to write to and read from cache after restart: %d", i)
+			}
+		} else {
+			// Cache miss after restart is acceptable - better safe than corrupt
+			t.Logf("cache miss for entry %d after restart (acceptable)", i)
+		}
+		mem.Free(data)
+	}
+
+	cache.Shutdown()
+
+}
+
+func TestLoadOrCreateChunkCacheVolumeClosesFilesOnError(t *testing.T) {
+	tmpDir := t.TempDir()
+	fileName := filepath.Join(tmpDir, "cache")
+
+	// Pre-create .dat so LoadOrCreateChunkCacheVolume opens it.
+	if err := os.WriteFile(fileName+".dat", []byte("data"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create .ldb as a regular file so NewLevelDbNeedleMap fails.
+	// leveldb.OpenFile expects a directory.
+	if err := os.WriteFile(fileName+".ldb", []byte("not a directory"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	before := openFDCount()
+
+	_, err := LoadOrCreateChunkCacheVolume(fileName, 1024)
+	if err == nil {
+		t.Fatal("expected error")
+	}
+
+	// On POSIX a leaked descriptor keeps the count elevated; the error path
+	// should close both files, returning the count to its baseline.
+	if before >= 0 {
+		if after := openFDCount(); after > before {
+			t.Errorf("descriptors leaked on error: before=%d after=%d", before, after)
+		}
+	}
+
+	// On Windows open files cannot be removed, so successful removal also
+	// confirms the descriptors were closed.
+	if runtime.GOOS == "windows" {
+		for _, ext := range []string{".dat", ".idx"} {
+			path := fileName + ext
+			if _, statErr := os.Stat(path); os.IsNotExist(statErr) {
+				continue
+			}
+			if rmErr := os.Remove(path); rmErr != nil {
+				t.Errorf("failed to remove %s: %v", path, rmErr)
+			}
+		}
+	}
+}
+
+// openFDCount returns the number of open file descriptors for the current
+// process, or -1 where it cannot be determined (e.g. Windows).
+// Readdirnames is used instead of os.ReadDir because the latter stats every
+// entry, which fails on macOS /dev/fd ("bad file descriptor").
+func openFDCount() int {
+	for _, dir := range []string{"/proc/self/fd", "/dev/fd"} {
+		f, err := os.Open(dir)
+		if err != nil {
+			continue
+		}
+		names, err := f.Readdirnames(-1)
+		f.Close()
+		if err != nil {
+			continue
+		}
+		return len(names)
+	}
+	return -1
+}

weed/util/chunk_cache/on_disk_cache_layer.go

@@ -0,0 +1,92 @@
+package chunk_cache
+
+import (
+	"fmt"
+	"path"
+	"slices"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+type OnDiskCacheLayer struct {
+	diskCaches []*ChunkCacheVolume
+}
+
+func NewOnDiskCacheLayer(dir, namePrefix string, diskSize int64, segmentCount int) *OnDiskCacheLayer {
+
+	volumeCount, volumeSize := int(diskSize/(30000*1024*1024)), int64(30000*1024*1024)
+	if volumeCount < segmentCount {
+		volumeCount, volumeSize = segmentCount, diskSize/int64(segmentCount)
+	}
+
+	c := &OnDiskCacheLayer{}
+	for i := 0; i < volumeCount; i++ {
+		fileName := path.Join(dir, fmt.Sprintf("%s_%d", namePrefix, i))
+		diskCache, err := LoadOrCreateChunkCacheVolume(fileName, volumeSize)
+		if err != nil {
+			glog.Errorf("failed to add cache %s : %v", fileName, err)
+		} else {
+			c.diskCaches = append(c.diskCaches, diskCache)
+		}
+	}
+
+	// keep newest cache to the front
+	slices.SortFunc(c.diskCaches, func(a, b *ChunkCacheVolume) int {
+		return b.lastModTime.Compare(a.lastModTime)
+	})
+	return c
+}
+
+func (c *OnDiskCacheLayer) setChunk(needleId types.NeedleId, data []byte) {
+
+	if len(c.diskCaches) == 0 {
+		return
+	}
+
+	if c.diskCaches[0].fileSize+int64(len(data)) > c.diskCaches[0].sizeLimit {
+		t, resetErr := c.diskCaches[len(c.diskCaches)-1].Reset()
+		if resetErr != nil {
+			glog.Errorf("failed to reset cache file %s", c.diskCaches[len(c.diskCaches)-1].fileName)
+			return
+		}
+		for i := len(c.diskCaches) - 1; i > 0; i-- {
+			c.diskCaches[i] = c.diskCaches[i-1]
+		}
+		c.diskCaches[0] = t
+	}
+
+	if err := c.diskCaches[0].WriteNeedle(needleId, data); err != nil {
+		glog.V(0).Infof("cache write %v size %d: %v", needleId, len(data), err)
+	}
+
+}
+
+func (c *OnDiskCacheLayer) readChunkAt(buffer []byte, needleId types.NeedleId, offset uint64) (n int, err error) {
+
+	for _, diskCache := range c.diskCaches {
+		n, err = diskCache.readNeedleSliceAt(buffer, needleId, offset)
+		if err == storage.ErrorNotFound {
+			continue
+		}
+		if err != nil {
+			glog.Warningf("failed to read cache file %s id %d: %v", diskCache.fileName, needleId, err)
+			continue
+		}
+		if n > 0 {
+			return
+		}
+	}
+
+	return
+
+}
+
+func (c *OnDiskCacheLayer) shutdown() {
+
+	for _, diskCache := range c.diskCaches {
+		diskCache.Shutdown()
+	}
+
+}

weed/util/cipher.go

@@ -0,0 +1,60 @@
+package util
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"errors"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+type CipherKey []byte
+
+func GenCipherKey() CipherKey {
+	key := make([]byte, 32)
+	if _, err := io.ReadFull(rand.Reader, key); err != nil {
+		glog.Fatalf("random key gen: %v", err)
+	}
+	return CipherKey(key)
+}
+
+func Encrypt(plaintext []byte, key CipherKey) ([]byte, error) {
+	c, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	gcm, err := cipher.NewGCM(c)
+	if err != nil {
+		return nil, err
+	}
+
+	nonce := make([]byte, gcm.NonceSize())
+	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
+		return nil, err
+	}
+
+	return gcm.Seal(nonce, nonce, plaintext, nil), nil
+}
+
+func Decrypt(ciphertext []byte, key CipherKey) ([]byte, error) {
+	c, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	gcm, err := cipher.NewGCM(c)
+	if err != nil {
+		return nil, err
+	}
+
+	nonceSize := gcm.NonceSize()
+	if len(ciphertext) < nonceSize {
+		return nil, errors.New("ciphertext too short")
+	}
+
+	nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:]
+	return gcm.Open(nil, nonce, ciphertext, nil)
+}

weed/util/cipher_test.go

@@ -0,0 +1,17 @@
+package util
+
+import (
+	"encoding/base64"
+	"testing"
+)
+
+func TestSameAsJavaImplementation(t *testing.T) {
+	str := "QVVhmqg112NMT7F+G/7QPynqSln3xPIhKdFGmTVKZD6IS0noyr2Z5kXFF6fPjZ/7Hq8kRhlmLeeqZUccxyaZHezOdgkjS6d4NTdHf5IjXzk7"
+	cipherText, _ := base64.StdEncoding.DecodeString(str)
+	secretKey := []byte("256-bit key for AES 256 GCM encr")
+	plaintext, err := Decrypt(cipherText, CipherKey(secretKey))
+	if err != nil {
+		println(err.Error())
+	}
+	println(string(plaintext))
+}

weed/util/compression.go

@@ -0,0 +1,153 @@
+package util
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	// "github.com/klauspost/compress/zstd"
+)
+
+var (
+	UnsupportedCompression = fmt.Errorf("unsupported compression")
+)
+
+func MaybeGzipData(input []byte) []byte {
+	if IsGzippedContent(input) {
+		return input
+	}
+	gzipped, err := GzipData(input)
+	if err != nil {
+		return input
+	}
+	if len(gzipped)*10 > len(input)*9 {
+		return input
+	}
+	return gzipped
+}
+
+func MaybeDecompressData(input []byte) []byte {
+	uncompressed, err := DecompressData(input)
+	if err != nil {
+		if err != UnsupportedCompression {
+			glog.Errorf("decompressed data: %v", err)
+		}
+		return input
+	}
+	return uncompressed
+}
+
+func GzipData(input []byte) ([]byte, error) {
+	w := new(bytes.Buffer)
+	_, err := GzipStream(w, bytes.NewReader(input))
+	if err != nil {
+		return nil, err
+	}
+	return w.Bytes(), nil
+}
+
+func ungzipData(input []byte) ([]byte, error) {
+	w := new(bytes.Buffer)
+	_, err := GunzipStream(w, bytes.NewReader(input))
+	if err != nil {
+		return nil, err
+	}
+	return w.Bytes(), nil
+}
+
+func DecompressData(input []byte) ([]byte, error) {
+	if IsGzippedContent(input) {
+		return ungzipData(input)
+	}
+	/*
+		if IsZstdContent(input) {
+			return unzstdData(input)
+		}
+	*/
+	return input, UnsupportedCompression
+}
+
+func IsGzippedContent(data []byte) bool {
+	if len(data) < 2 {
+		return false
+	}
+	return data[0] == 31 && data[1] == 139
+}
+
+/*
+var zstdEncoder, _ = zstd.NewWriter(nil)
+
+func ZstdData(input []byte) ([]byte, error) {
+	return zstdEncoder.EncodeAll(input, nil), nil
+}
+
+var decoder, _ = zstd.NewReader(nil)
+
+func unzstdData(input []byte) ([]byte, error) {
+	return decoder.DecodeAll(input, nil)
+}
+
+func IsZstdContent(data []byte) bool {
+	if len(data) < 4 {
+		return false
+	}
+	return data[3] == 0xFD && data[2] == 0x2F && data[1] == 0xB5 && data[0] == 0x28
+}
+*/
+
+/*
+* Default not to compressed since compression can be done on client side.
+ */func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool) {
+
+	// text
+	if strings.HasPrefix(mtype, "text/") {
+		return true, true
+	}
+
+	// images
+	switch ext {
+	case ".svg", ".bmp", ".wav":
+		return true, true
+	}
+	if strings.HasPrefix(mtype, "image/") {
+		return false, true
+	}
+
+	// by file name extension
+	switch ext {
+	case ".zip", ".rar", ".gz", ".bz2", ".xz", ".zst", ".br":
+		return false, true
+	case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
+		return true, true
+	case ".php", ".java", ".go", ".rb", ".c", ".cpp", ".h", ".hpp":
+		return true, true
+	case ".png", ".jpg", ".jpeg":
+		return false, true
+	}
+
+	// by mime type
+	if strings.HasPrefix(mtype, "application/") {
+		if strings.HasSuffix(mtype, "zstd") {
+			return false, true
+		}
+		if strings.HasSuffix(mtype, "xml") {
+			return true, true
+		}
+		if strings.HasSuffix(mtype, "script") {
+			return true, true
+		}
+		if strings.HasSuffix(mtype, "vnd.rar") {
+			return false, true
+		}
+	}
+
+	if strings.HasPrefix(mtype, "audio/") {
+		switch strings.TrimPrefix(mtype, "audio/") {
+		case "wave", "wav", "x-wav", "x-pn-wav":
+			return true, true
+		}
+	}
+
+	return false, false
+}

weed/util/compression_stream.go

@@ -0,0 +1,53 @@
+package util
+
+import (
+	"compress/gzip"
+	"fmt"
+	"io"
+	"sync"
+)
+
+var (
+	gzipReaderPool = sync.Pool{
+		New: func() interface{} {
+			return new(gzip.Reader)
+			//return gzip.NewReader()
+		},
+	}
+
+	gzipWriterPool = sync.Pool{
+		New: func() interface{} {
+			w, _ := gzip.NewWriterLevel(nil, gzip.BestSpeed)
+			return w
+		},
+	}
+)
+
+func GzipStream(w io.Writer, r io.Reader) (int64, error) {
+	gw, ok := gzipWriterPool.Get().(*gzip.Writer)
+	if !ok {
+		return 0, fmt.Errorf("gzip: new writer error")
+	}
+	gw.Reset(w)
+	defer func() {
+		gw.Close()
+		gzipWriterPool.Put(gw)
+	}()
+	return io.Copy(gw, r)
+}
+
+func GunzipStream(w io.Writer, r io.Reader) (int64, error) {
+	gr, ok := gzipReaderPool.Get().(*gzip.Reader)
+	if !ok {
+		return 0, fmt.Errorf("gzip: new reader error")
+	}
+
+	if err := gr.Reset(r); err != nil {
+		return 0, err
+	}
+	defer func() {
+		gr.Close()
+		gzipReaderPool.Put(gr)
+	}()
+	return io.Copy(w, gr)
+}

weed/util/concurrent_read_map.go

@@ -0,0 +1,64 @@
+package util
+
+import (
+	"sync"
+)
+
+// A mostly for read map, which can thread-safely
+// initialize the map entries.
+type ConcurrentReadMap struct {
+	sync.RWMutex
+
+	items map[string]interface{}
+}
+
+func NewConcurrentReadMap() *ConcurrentReadMap {
+	return &ConcurrentReadMap{items: make(map[string]interface{})}
+}
+
+func (m *ConcurrentReadMap) initMapEntry(key string, newEntry func() interface{}) (value interface{}) {
+	m.Lock()
+	defer m.Unlock()
+	if value, ok := m.items[key]; ok {
+		return value
+	}
+	value = newEntry()
+	m.items[key] = value
+	return value
+}
+
+func (m *ConcurrentReadMap) Get(key string, newEntry func() interface{}) interface{} {
+	m.RLock()
+	if value, ok := m.items[key]; ok {
+		m.RUnlock()
+		return value
+	}
+	m.RUnlock()
+	return m.initMapEntry(key, newEntry)
+}
+
+func (m *ConcurrentReadMap) Find(key string) (interface{}, bool) {
+	m.RLock()
+	value, ok := m.items[key]
+	m.RUnlock()
+	return value, ok
+}
+
+func (m *ConcurrentReadMap) Items() (itemsCopy []interface{}) {
+	m.RLock()
+	for _, i := range m.items {
+		itemsCopy = append(itemsCopy, i)
+	}
+	m.RUnlock()
+	return itemsCopy
+}
+
+// Delete removes the key and returns what it held, so a caller that has to
+// wind the entry down does not race another deleter for it.
+func (m *ConcurrentReadMap) Delete(key string) (interface{}, bool) {
+	m.Lock()
+	value, ok := m.items[key]
+	delete(m.items, key)
+	m.Unlock()
+	return value, ok
+}

weed/util/cond_wait.go

@@ -0,0 +1,32 @@
+package util
+
+import (
+	"context"
+	"net/http"
+	"sync"
+	"time"
+)
+
+const HttpStatusCancelled = 499
+
+func WaitWithTimeout(ctx context.Context, cond *sync.Cond, timer *time.Timer) int {
+	waitDone := make(chan struct{})
+
+	go func() {
+		cond.L.Lock()
+		defer cond.L.Unlock()
+		cond.Wait()
+		defer close(waitDone)
+	}()
+
+	select {
+	case <-waitDone:
+		return http.StatusOK
+	case <-timer.C:
+		cond.Broadcast()
+		return http.StatusTooManyRequests
+	case <-ctx.Done():
+		cond.Broadcast()
+		return HttpStatusCancelled
+	}
+}

weed/util/config.go

@@ -0,0 +1,126 @@
+package util
+
+import (
+	"strings"
+	"sync"
+
+	"github.com/spf13/viper"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+var (
+	ConfigurationFileDirectory DirectoryValueType
+	loadSecurityConfigOnce     sync.Once
+)
+
+type DirectoryValueType string
+
+func (s *DirectoryValueType) Set(value string) error {
+	*s = DirectoryValueType(value)
+	return nil
+}
+func (s *DirectoryValueType) String() string {
+	return string(*s)
+}
+
+type Configuration interface {
+	GetString(key string) string
+	GetBool(key string) bool
+	GetInt(key string) int
+	GetStringSlice(key string) []string
+	SetDefault(key string, value interface{})
+}
+
+func LoadSecurityConfiguration() {
+	loadSecurityConfigOnce.Do(func() {
+		LoadConfiguration("security", false)
+	})
+}
+
+func LoadConfiguration(configFileName string, required bool) (loaded bool) {
+
+	// find a filer store
+	viper.SetConfigName(configFileName)                                   // name of config file (without extension)
+	viper.AddConfigPath(ResolvePath(ConfigurationFileDirectory.String())) // path to look for the config file in
+	viper.AddConfigPath(".")                                              // optionally look for config in the working directory
+	viper.AddConfigPath("$HOME/.seaweedfs")                               // call multiple times to add many search paths
+	viper.AddConfigPath("/usr/local/etc/seaweedfs/")                      // search path for bsd-style config directory in
+	viper.AddConfigPath("/etc/seaweedfs/")                                // path to look for the config file in
+
+	if err := viper.MergeInConfig(); err != nil { // Handle errors reading the config file
+		if strings.Contains(err.Error(), "Not Found") {
+			glog.V(1).Infof("Reading %s: %v", viper.ConfigFileUsed(), err)
+		} else {
+			// If the config is required, fail immediately
+			if required {
+				glog.Fatalf("Reading %s: %v", viper.ConfigFileUsed(), err)
+			}
+			// If the config is optional, log a warning but don't crash
+			glog.Warningf("Reading %s: %v. Skipping optional configuration.", viper.ConfigFileUsed(), err)
+		}
+		if required {
+			glog.Fatalf("Failed to load %s.toml file from current directory, or $HOME/.seaweedfs/, or /etc/seaweedfs/"+
+				"\n\nPlease use this command to generate the default %s.toml file\n"+
+				"    weed scaffold -config=%s -output=.\n\n\n",
+				configFileName, configFileName, configFileName)
+		}
+		return false
+	}
+	glog.V(1).Infof("Reading %s.toml from %s", configFileName, viper.ConfigFileUsed())
+
+	return true
+}
+
+type ViperProxy struct {
+	*viper.Viper
+	sync.Mutex
+}
+
+var (
+	vp = &ViperProxy{}
+)
+
+func (vp *ViperProxy) SetDefault(key string, value interface{}) {
+	vp.Lock()
+	defer vp.Unlock()
+	vp.Viper.SetDefault(key, value)
+}
+
+func (vp *ViperProxy) GetString(key string) string {
+	vp.Lock()
+	defer vp.Unlock()
+	return vp.Viper.GetString(key)
+}
+
+func (vp *ViperProxy) GetBool(key string) bool {
+	vp.Lock()
+	defer vp.Unlock()
+	return vp.Viper.GetBool(key)
+}
+
+func (vp *ViperProxy) GetInt(key string) int {
+	vp.Lock()
+	defer vp.Unlock()
+	return vp.Viper.GetInt(key)
+}
+
+func (vp *ViperProxy) GetStringSlice(key string) []string {
+	vp.Lock()
+	defer vp.Unlock()
+	return vp.Viper.GetStringSlice(key)
+}
+
+func GetViper() *ViperProxy {
+	vp.Lock()
+	defer vp.Unlock()
+
+	if vp.Viper == nil {
+		vp.Viper = viper.GetViper()
+		vp.AutomaticEnv()
+		vp.SetEnvPrefix("weed")
+		vp.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
+	}
+
+	return vp
+}

weed/util/constants/filer.go

@@ -0,0 +1,15 @@
+package constants
+
+// Filer error messages — shared between filer (producing) and S3 API (matching).
+// These cross a gRPC boundary as strings in CreateEntryResponse.Error, so typed
+// errors cannot be used. Keeping the strings in one place prevents mismatches.
+const (
+	ErrMsgOperationNotPermitted = "operation not permitted"
+	ErrMsgBadDigest             = "The Content-Md5 you specified did not match what we received."
+	ErrMsgEntryNameTooLong      = "entry name too long"
+	// Suffix appended to a path, e.g. "/buckets/b/foo/bar is a file"
+	ErrMsgIsAFile = " is a file"
+	// Prefix + suffix, e.g. "existing /buckets/b/foo is a directory"
+	ErrMsgExistingPrefix = "existing "
+	ErrMsgIsADirectory   = " is a directory"
+)

weed/util/constants_4bytes.go

@@ -0,0 +1,8 @@
+//go:build !5BytesOffset
+
+package util
+
+const (
+	SizeLimit         = "30GB"
+	VolumeSizeLimitGB = 30
+)

weed/util/constants_5bytes.go

@@ -0,0 +1,8 @@
+//go:build 5BytesOffset
+
+package util
+
+const (
+	SizeLimit         = "8000GB"
+	VolumeSizeLimitGB = 8000
+)

weed/util/constants_lifecycle_interval_10sec.go

@@ -0,0 +1,7 @@
+//go:build s3tests
+
+package util
+
+import "time"
+
+const LifeCycleInterval = 10 * time.Second

weed/util/constants_lifecycle_interval_day.go

@@ -0,0 +1,7 @@
+//go:build !s3tests
+
+package util
+
+import "time"
+
+const LifeCycleInterval = 24 * time.Hour

weed/util/error_wait_group.go

@@ -0,0 +1,94 @@
+package util
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+)
+
+// ErrorWaitGroup implements a goroutine wait group which aggregates errors, if any.
+type ErrorWaitGroup struct {
+	maxConcurrency int
+	wg             *sync.WaitGroup
+	wgSem          chan bool
+	errors         []error
+	errorsMu       sync.Mutex
+}
+
+type ErrorWaitGroupTask func() error
+
+// ExecuteParallelTaskGroups runs tasks from each group sequentially and runs
+// independent groups with up to maxParallelization concurrency. EC balancing
+// uses one group per volume because its shard sidecar files are shared; volume
+// balancing uses one group per already-reserved volume move.
+func ExecuteParallelTaskGroups(maxParallelization int, taskGroups [][]ErrorWaitGroupTask) error {
+	ewg := NewErrorWaitGroup(maxParallelization)
+	for _, taskGroup := range taskGroups {
+		taskGroup := taskGroup
+		ewg.Add(func() error {
+			for _, task := range taskGroup {
+				if err := task(); err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+	}
+	return ewg.Wait()
+}
+
+func NewErrorWaitGroup(maxConcurrency int) *ErrorWaitGroup {
+	if maxConcurrency <= 0 {
+		// no concurrency = one task at the time
+		maxConcurrency = 1
+	}
+	return &ErrorWaitGroup{
+		maxConcurrency: maxConcurrency,
+		wg:             &sync.WaitGroup{},
+		wgSem:          make(chan bool, maxConcurrency),
+	}
+}
+
+// Reset restarts an ErrorWaitGroup, keeping original settings. Errors and pending goroutines, if any, are flushed.
+func (ewg *ErrorWaitGroup) Reset() {
+	close(ewg.wgSem)
+
+	ewg.wg = &sync.WaitGroup{}
+	ewg.wgSem = make(chan bool, ewg.maxConcurrency)
+	ewg.errors = nil
+}
+
+// Add queues an ErrorWaitGroupTask to be executed as a goroutine.
+func (ewg *ErrorWaitGroup) Add(f ErrorWaitGroupTask) {
+	if ewg.maxConcurrency <= 1 {
+		// keep run order deterministic when parallelization is off
+		ewg.errors = append(ewg.errors, f())
+		return
+	}
+
+	ewg.wg.Add(1)
+	go func() {
+		ewg.wgSem <- true
+
+		err := f()
+		ewg.errorsMu.Lock()
+		ewg.errors = append(ewg.errors, err)
+		ewg.errorsMu.Unlock()
+
+		<-ewg.wgSem
+		ewg.wg.Done()
+	}()
+}
+
+// AddErrorf adds an error to an ErrorWaitGroupTask result, without queueing any goroutines.
+func (ewg *ErrorWaitGroup) AddErrorf(format string, a ...interface{}) {
+	ewg.errorsMu.Lock()
+	ewg.errors = append(ewg.errors, fmt.Errorf(format, a...))
+	ewg.errorsMu.Unlock()
+}
+
+// Wait sleeps until all ErrorWaitGroupTasks are completed, then returns errors for them.
+func (ewg *ErrorWaitGroup) Wait() error {
+	ewg.wg.Wait()
+	return errors.Join(ewg.errors...)
+}

weed/util/error_wait_group_test.go

@@ -0,0 +1,94 @@
+package util
+
+import (
+	"errors"
+	"testing"
+	"time"
+)
+
+func TestExecuteParallelTaskGroups(t *testing.T) {
+	firstTaskStarted := make(chan struct{})
+	firstTaskRelease := make(chan struct{})
+	secondTaskStarted := make(chan struct{})
+	secondTaskInFirstGroupStarted := make(chan struct{})
+	done := make(chan error, 1)
+
+	go func() {
+		done <- ExecuteParallelTaskGroups(2, [][]ErrorWaitGroupTask{
+			{
+				func() error {
+					close(firstTaskStarted)
+					<-firstTaskRelease
+					return nil
+				},
+				func() error {
+					close(secondTaskInFirstGroupStarted)
+					return nil
+				},
+			},
+			{
+				func() error {
+					close(secondTaskStarted)
+					return nil
+				},
+			},
+		})
+	}()
+
+	select {
+	case <-firstTaskStarted:
+	case <-time.After(time.Second):
+		t.Fatal("first task did not start")
+	}
+	select {
+	case <-secondTaskStarted:
+	case <-time.After(time.Second):
+		t.Fatal("independent task group did not start in parallel")
+	}
+	select {
+	case <-secondTaskInFirstGroupStarted:
+		t.Fatal("tasks in the same group ran in parallel")
+	default:
+	}
+
+	close(firstTaskRelease)
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Fatalf("execute parallel task groups: %v", err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("parallel task groups did not finish")
+	}
+}
+
+func TestExecuteParallelTaskGroupsStopsOnlyFailedGroup(t *testing.T) {
+	expectedErr := errors.New("move failed")
+	failedGroupContinued := false
+	otherGroupRan := false
+
+	err := ExecuteParallelTaskGroups(1, [][]ErrorWaitGroupTask{
+		{
+			func() error { return expectedErr },
+			func() error {
+				failedGroupContinued = true
+				return nil
+			},
+		},
+		{
+			func() error {
+				otherGroupRan = true
+				return nil
+			},
+		},
+	})
+	if !errors.Is(err, expectedErr) {
+		t.Fatalf("expected task error, got %v", err)
+	}
+	if failedGroupContinued {
+		t.Fatal("task after a failed task in the same group ran")
+	}
+	if !otherGroupRan {
+		t.Fatal("independent task group did not run")
+	}
+}

weed/util/fadvise_linux.go

@@ -0,0 +1,13 @@
+//go:build linux
+
+package util
+
+import "golang.org/x/sys/unix"
+
+// DropOSPageCache advises the kernel that the given byte range is no longer
+// needed in the page cache. This is useful after reading from a user-space
+// cache (e.g., on-disk chunk cache) to prevent the kernel from double-caching
+// the same data.
+func DropOSPageCache(fd int, offset int64, length int64) error {
+	return unix.Fadvise(fd, offset, length, unix.FADV_DONTNEED)
+}

weed/util/fadvise_linux_test.go

@@ -0,0 +1,37 @@
+//go:build linux
+
+package util
+
+import (
+	"os"
+	"testing"
+)
+
+func TestDropOSPageCache(t *testing.T) {
+	f, err := os.CreateTemp("", "fadvise_test")
+	if err != nil {
+		t.Fatalf("failed to create temp file: %v", err)
+	}
+	defer os.Remove(f.Name())
+	defer f.Close()
+
+	data := make([]byte, 4096)
+	for i := range data {
+		data[i] = byte(i % 256)
+	}
+	if _, err := f.Write(data); err != nil {
+		t.Fatalf("failed to write test data: %v", err)
+	}
+
+	// Read the data back to populate page cache
+	buf := make([]byte, 4096)
+	if _, err := f.ReadAt(buf, 0); err != nil {
+		t.Fatalf("failed to read test data: %v", err)
+	}
+
+	// Call DropOSPageCache and verify no error
+	fd := int(f.Fd())
+	if err := DropOSPageCache(fd, 0, int64(len(data))); err != nil {
+		t.Errorf("DropOSPageCache returned error: %v", err)
+	}
+}

weed/util/fadvise_other.go

@@ -0,0 +1,8 @@
+//go:build !linux
+
+package util
+
+// DropOSPageCache is a no-op on non-Linux platforms.
+func DropOSPageCache(fd int, offset int64, length int64) error {
+	return nil
+}

weed/util/file_util.go

@@ -0,0 +1,210 @@
+package util
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+const maxFilenameLength = 255
+
+func TestFolderWritable(folder string) (err error) {
+	fileInfo, err := os.Stat(folder)
+	if err != nil {
+		return err
+	}
+	if !fileInfo.IsDir() {
+		return errors.New("Not a valid folder!")
+	}
+	perm := fileInfo.Mode().Perm()
+	glog.V(0).Infoln("Folder", folder, "Permission:", perm)
+	if 0200&perm != 0 {
+		return nil
+	}
+	return errors.New("Not writable!")
+}
+
+func GetFileSize(file *os.File) (size int64, err error) {
+	var fi os.FileInfo
+	if fi, err = file.Stat(); err == nil {
+		size = fi.Size()
+	}
+	return
+}
+
+// FsyncDir flushes a directory entry so a rename/create/unlink inside it
+// survives a crash. Directory fsync is not supported on every platform
+// (notably Windows), where it is skipped rather than treated as an error.
+func FsyncDir(dir string) error {
+	if runtime.GOOS == "windows" {
+		return nil
+	}
+	d, err := os.Open(dir)
+	if err != nil {
+		return err
+	}
+	defer d.Close()
+	if err = d.Sync(); err != nil && !errors.Is(err, os.ErrInvalid) {
+		return fmt.Errorf("sync dir %s: %v", dir, err)
+	}
+	return nil
+}
+
+func FileExists(filename string) bool {
+
+	_, err := os.Stat(filename)
+	if os.IsNotExist(err) {
+		return false
+	}
+	return true
+
+}
+
+func FolderExists(folder string) bool {
+
+	fileInfo, err := os.Stat(folder)
+	if err != nil {
+		return false
+	}
+	return fileInfo.IsDir()
+
+}
+
+func CheckFile(filename string) (exists, canRead, canWrite bool, modTime time.Time, fileSize int64) {
+	exists = true
+	fi, err := os.Stat(filename)
+	if os.IsNotExist(err) {
+		exists = false
+		return
+	}
+	if err != nil {
+		glog.Errorf("check %s: %v", filename, err)
+		return
+	}
+	if fi.Mode()&0400 != 0 {
+		canRead = true
+	}
+	if fi.Mode()&0200 != 0 {
+		canWrite = true
+	}
+	modTime = fi.ModTime()
+	fileSize = fi.Size()
+	return
+}
+
+// ResolvePath expands a leading "~", "~/", or "~username/" in path to the
+// corresponding home directory, mirroring shell tilde expansion. A path
+// without a leading "~" or whose tilde cannot be resolved is returned
+// unchanged so callers can pass any user-supplied path through this helper
+// without worrying about non-tilde inputs or lookup failures.
+//
+// Forward slashes are always recognised as separators after the tilde;
+// the platform-native separator is also accepted so "~\\data" works on
+// Windows. Backslashes are deliberately not treated as separators on
+// other platforms because they are legal characters in usernames and
+// path segments there.
+func ResolvePath(path string) string {
+
+	if !strings.HasPrefix(path, "~") {
+		return path
+	}
+
+	if path == "~" {
+		if usr, err := user.Current(); err == nil {
+			return usr.HomeDir
+		}
+		return path
+	}
+
+	isSep := func(b byte) bool {
+		return b == '/' || b == byte(filepath.Separator)
+	}
+
+	if isSep(path[1]) {
+		if usr, err := user.Current(); err == nil {
+			return filepath.Join(usr.HomeDir, path[2:])
+		}
+		return path
+	}
+
+	// "~username" or "~username<sep>rest"
+	name := path[1:]
+	rest := ""
+	for i := 0; i < len(name); i++ {
+		if isSep(name[i]) {
+			rest = name[i+1:]
+			name = name[:i]
+			break
+		}
+	}
+	usr, err := user.Lookup(name)
+	if err != nil {
+		return path
+	}
+	if rest == "" {
+		return usr.HomeDir
+	}
+	return filepath.Join(usr.HomeDir, rest)
+}
+
+// ResolveCommaSeparatedPaths splits paths on "," and runs each entry through
+// ResolvePath, then rejoins them. This lets flags like `weed mini -dir` or
+// `weed volume -dir` accept tilde-prefixed entries (e.g. "~/d1,~/d2") even
+// in the comma-separated form the help text advertises.
+func ResolveCommaSeparatedPaths(paths string) string {
+	if !strings.Contains(paths, "~") {
+		return paths
+	}
+	parts := strings.Split(paths, ",")
+	for i, p := range parts {
+		parts[i] = ResolvePath(p)
+	}
+	return strings.Join(parts, ",")
+}
+
+func FileNameBase(filename string) string {
+	lastDotIndex := strings.LastIndex(filename, ".")
+	if lastDotIndex < 0 {
+		return filename
+	}
+	return filename[:lastDotIndex]
+}
+
+func ToShortFileName(path string) string {
+	fileName := filepath.Base(path)
+	if fileNameBytes := []byte(fileName); len(fileNameBytes) > maxFilenameLength {
+		shaStr := fmt.Sprintf("%x", sha256.Sum256(fileNameBytes))
+		fileNameBase := FileNameBase(fileName)
+		fileExt := fileName[len(fileNameBase):]
+		fileNameBaseBates := bytes.ToValidUTF8([]byte(fileNameBase)[:maxFilenameLength-len([]byte(fileExt))-8], []byte{})
+		shortFileName := string(fileNameBaseBates) + shaStr[len(shaStr)-8:]
+		return filepath.Join(filepath.Dir(path), shortFileName) + fileExt
+	}
+	return path
+}
+
+// Copied from os.WriteFile(), adding file sync.
+// see https://github.com/golang/go/issues/20599
+func WriteFile(name string, data []byte, perm os.FileMode) error {
+	f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(data)
+	if err1 := f.Sync(); err1 != nil && err == nil {
+		err = err1
+	}
+	if err1 := f.Close(); err1 != nil && err == nil {
+		err = err1
+	}
+	return err
+}

weed/util/file_util_non_posix.go

@@ -0,0 +1,12 @@
+//go:build linux || darwin || freebsd || netbsd || openbsd || plan9 || solaris || zos
+
+package util
+
+import (
+	"os"
+	"syscall"
+)
+
+func GetFileUidGid(fi os.FileInfo) (uid, gid uint32) {
+	return fi.Sys().(*syscall.Stat_t).Uid, fi.Sys().(*syscall.Stat_t).Gid
+}

weed/util/file_util_posix.go

@@ -0,0 +1,11 @@
+//go:build windows
+
+package util
+
+import (
+	"os"
+)
+
+func GetFileUidGid(fi os.FileInfo) (uid, gid uint32) {
+	return 0, 0
+}

weed/util/file_util_test.go

@@ -0,0 +1,88 @@
+package util
+
+import (
+	"os/user"
+	"path/filepath"
+	"testing"
+)
+
+func TestToShortFileName(t *testing.T) {
+	tests := []struct {
+		in    string
+		value string
+	}{
+		{"/data/a/b/c/d.txt", "/data/a/b/c/d.txt"},
+		{"/data/a/b/c/очень_длинное_имя_файла_c_подробным_указанием_наименования_и_содержания_стандартизованных_форм_за_анварь_-_июнь_2023_года(РОГА_И_КОПЫТА_ООО).txt", "/data/a/b/c/очень_длинное_имя_файла_c_подробным_указанием_наименования_и_содержания_стандартизованных_форм_за_анварь_-_июнь_2023_года(РОГА_И_КОПЫТ354fcaf4.txt"},
+		{"/data/a/b/c/очень_длинное_имя_файла_c_подробным_указанием_наименования_и_содержания_стандартизованных_форм_за_анварь_-_июнь_2023_года(РОГА_И_КОПЫТА_ООО)_without_extension", "/data/a/b/c/очень_длинное_имя_файла_c_подробным_указанием_наименования_и_содержания_стандартизованных_форм_за_анварь_-_июнь_2023_года(РОГА_И_КОПЫТА_О21a6e47a"},
+	}
+	for _, p := range tests {
+		got := ToShortFileName(p.in)
+		if got != p.value {
+			t.Errorf("failed to test: got %v, want %v", got, p.value)
+		}
+	}
+}
+
+func TestResolvePath(t *testing.T) {
+	usr, err := user.Current()
+	if err != nil {
+		t.Fatalf("user.Current: %v", err)
+	}
+	home := usr.HomeDir
+
+	cases := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{"empty", "", ""},
+		{"absolute", "/var/data", "/var/data"},
+		{"relative", "data", "data"},
+		{"tilde mid path is literal", "/foo/~/bar", "/foo/~/bar"},
+		{"bare tilde", "~", home},
+		{"tilde slash", "~/", home},
+		{"tilde with subpath", "~/data", filepath.Join(home, "data")},
+		{"tilde with current user", "~" + usr.Username, home},
+		{"tilde with current user and subpath", "~" + usr.Username + "/data", filepath.Join(home, "data")},
+		{"tilde unknown user falls back to literal", "~no-such-user-xyz/data", "~no-such-user-xyz/data"},
+		// Native separator: identical to "~/data" on Unix; exercises backslash on Windows.
+		{"tilde with native separator", "~" + string(filepath.Separator) + "data", filepath.Join(home, "data")},
+		{"tilde user with native separator", "~" + usr.Username + string(filepath.Separator) + "data", filepath.Join(home, "data")},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			if got := ResolvePath(c.in); got != c.want {
+				t.Errorf("ResolvePath(%q) = %q; want %q", c.in, got, c.want)
+			}
+		})
+	}
+}
+
+func TestResolveCommaSeparatedPaths(t *testing.T) {
+	usr, err := user.Current()
+	if err != nil {
+		t.Fatalf("user.Current: %v", err)
+	}
+	home := usr.HomeDir
+
+	cases := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{"empty", "", ""},
+		{"single absolute", "/a", "/a"},
+		{"single tilde", "~/a", filepath.Join(home, "a")},
+		{"two absolutes", "/a,/b", "/a,/b"},
+		{"two tildes", "~/a,~/b", filepath.Join(home, "a") + "," + filepath.Join(home, "b")},
+		{"mixed", "/a,~/b,/c", "/a," + filepath.Join(home, "b") + ",/c"},
+		{"no tilde fast path", "/a,/b,/c", "/a,/b,/c"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			if got := ResolveCommaSeparatedPaths(c.in); got != c.want {
+				t.Errorf("ResolveCommaSeparatedPaths(%q) = %q; want %q", c.in, got, c.want)
+			}
+		})
+	}
+}

weed/util/fullpath.go

@@ -0,0 +1,162 @@
+package util
+
+import (
+	"path"
+	"strings"
+	"unicode/utf8"
+)
+
+type FullPath string
+
+// invalidUTF8Replacement is the single-byte replacement used everywhere a name
+// or path from an untrusted source (kernel FUSE input, external clients, store
+// imports) may contain bytes that are not valid UTF-8. Proto3 `string` fields
+// require valid UTF-8, so any such bytes must be substituted before the value
+// enters a gRPC request; otherwise marshaling fails for the whole RPC.
+//
+// '_' is URL-safe: these sanitized strings also flow into HTTP URLs
+// (volume-server uploads, filer HTTP API, S3/WebDAV gateways). Using '?'
+// would cause it to be interpreted as the query-string delimiter the first
+// time the name lands in a URL and split the path.
+const invalidUTF8Replacement = "_"
+
+// SanitizeUTF8Name replaces every invalid-UTF-8 byte in s with
+// invalidUTF8Replacement. For the common, valid-UTF-8 case the input is
+// returned unchanged with no allocation. Use this for any byte sequence
+// that will be assigned to a proto string field (names, paths) from an
+// untrusted source; centralising the replacement keeps the chosen character
+// consistent across the codebase.
+func SanitizeUTF8Name(s string) string {
+	if utf8.ValidString(s) {
+		return s
+	}
+	return strings.ToValidUTF8(s, invalidUTF8Replacement)
+}
+
+func NewFullPath(dir, name string) FullPath {
+	name = strings.TrimSuffix(name, "/")
+	return FullPath(dir).Child(name)
+}
+
+func (fp FullPath) DirAndName() (string, string) {
+	dir, name := path.Split(string(fp))
+	name = SanitizeUTF8Name(name)
+	if dir == "/" {
+		return dir, name
+	}
+	if len(dir) < 1 {
+		return "/", ""
+	}
+	return dir[:len(dir)-1], name
+}
+
+// Name returns the last path component, with any invalid-UTF-8 bytes replaced
+// via SanitizeUTF8Name so the result is always safe to place in a proto
+// string field or HTTP URL.
+func (fp FullPath) Name() string {
+	_, name := path.Split(string(fp))
+	return SanitizeUTF8Name(name)
+}
+
+// Sanitized returns the full path with every invalid-UTF-8 byte — in any
+// component, not just the last — replaced via SanitizeUTF8Name. Use this
+// before assigning the path to a proto string field (e.g. Directory,
+// AssignVolumeRequest.Path) when the path may have been produced from
+// sources that do not enforce UTF-8 (cache populated from an external
+// store, legacy metadata, shell traversals of existing filer entries).
+func (fp FullPath) Sanitized() string {
+	return SanitizeUTF8Name(string(fp))
+}
+
+func (fp FullPath) IsLongerFileName(maxFilenameLength uint32) bool {
+	if maxFilenameLength == 0 {
+		return false
+	}
+	return uint32(len([]byte(fp.Name()))) > maxFilenameLength
+}
+
+func (fp FullPath) Child(name string) FullPath {
+	dir := string(fp)
+	noPrefix := name
+	if strings.HasPrefix(name, "/") {
+		noPrefix = name[1:]
+	}
+	if strings.HasSuffix(dir, "/") {
+		return FullPath(dir + noPrefix)
+	}
+	return FullPath(dir + "/" + noPrefix)
+}
+
+// AsInode an in-memory only inode representation
+func (fp FullPath) AsInode(unixTime int64) uint64 {
+	inode := uint64(HashStringToLong(string(fp)))
+	inode = inode + uint64(unixTime)*37
+	return inode
+}
+
+// split, but skipping the root
+func (fp FullPath) Split() []string {
+	if fp == "" || fp == "/" {
+		return []string{}
+	}
+	return strings.Split(string(fp)[1:], "/")
+}
+
+func Join(names ...string) string {
+	return path.Join(names...)
+}
+
+func JoinPath(names ...string) FullPath {
+	return FullPath(Join(names...))
+}
+
+func (fp FullPath) IsUnder(other FullPath) bool {
+	if other == "/" {
+		return true
+	}
+	return strings.HasPrefix(string(fp), string(other)+"/")
+}
+
+// IsEqualOrUnder reports whether candidate is equal to or a descendant of
+// other using proper directory boundaries (not a plain string prefix check).
+// Empty strings always return false.
+func IsEqualOrUnder(candidate, other string) bool {
+	candidatePath := NormalizePath(candidate)
+	otherPath := NormalizePath(other)
+	if candidatePath == "" || otherPath == "" {
+		return false
+	}
+	return candidatePath == otherPath || candidatePath.IsUnder(otherPath)
+}
+
+// NormalizePath trims a trailing slash and returns a FullPath.
+// Empty input returns "" (callers should treat this as "no path").
+func NormalizePath(p string) FullPath {
+	if p == "" {
+		return ""
+	}
+	trimmed := strings.TrimSuffix(p, "/")
+	if trimmed == "" {
+		return "/"
+	}
+	return FullPath(trimmed)
+}
+
+func StringSplit(separatedValues string, sep string) []string {
+	if separatedValues == "" {
+		return nil
+	}
+	return strings.Split(separatedValues, sep)
+}
+
+// CleanWindowsPath normalizes Windows-style backslashes to forward slashes.
+// This handles paths from Windows clients where paths use backslashes.
+func CleanWindowsPath(p string) string {
+	return strings.ReplaceAll(p, "\\", "/")
+}
+
+// CleanWindowsPathBase normalizes Windows-style backslashes to forward slashes
+// and returns the base name of the path.
+func CleanWindowsPathBase(p string) string {
+	return path.Base(strings.ReplaceAll(p, "\\", "/"))
+}

weed/util/fullpath_test.go

@@ -0,0 +1,175 @@
+package util
+
+import (
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+// TestSanitizeUTF8Name_ValidPassThrough asserts the fast path returns the
+// input unchanged (no allocation, no byte alteration).
+func TestSanitizeUTF8Name_ValidPassThrough(t *testing.T) {
+	for _, s := range []string{
+		"",
+		"plain.txt",
+		"日本語.txt",
+		"🦑 squid",
+	} {
+		if got := SanitizeUTF8Name(s); got != s {
+			t.Fatalf("SanitizeUTF8Name(%q) = %q, want unchanged", s, got)
+		}
+	}
+}
+
+// TestSanitizeUTF8Name_InvalidBytes asserts invalid bytes are replaced with a
+// single '_' (URL-safe, single-byte) and the output is valid UTF-8. The
+// replacement char is load-bearing — downstream code places these strings in
+// HTTP URLs, where '?' would be parsed as the query delimiter.
+func TestSanitizeUTF8Name_InvalidBytes(t *testing.T) {
+	out := SanitizeUTF8Name("foo\x80bar")
+	if !utf8.ValidString(out) {
+		t.Fatalf("result is not valid UTF-8: %q", out)
+	}
+	if out != "foo_bar" {
+		t.Fatalf("SanitizeUTF8Name = %q, want %q", out, "foo_bar")
+	}
+	if strings.ContainsRune(out, '?') {
+		t.Fatalf("replacement must be URL-safe, got %q", out)
+	}
+}
+
+// TestFullPathSanitized_WholePath ensures Sanitized() scrubs invalid bytes in
+// every component, not just the last — that's the difference from Name() and
+// the reason call sites that need to pass a full path to a proto field must
+// use Sanitized(), not (dir, _) := DirAndName().
+func TestFullPathSanitized_WholePath(t *testing.T) {
+	// Invalid byte sits in the middle component.
+	fp := FullPath("/home/bad\x80dir/file.txt")
+	got := fp.Sanitized()
+	want := "/home/bad_dir/file.txt"
+	if got != want {
+		t.Fatalf("Sanitized() = %q, want %q", got, want)
+	}
+
+	// Bytes in every component — all get replaced, structure preserved.
+	fp = FullPath("/a\xffb/c\xffd/e\xfff")
+	got = fp.Sanitized()
+	want = "/a_b/c_d/e_f"
+	if got != want {
+		t.Fatalf("Sanitized() = %q, want %q", got, want)
+	}
+	if !utf8.ValidString(got) {
+		t.Fatalf("Sanitized() returned non-UTF-8: %q", got)
+	}
+}
+
+// TestFullPathDirAndName_OnlyNameSanitized documents a (deliberate) sharp
+// edge: DirAndName() sanitizes only the trailing name, not dir. Callers who
+// need a sanitized full path must use Sanitized(); using dir from DirAndName
+// will still carry invalid bytes in parent components. This test pins the
+// existing behavior so it is not accidentally "fixed" in a way that changes
+// the (dir, name) semantics that everything else depends on.
+func TestFullPathDirAndName_OnlyNameSanitized(t *testing.T) {
+	fp := FullPath("/home/bad\x80dir/child\xffname")
+	dir, name := fp.DirAndName()
+	if !utf8.ValidString(name) {
+		t.Fatalf("name must be sanitized: %q", name)
+	}
+	// dir still contains the invalid byte — this is by design, because dir is
+	// used positionally (e.g. as a parent key) and changing its bytes would
+	// change identity. Sanitized() is the method to use for proto fields.
+	if utf8.ValidString(dir) {
+		t.Fatalf("regression: dir should remain raw (%q); callers needing a clean path must use Sanitized()", dir)
+	}
+}
+
+// TestFullPathBackslashNotSeparator ensures a literal backslash in a filer
+// path component is treated as a regular character, not as a path separator.
+// Filer paths always use "/" as the separator; using filepath.Split (which
+// treats "\" as a separator on Windows) corrupts paths that contain literal
+// backslashes (#11243). path.Split is OS-independent and only splits on "/".
+func TestFullPathBackslashNotSeparator(t *testing.T) {
+	tests := []struct {
+		fullPath string
+		wantDir  string
+		wantName string
+	}{
+		// Backslash in the filename — must stay in the name, not split the path.
+		{"/test/special\\reverseslash4.jpg", "/test", "special\\reverseslash4.jpg"},
+		// Backslash in a directory component — the dir keeps it, name is after last "/".
+		{"/a\\b/c.jpg", "/a\\b", "c.jpg"},
+		// Multiple backslashes in the filename.
+		{"/dir/a\\b\\c.txt", "/dir", "a\\b\\c.txt"},
+		// Backslash at the start of the filename.
+		{"/dir/\\file.txt", "/dir", "\\file.txt"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.fullPath, func(t *testing.T) {
+			fp := FullPath(tc.fullPath)
+			dir, name := fp.DirAndName()
+			if dir != tc.wantDir {
+				t.Errorf("DirAndName dir: got %q want %q", dir, tc.wantDir)
+			}
+			if name != tc.wantName {
+				t.Errorf("DirAndName name: got %q want %q", name, tc.wantName)
+			}
+			if got := fp.Name(); got != tc.wantName {
+				t.Errorf("Name: got %q want %q", got, tc.wantName)
+			}
+		})
+	}
+}
+
+// TestFullPathSpecialCharactersInName ensures that special characters that are
+// valid in filenames (#, ?, %) are preserved by DirAndName and Name. These
+// characters must be percent-encoded by the client when constructing the HTTP
+// request URL, but once decoded by the server they are regular path bytes.
+// The "/" in the path is always a separator; special chars in a directory
+// component stay in the dir, and the last component is the name.
+func TestFullPathSpecialCharactersInName(t *testing.T) {
+	tests := []struct {
+		fullPath string
+		wantDir  string
+		wantName string
+	}{
+		// "#" in a directory component — dir keeps it, name is after last "/".
+		{"/test/special#/endhashfolder.jpg", "/test/special#", "endhashfolder.jpg"},
+		// "?" in a directory component.
+		{"/test/special?/endqfolder.jpg", "/test/special?", "endqfolder.jpg"},
+		// "%" in a directory component.
+		{"/test/special%/endpctfolder.jpg", "/test/special%", "endpctfolder.jpg"},
+		// "&" in a directory component.
+		{"/test/special&/endampfolder.jpg", "/test/special&", "endampfolder.jpg"},
+		// "=" in a directory component.
+		{"/test/special=/endeqfolder.jpg", "/test/special=", "endeqfolder.jpg"},
+		// Special chars in the filename itself (no "/" after them).
+		{"/test/hash#file.jpg", "/test", "hash#file.jpg"},
+		{"/test/q?file.jpg", "/test", "q?file.jpg"},
+		{"/test/pct%file.jpg", "/test", "pct%file.jpg"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.fullPath, func(t *testing.T) {
+			fp := FullPath(tc.fullPath)
+			dir, name := fp.DirAndName()
+			if dir != tc.wantDir {
+				t.Errorf("DirAndName dir: got %q want %q", dir, tc.wantDir)
+			}
+			if name != tc.wantName {
+				t.Errorf("DirAndName name: got %q want %q", name, tc.wantName)
+			}
+			if got := fp.Name(); got != tc.wantName {
+				t.Errorf("Name: got %q want %q", got, tc.wantName)
+			}
+		})
+	}
+}
+
+// TestJoinPreservesBackslash ensures Join does not convert backslashes to
+// forward slashes — they are regular filename characters in filer paths.
+func TestJoinPreservesBackslash(t *testing.T) {
+	got := Join("/parent", "child\\name.txt")
+	want := "/parent/child\\name.txt"
+	if got != want {
+		t.Fatalf("Join: got %q want %q", got, want)
+	}
+}

weed/util/grace/pprof.go

@@ -0,0 +1,71 @@
+package grace
+
+import (
+	"fmt"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+	"runtime"
+	"runtime/pprof"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+// StartDebugServer starts an HTTP server for pprof debugging on the specified port.
+// The server runs in a goroutine and serves pprof endpoints at /debug/pprof/*.
+func StartDebugServer(debugPort int) {
+	go func() {
+		addr := fmt.Sprintf("127.0.0.1:%d", debugPort)
+		glog.V(0).Infof("Starting debug server for pprof at http://%s/debug/pprof/", addr)
+		if err := http.ListenAndServe(addr, nil); err != nil && err != http.ErrServerClosed {
+			glog.Errorf("Failed to start debug server on %s: %v", addr, err)
+		}
+	}()
+}
+
+func SetupProfiling(cpuProfile, memProfile string) {
+	if cpuProfile != "" {
+		f, err := os.Create(cpuProfile)
+		if err != nil {
+			glog.Fatal(err)
+		}
+		runtime.SetBlockProfileRate(1)
+		runtime.SetMutexProfileFraction(1)
+		pprof.StartCPUProfile(f)
+		OnInterrupt(func() {
+			pprof.StopCPUProfile()
+			f.Close()
+
+			// write block pprof
+			blockF, err := os.Create(cpuProfile + ".block")
+			if err != nil {
+				return
+			}
+			p := pprof.Lookup("block")
+			p.WriteTo(blockF, 0)
+			blockF.Close()
+
+			// write mutex pprof
+			mutexF, err := os.Create(cpuProfile + ".mutex")
+			if err != nil {
+				return
+			}
+			p = pprof.Lookup("mutex")
+			p.WriteTo(mutexF, 0)
+			mutexF.Close()
+
+		})
+	}
+	if memProfile != "" {
+		runtime.MemProfileRate = 1
+		f, err := os.Create(memProfile)
+		if err != nil {
+			glog.Fatal(err)
+		}
+		OnInterrupt(func() {
+			pprof.WriteHeapProfile(f)
+			f.Close()
+		})
+	}
+
+}

weed/util/grace/signal_handling.go

@@ -0,0 +1,76 @@
+//go:build !plan9
+
+package grace
+
+import (
+	"os"
+	"os/signal"
+	"reflect"
+	"runtime"
+	"sync"
+	"syscall"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+var signalChan chan os.Signal
+var interruptHooks = make([]func(), 0)
+var interruptHookLock sync.RWMutex
+var reloadHooks = make([]func(), 0)
+var reloadHookLock sync.RWMutex
+
+func GetFunctionName(i interface{}) string {
+	return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
+}
+
+func init() {
+	signalChan = make(chan os.Signal, 1)
+	signal.Notify(signalChan,
+		os.Interrupt,
+		os.Kill,
+		syscall.SIGALRM,
+		syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGTERM,
+		// syscall.SIGQUIT,
+	)
+	go func() {
+		for s := range signalChan {
+			if s.String() == syscall.SIGHUP.String() {
+				reloadHookLock.RLock()
+				for _, hook := range reloadHooks {
+					hook()
+				}
+				reloadHookLock.RUnlock()
+			} else {
+				interruptHookLock.RLock()
+				// Execute hooks in reverse registration order (LIFO/defer-style) so
+				// later-started services shut down before the services they depend on.
+				for i := len(interruptHooks) - 1; i >= 0; i-- {
+					hook := interruptHooks[i]
+					glog.V(4).Infof("exec interrupt hook func name:%s", GetFunctionName(hook))
+					hook()
+				}
+				interruptHookLock.RUnlock()
+				os.Exit(0)
+			}
+		}
+	}()
+}
+
+func OnReload(fn func()) {
+	// prevent reentry
+	reloadHookLock.Lock()
+	defer reloadHookLock.Unlock()
+	reloadHooks = append(reloadHooks, fn)
+}
+
+func OnInterrupt(fn func()) {
+	// prevent reentry
+	interruptHookLock.Lock()
+	defer interruptHookLock.Unlock()
+
+	// deal with control+c,etc
+	// controlling terminal close, daemon not exit
+	interruptHooks = append(interruptHooks, fn)
+}

weed/util/grace/signal_handling_notsupported.go

@@ -0,0 +1,6 @@
+//go:build plan9
+
+package grace
+
+func OnInterrupt(fn func()) {
+}

weed/util/http/chunk_proxy.go

@@ -0,0 +1,62 @@
+package http
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// ProxyChunkIdParam is the query parameter a filer reads to serve one chunk
+// from the volume server holding it, for callers that cannot reach volume
+// servers directly.
+const ProxyChunkIdParam = "proxyChunkId"
+
+var (
+	filerSigningKey            security.SigningKey
+	filerSigningKeyExpires     int
+	filerReadSigningKey        security.SigningKey
+	filerReadSigningKeyExpires int
+	loadFilerJwtConfigOnce     sync.Once
+)
+
+func loadFilerJwtConfig() {
+	v := util.GetViper()
+	filerSigningKey = security.SigningKey(v.GetString("jwt.filer_signing.key"))
+	filerSigningKeyExpires = v.GetInt("jwt.filer_signing.expires_after_seconds")
+	if filerSigningKeyExpires == 0 {
+		filerSigningKeyExpires = 10
+	}
+	filerReadSigningKey = security.SigningKey(v.GetString("jwt.filer_signing.read.key"))
+	filerReadSigningKeyExpires = v.GetInt("jwt.filer_signing.read.expires_after_seconds")
+	if filerReadSigningKeyExpires == 0 {
+		filerReadSigningKeyExpires = 60
+	}
+}
+
+// JwtForFilerServer generates a JWT for the filer's HTTP API if jwt.filer_signing
+// is configured for that access level, the way filer.JwtForVolumeServer does for
+// volume servers. Empty when the key is unset, which is also when the filer
+// serves the request unsigned.
+func JwtForFilerServer(isWrite bool) string {
+	loadFilerJwtConfigOnce.Do(loadFilerJwtConfig)
+	if isWrite {
+		return string(security.GenJwtForFilerServer(filerSigningKey, filerSigningKeyExpires))
+	}
+	return string(security.GenJwtForFilerServer(filerReadSigningKey, filerReadSigningKeyExpires))
+}
+
+// ProxyChunkUrl builds the request that reads or writes one chunk through a
+// filer instead of the volume server holding it.
+func ProxyChunkUrl(filerAddress string, fileId string) string {
+	return fmt.Sprintf("http://%s/?%s=%s", filerAddress, ProxyChunkIdParam, fileId)
+}
+
+// IsProxyChunkUrl reports whether a chunk URL addresses a filer's chunk proxy
+// rather than a volume server. Such a request is authorized by the filer, which
+// attaches the volume credential itself, so the token it carries is a filer one.
+func IsProxyChunkUrl(urlString string) bool {
+	return strings.Contains(urlString, ProxyChunkIdParam+"=")
+}

weed/util/http/chunk_proxy_test.go

@@ -0,0 +1,74 @@
+package http
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestIsProxyChunkUrl(t *testing.T) {
+	for _, tc := range []struct {
+		urlString string
+		want      bool
+	}{
+		{ProxyChunkUrl("filer:8888", "3,01637037d6"), true},
+		{"http://filer:8888/?proxyChunkId=3,01637037d6&readDeleted=true", true},
+		{"http://volume:8080/3,01637037d6", false},
+		{"http://volume:8080/3,01637037d6?readDeleted=true", false},
+		{"", false},
+	} {
+		if got := IsProxyChunkUrl(tc.urlString); got != tc.want {
+			t.Errorf("IsProxyChunkUrl(%q) = %v, want %v", tc.urlString, got, tc.want)
+		}
+	}
+}
+
+// The credential a proxied chunk carries is a filer one, signed with the key
+// for the access level the caller is about to use.
+func TestJwtForFilerServer(t *testing.T) {
+	const (
+		writeKey = "filer-write-key"
+		readKey  = "filer-read-key"
+	)
+	v := util.GetViper()
+	v.Set("jwt.filer_signing.key", writeKey)
+	v.Set("jwt.filer_signing.read.key", readKey)
+
+	for _, tc := range []struct {
+		name     string
+		isWrite  bool
+		signedBy string
+		otherKey string
+	}{
+		{"read", false, readKey, writeKey},
+		{"write", true, writeKey, readKey},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			token := security.EncodedJwt(JwtForFilerServer(tc.isWrite))
+			claims := &security.SeaweedFilerClaims{}
+			if _, err := security.DecodeJwt(security.SigningKey(tc.signedBy), token, claims); err != nil {
+				t.Fatalf("token does not validate against the %s key: %v", tc.name, err)
+			}
+			if claims.ExpiresAt == nil {
+				t.Fatal("token never expires")
+			}
+			if _, err := security.DecodeJwt(security.SigningKey(tc.otherKey), token, &security.SeaweedFilerClaims{}); err == nil {
+				t.Fatal("token also validates against the other access level's key")
+			}
+		})
+	}
+}
+
+func TestJwtForFilerServerWithoutKeys(t *testing.T) {
+	loadFilerJwtConfigOnce.Do(func() {})
+	write, read := filerSigningKey, filerReadSigningKey
+	t.Cleanup(func() { filerSigningKey, filerReadSigningKey = write, read })
+	filerSigningKey, filerReadSigningKey = nil, nil
+
+	for _, isWrite := range []bool{false, true} {
+		if token := JwtForFilerServer(isWrite); token != "" {
+			t.Errorf("unconfigured signing key still produced %q", token)
+		}
+	}
+}

weed/util/http/client/http_client.go

@@ -0,0 +1,340 @@
+package client
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc/credentials/tls/certprovider"
+
+	"github.com/seaweedfs/seaweedfs/weed/security/certreload"
+	util "github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/spf13/viper"
+)
+
+var (
+	loadSecurityConfigOnce sync.Once
+)
+
+// Intra-cluster peers (volume/filer/master) answer headers within milliseconds.
+// A peer that is TCP-reachable but not answering -- a volume server still loading
+// after a restart, or a stale keep-alive to a container that returned on a new IP
+// -- otherwise blocks a chunk read or a replicated write forever. The response
+// timeout bounds that wait so the read fails over to another replica and the
+// write fails fast to retry; the idle timeout evicts sockets to a departed server.
+const (
+	responseHeaderTimeout = 30 * time.Second
+	idleConnTimeout       = 90 * time.Second
+)
+
+type HTTPClient struct {
+	Client            *http.Client
+	Transport         *http.Transport
+	expectHttpsScheme bool
+	// certProvider, when non-nil, owns a background refresh goroutine for
+	// the client mTLS cert/key pair. Close() must be called to stop it.
+	certProvider certprovider.Provider
+}
+
+// Close stops any background cert refresh goroutine. Safe to call on a
+// client that was constructed without mTLS. Existing pooled connections
+// are also closed via CloseIdleConnections.
+func (httpClient *HTTPClient) Close() {
+	if httpClient == nil {
+		return
+	}
+	if httpClient.certProvider != nil {
+		httpClient.certProvider.Close()
+		httpClient.certProvider = nil
+	}
+	if httpClient.Client != nil {
+		httpClient.Client.CloseIdleConnections()
+	}
+}
+
+func (httpClient *HTTPClient) Do(req *http.Request) (*http.Response, error) {
+	req.URL.Scheme = httpClient.GetHttpScheme()
+	return httpClient.Client.Do(req)
+}
+
+func (httpClient *HTTPClient) Get(url string) (resp *http.Response, err error) {
+	url, err = httpClient.NormalizeHttpScheme(url)
+	if err != nil {
+		return nil, err
+	}
+	return httpClient.Client.Get(url)
+}
+
+func (httpClient *HTTPClient) Post(url, contentType string, body io.Reader) (resp *http.Response, err error) {
+	url, err = httpClient.NormalizeHttpScheme(url)
+	if err != nil {
+		return nil, err
+	}
+	return httpClient.Client.Post(url, contentType, body)
+}
+
+func (httpClient *HTTPClient) PostForm(url string, data url.Values) (resp *http.Response, err error) {
+	url, err = httpClient.NormalizeHttpScheme(url)
+	if err != nil {
+		return nil, err
+	}
+	return httpClient.Client.PostForm(url, data)
+}
+
+func (httpClient *HTTPClient) Head(url string) (resp *http.Response, err error) {
+	url, err = httpClient.NormalizeHttpScheme(url)
+	if err != nil {
+		return nil, err
+	}
+	return httpClient.Client.Head(url)
+}
+func (httpClient *HTTPClient) CloseIdleConnections() {
+	httpClient.Client.CloseIdleConnections()
+}
+
+func (httpClient *HTTPClient) GetClientTransport() *http.Transport {
+	return httpClient.Transport
+}
+
+func (httpClient *HTTPClient) GetHttpScheme() string {
+	if httpClient.expectHttpsScheme {
+		return "https"
+	}
+	return "http"
+}
+
+func (httpClient *HTTPClient) IsTLSVerified() bool {
+	return httpClient.expectHttpsScheme && httpClient.Transport != nil && (httpClient.Transport.TLSClientConfig == nil || !httpClient.Transport.TLSClientConfig.InsecureSkipVerify)
+}
+
+func rejectHttpsDowngrade(req *http.Request, via []*http.Request) error {
+	if len(via) >= 10 {
+		return http.ErrUseLastResponse
+	}
+	if len(via) > 0 && via[0].URL.Scheme == "https" && req.URL.Scheme != "https" {
+		return http.ErrUseLastResponse
+	}
+	return nil
+}
+
+func (httpClient *HTTPClient) NormalizeHttpScheme(rawURL string) (string, error) {
+	expectedScheme := httpClient.GetHttpScheme()
+
+	if !(strings.HasPrefix(rawURL, "http://") || strings.HasPrefix(rawURL, "https://")) {
+		return expectedScheme + "://" + rawURL, nil
+	}
+
+	parsedURL, err := url.Parse(rawURL)
+	if err != nil {
+		return "", err
+	}
+
+	if expectedScheme != parsedURL.Scheme {
+		parsedURL.Scheme = expectedScheme
+	}
+	return parsedURL.String(), nil
+}
+
+func NewHttpClient(clientName ClientName, opts ...HttpClientOpt) (*HTTPClient, error) {
+	httpClient := HTTPClient{}
+	httpClient.expectHttpsScheme = checkIsHttpsClientEnabled(clientName)
+	var tlsConfig *tls.Config = nil
+
+	if httpClient.expectHttpsScheme {
+		certFileName, keyFileName, hasClientCert, err := clientCertPaths(clientName)
+		if err != nil {
+			return nil, err
+		}
+
+		clientCaCert, clientCaCertName, err := getClientCaCert(clientName)
+		if err != nil {
+			return nil, err
+		}
+
+		if hasClientCert || len(clientCaCert) != 0 {
+			caCertPool, err := createHTTPClientCertPool(clientCaCert, clientCaCertName)
+			if err != nil {
+				return nil, err
+			}
+
+			tlsConfig = &tls.Config{
+				RootCAs:            caCertPool,
+				InsecureSkipVerify: false,
+			}
+
+			if hasClientCert {
+				getClientCert, provider, err := certreload.NewClientGetCertificate(certFileName, keyFileName)
+				if err != nil {
+					return nil, fmt.Errorf("error loading client certificate and key: %s", err)
+				}
+				tlsConfig.GetClientCertificate = getClientCert
+				httpClient.certProvider = provider
+			}
+		}
+
+		if getBoolOptionFromSecurityConfiguration(clientName, "insecure_skip_verify") {
+			if tlsConfig == nil {
+				tlsConfig = &tls.Config{}
+			}
+			tlsConfig.InsecureSkipVerify = true
+		}
+	}
+
+	httpClient.Transport = &http.Transport{
+		MaxIdleConns:        1024,
+		MaxIdleConnsPerHost: 1024,
+		TLSClientConfig:     tlsConfig,
+		// Bind outbound HTTP to the -ip.bind source address.
+		DialContext:           util.OutboundDialContext,
+		ResponseHeaderTimeout: responseHeaderTimeout,
+		IdleConnTimeout:       idleConnTimeout,
+	}
+	httpClient.Client = &http.Client{
+		Transport:     httpClient.Transport,
+		CheckRedirect: rejectHttpsDowngrade,
+	}
+
+	for _, opt := range opts {
+		opt(&httpClient)
+	}
+	return &httpClient, nil
+}
+
+func getStringOptionFromSecurityConfiguration(clientName ClientName, stringOptionName string) string {
+	util.LoadSecurityConfiguration()
+	return viper.GetString(fmt.Sprintf("https.%s.%s", clientName.LowerCaseString(), stringOptionName))
+}
+
+func getBoolOptionFromSecurityConfiguration(clientName ClientName, boolOptionName string) bool {
+	util.LoadSecurityConfiguration()
+	return viper.GetBool(fmt.Sprintf("https.%s.%s", clientName.LowerCaseString(), boolOptionName))
+}
+
+func checkIsHttpsClientEnabled(clientName ClientName) bool {
+	return getBoolOptionFromSecurityConfiguration(clientName, "enabled")
+}
+
+func getFileContentFromSecurityConfiguration(clientName ClientName, fileType string) ([]byte, string, error) {
+	if fileName := getStringOptionFromSecurityConfiguration(clientName, fileType); fileName != "" {
+		fileContent, err := os.ReadFile(fileName)
+		if err != nil {
+			return nil, fileName, err
+		}
+		return fileContent, fileName, err
+	}
+	return nil, "", nil
+}
+
+// clientCertPaths reads the https.<clientName>.{cert,key} paths from the
+// security config, validates they're either both set or both empty, and
+// returns them along with a hasClientCert flag. Loading is deferred to
+// certreload so the cert/key pair is picked up from disk on rotation.
+func clientCertPaths(clientName ClientName) (certFile, keyFile string, hasClientCert bool, err error) {
+	certFile = getStringOptionFromSecurityConfiguration(clientName, "cert")
+	keyFile = getStringOptionFromSecurityConfiguration(clientName, "key")
+	if certFile == "" && keyFile == "" {
+		return "", "", false, nil
+	}
+	if certFile == "" || keyFile == "" {
+		return "", "", false, fmt.Errorf("https.%s: both cert and key must be set (got cert=%q key=%q)", clientName.LowerCaseString(), certFile, keyFile)
+	}
+	return certFile, keyFile, true, nil
+}
+
+func getClientCaCert(clientName ClientName) ([]byte, string, error) {
+	return getFileContentFromSecurityConfiguration(clientName, "ca")
+}
+
+// NewHttpClientWithTLS creates an HTTPClient with explicit TLS certificate
+// parameters instead of reading from the global security configuration.
+// This is used by filer.sync to create per-cluster HTTP clients when clusters
+// use different certificates.
+func NewHttpClientWithTLS(certFile, keyFile, caFile string, insecureSkipVerify bool, opts ...HttpClientOpt) (*HTTPClient, error) {
+	httpClient := HTTPClient{}
+	httpClient.expectHttpsScheme = true
+	var tlsConfig *tls.Config
+
+	if (certFile == "") != (keyFile == "") {
+		return nil, fmt.Errorf("both cert and key are required for mTLS, got cert=%q key=%q", certFile, keyFile)
+	}
+
+	var getClientCert func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
+	if certFile != "" && keyFile != "" {
+		cb, provider, err := certreload.NewClientGetCertificate(certFile, keyFile)
+		if err != nil {
+			return nil, fmt.Errorf("error loading client certificate and key: %s", err)
+		}
+		getClientCert = cb
+		httpClient.certProvider = provider
+	}
+	// closeProviderOnError ensures the cert reloader's background refresh
+	// goroutine is shut down if any subsequent step fails before we hand
+	// the client back to the caller.
+	closeProviderOnError := func() {
+		if httpClient.certProvider != nil {
+			httpClient.certProvider.Close()
+			httpClient.certProvider = nil
+		}
+	}
+
+	var caCertPool *x509.CertPool
+	if caFile != "" {
+		caCert, err := os.ReadFile(caFile)
+		if err != nil {
+			closeProviderOnError()
+			return nil, fmt.Errorf("error reading CA cert %s: %s", caFile, err)
+		}
+		caCertPool, err = createHTTPClientCertPool(caCert, caFile)
+		if err != nil {
+			closeProviderOnError()
+			return nil, err
+		}
+	}
+
+	if getClientCert != nil || caCertPool != nil || insecureSkipVerify {
+		tlsConfig = &tls.Config{
+			GetClientCertificate: getClientCert,
+			RootCAs:              caCertPool,
+			InsecureSkipVerify:   insecureSkipVerify,
+		}
+	}
+
+	httpClient.Transport = &http.Transport{
+		MaxIdleConns:        1024,
+		MaxIdleConnsPerHost: 1024,
+		TLSClientConfig:     tlsConfig,
+		// Bind outbound HTTP to the -ip.bind source address.
+		DialContext:           util.OutboundDialContext,
+		ResponseHeaderTimeout: responseHeaderTimeout,
+		IdleConnTimeout:       idleConnTimeout,
+	}
+	httpClient.Client = &http.Client{
+		Transport:     httpClient.Transport,
+		CheckRedirect: rejectHttpsDowngrade,
+	}
+
+	for _, opt := range opts {
+		opt(&httpClient)
+	}
+	return &httpClient, nil
+}
+
+func createHTTPClientCertPool(certContent []byte, fileName string) (*x509.CertPool, error) {
+	certPool := x509.NewCertPool()
+	if len(certContent) == 0 {
+		return certPool, nil
+	}
+
+	ok := certPool.AppendCertsFromPEM(certContent)
+	if !ok {
+		return nil, fmt.Errorf("error processing certificate in %s", fileName)
+	}
+	return certPool, nil
+}

weed/util/http/client/http_client_interface.go

@@ -0,0 +1,16 @@
+package client
+
+import (
+	"io"
+	"net/http"
+	"net/url"
+)
+
+type HTTPClientInterface interface {
+	Do(req *http.Request) (*http.Response, error)
+	Get(url string) (resp *http.Response, err error)
+	Post(url, contentType string, body io.Reader) (resp *http.Response, err error)
+	PostForm(url string, data url.Values) (resp *http.Response, err error)
+	Head(url string) (resp *http.Response, err error)
+	CloseIdleConnections()
+}

weed/util/http/client/http_client_name.go

@@ -0,0 +1,14 @@
+package client
+
+import "strings"
+
+type ClientName int
+
+//go:generate stringer -type=ClientName -output=http_client_name_string.go
+const (
+	Client ClientName = iota
+)
+
+func (name *ClientName) LowerCaseString() string {
+	return strings.ToLower(name.String())
+}

weed/util/http/client/http_client_name_string.go

@@ -0,0 +1,23 @@
+// Code generated by "stringer -type=ClientName -output=http_client_name_string.go"; DO NOT EDIT.
+
+package client
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[Client-0]
+}
+
+const _ClientName_name = "Client"
+
+var _ClientName_index = [...]uint8{0, 6}
+
+func (i ClientName) String() string {
+	if i < 0 || i >= ClientName(len(_ClientName_index)-1) {
+		return "ClientName(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _ClientName_name[_ClientName_index[i]:_ClientName_index[i+1]]
+}

weed/util/http/client/http_client_opt.go

@@ -0,0 +1,29 @@
+package client
+
+import (
+	"net"
+	"time"
+)
+
+type HttpClientOpt = func(clientCfg *HTTPClient)
+
+func AddDialContext(httpClient *HTTPClient) {
+	dialContext := (&net.Dialer{
+		Timeout:   10 * time.Second,
+		KeepAlive: 10 * time.Second,
+	}).DialContext
+
+	httpClient.Transport.DialContext = dialContext
+	httpClient.Client.Transport = httpClient.Transport
+}
+
+// WithResponseHeaderTimeout overrides the transport's default response-header
+// timeout. Mainly for tests that need a short deadline against an unresponsive
+// peer without mutating the package default.
+func WithResponseHeaderTimeout(timeout time.Duration) HttpClientOpt {
+	return func(httpClient *HTTPClient) {
+		if httpClient.Transport != nil {
+			httpClient.Transport.ResponseHeaderTimeout = timeout
+		}
+	}
+}

weed/util/http/client/http_client_test.go

@@ -0,0 +1,127 @@
+package client
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestNewHttpClientSetsResponseTimeouts(t *testing.T) {
+	c, err := NewHttpClient(Client)
+	if err != nil {
+		t.Fatalf("NewHttpClient: %v", err)
+	}
+	if got := c.Transport.ResponseHeaderTimeout; got != responseHeaderTimeout {
+		t.Errorf("ResponseHeaderTimeout = %v, want %v", got, responseHeaderTimeout)
+	}
+	if got := c.Transport.IdleConnTimeout; got != idleConnTimeout {
+		t.Errorf("IdleConnTimeout = %v, want %v", got, idleConnTimeout)
+	}
+	// AddDialContext must keep the response timeouts intact.
+	c2, err := NewHttpClient(Client, AddDialContext)
+	if err != nil {
+		t.Fatalf("NewHttpClient(AddDialContext): %v", err)
+	}
+	if got := c2.Transport.ResponseHeaderTimeout; got != responseHeaderTimeout {
+		t.Errorf("AddDialContext dropped ResponseHeaderTimeout: got %v", got)
+	}
+	// WithResponseHeaderTimeout overrides the default.
+	c3, err := NewHttpClient(Client, WithResponseHeaderTimeout(time.Second))
+	if err != nil {
+		t.Fatalf("NewHttpClient(WithResponseHeaderTimeout): %v", err)
+	}
+	if got := c3.Transport.ResponseHeaderTimeout; got != time.Second {
+		t.Errorf("WithResponseHeaderTimeout not applied: got %v", got)
+	}
+	// The TLS constructor must carry the same defaults.
+	cTLS, err := NewHttpClientWithTLS("", "", "", true)
+	if err != nil {
+		t.Fatalf("NewHttpClientWithTLS: %v", err)
+	}
+	if got := cTLS.Transport.ResponseHeaderTimeout; got != responseHeaderTimeout {
+		t.Errorf("NewHttpClientWithTLS ResponseHeaderTimeout = %v, want %v", got, responseHeaderTimeout)
+	}
+	if got := cTLS.Transport.IdleConnTimeout; got != idleConnTimeout {
+		t.Errorf("NewHttpClientWithTLS IdleConnTimeout = %v, want %v", got, idleConnTimeout)
+	}
+}
+
+// A peer that is TCP-reachable but never answers -- a volume server still
+// loading its volumes after a restart, or a stale keep-alive to a container
+// that came back on a new IP -- must not block a read or a replicated write
+// forever. ResponseHeaderTimeout turns that into a prompt timeout error so the
+// caller can fail over to another replica or retry.
+func TestResponseHeaderTimeoutUnblocksUnresponsivePeer(t *testing.T) {
+	server, release := newUnresponsiveServer(t, false)
+	defer server.Close()
+	defer close(release)
+
+	c, err := NewHttpClient(Client, WithResponseHeaderTimeout(200*time.Millisecond))
+	if err != nil {
+		t.Fatalf("NewHttpClient: %v", err)
+	}
+	assertTimesOut(t, c, server.URL)
+}
+
+// Same guarantee over the TLS client path (NewHttpClientWithTLS).
+func TestResponseHeaderTimeoutUnblocksUnresponsivePeerTLS(t *testing.T) {
+	server, release := newUnresponsiveServer(t, true)
+	defer server.Close()
+	defer close(release)
+
+	c, err := NewHttpClientWithTLS("", "", "", true, WithResponseHeaderTimeout(200*time.Millisecond))
+	if err != nil {
+		t.Fatalf("NewHttpClientWithTLS: %v", err)
+	}
+	assertTimesOut(t, c, server.URL)
+}
+
+// newUnresponsiveServer returns a server whose handler blocks until the returned
+// channel is closed, so it accepts the connection but never sends response
+// headers. Callers must close the channel before server.Close() -- defers run
+// LIFO, so register the Close defer first.
+func newUnresponsiveServer(t *testing.T, tls bool) (*httptest.Server, chan struct{}) {
+	t.Helper()
+	release := make(chan struct{})
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		<-release
+	})
+	if tls {
+		return httptest.NewTLSServer(handler), release
+	}
+	return httptest.NewServer(handler), release
+}
+
+func assertTimesOut(t *testing.T, c *HTTPClient, url string) {
+	t.Helper()
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
+	if err != nil {
+		t.Fatalf("new request: %v", err)
+	}
+
+	done := make(chan error, 1)
+	go func() {
+		resp, doErr := c.Do(req)
+		if resp != nil {
+			resp.Body.Close()
+		}
+		done <- doErr
+	}()
+
+	select {
+	case doErr := <-done:
+		if doErr == nil {
+			t.Fatal("expected a timeout error from an unresponsive peer, got nil")
+		}
+		var netErr net.Error
+		if !errors.As(doErr, &netErr) || !netErr.Timeout() {
+			t.Fatalf("expected a timeout error, got %v", doErr)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Do() blocked well past the response header timeout")
+	}
+}

weed/util/http/http_global_client_init.go

@@ -0,0 +1,37 @@
+package http
+
+import (
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client"
+)
+
+var (
+	globalHttpClient     *util_http_client.HTTPClient
+	globalHttpClientOnce sync.Once
+)
+
+func NewGlobalHttpClient(opt ...util_http_client.HttpClientOpt) (*util_http_client.HTTPClient, error) {
+	return util_http_client.NewHttpClient(util_http_client.Client, opt...)
+}
+
+// GetGlobalHttpClient returns the process-wide HTTP client, initializing it on
+// first use. Lazy init keeps callers that bypass weed.go's main (in-process
+// test harnesses, libraries) from dereferencing a nil client.
+func GetGlobalHttpClient() *util_http_client.HTTPClient {
+	globalHttpClientOnce.Do(initGlobalHttpClient)
+	return globalHttpClient
+}
+
+func InitGlobalHttpClient() {
+	globalHttpClientOnce.Do(initGlobalHttpClient)
+}
+
+func initGlobalHttpClient() {
+	client, err := NewGlobalHttpClient()
+	if err != nil {
+		glog.Fatalf("error init global http client: %v", err)
+	}
+	globalHttpClient = client
+}

weed/util/http/http_global_client_util.go

@@ -0,0 +1,814 @@
+package http
+
+import (
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/mem"
+	"github.com/seaweedfs/seaweedfs/weed/util/request_id"
+
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client"
+)
+
+var ErrNotFound = fmt.Errorf("not found")
+var ErrTooManyRequests = fmt.Errorf("too many requests")
+
+type jwtSigningReadConfig struct {
+	key     security.SigningKey
+	expires int
+}
+
+var (
+	jwtSigningReadConfigPtr atomic.Pointer[jwtSigningReadConfig]
+	loadJwtConfigOnce       sync.Once
+)
+
+func AppendQueryParameter(rawURL, key, value string) string {
+	encoded := url.Values{key: []string{value}}.Encode()
+	fragment := ""
+	if fragmentIndex := strings.Index(rawURL, "#"); fragmentIndex >= 0 {
+		fragment = rawURL[fragmentIndex:]
+		rawURL = rawURL[:fragmentIndex]
+	}
+
+	var result string
+	switch {
+	case strings.Contains(rawURL, "?"):
+		if strings.HasSuffix(rawURL, "?") || strings.HasSuffix(rawURL, "&") {
+			result = rawURL + encoded
+		} else {
+			result = rawURL + "&" + encoded
+		}
+	default:
+		result = rawURL + "?" + encoded
+	}
+	return result + fragment
+}
+
+func loadJwtConfig() {
+	v := util.GetViper()
+	jwtSigningReadConfigPtr.Store(&jwtSigningReadConfig{
+		key:     security.SigningKey(v.GetString("jwt.signing.read.key")),
+		expires: v.GetInt("jwt.signing.read.expires_after_seconds"),
+	})
+}
+
+// ReloadJwtSigningReadConfig re-reads the volume read-signing key from the
+// already-reloaded security config, so operators can rotate it via SIGHUP
+// without restarting the process.
+func ReloadJwtSigningReadConfig() {
+	loadJwtConfig()
+}
+
+func Post(url string, values url.Values) ([]byte, error) {
+	r, err := GetGlobalHttpClient().PostForm(url, values)
+	if err != nil {
+		return nil, err
+	}
+	defer r.Body.Close()
+	b, err := io.ReadAll(r.Body)
+	if r.StatusCode >= 400 {
+		if err != nil {
+			return nil, fmt.Errorf("%s: %d - %s", url, r.StatusCode, string(b))
+		} else {
+			return nil, fmt.Errorf("%s: %s", url, r.Status)
+		}
+	}
+	if err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+// github.com/seaweedfs/seaweedfs/unmaintained/repeated_vacuum/repeated_vacuum.go
+// may need increasing http.Client.Timeout
+func Get(url string) ([]byte, bool, error) {
+	return GetAuthenticated(context.Background(), url, "")
+}
+
+func GetAuthenticated(ctx context.Context, url, jwt string) ([]byte, bool, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, true, err
+	}
+	maybeAddAuth(request, jwt)
+	request.Header.Add("Accept-Encoding", "gzip")
+
+	response, err := GetGlobalHttpClient().Do(request)
+	if err != nil {
+		if ctx.Err() == nil {
+			recordUnreachable(request.URL.Host)
+		}
+		return nil, true, err
+	}
+	recordReachable(request.URL.Host)
+	defer CloseResponse(response)
+
+	var reader io.ReadCloser
+	switch response.Header.Get("Content-Encoding") {
+	case "gzip":
+		reader, err = gzip.NewReader(response.Body)
+		if err != nil {
+			return nil, true, err
+		}
+		defer reader.Close()
+	default:
+		reader = response.Body
+	}
+
+	b, err := io.ReadAll(reader)
+	if response.StatusCode >= 400 {
+		retryable := response.StatusCode >= 500
+		return nil, retryable, fmt.Errorf("%s: %s", url, response.Status)
+	}
+	if err != nil {
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			return nil, false, ctxErr
+		}
+		return nil, false, err
+	}
+	return b, false, nil
+}
+
+func Head(url string) (http.Header, error) {
+	r, err := GetGlobalHttpClient().Head(url)
+	if err != nil {
+		return nil, err
+	}
+	defer CloseResponse(r)
+	if r.StatusCode >= 400 {
+		return nil, fmt.Errorf("%s: %s", url, r.Status)
+	}
+	return r.Header, nil
+}
+
+func maybeAddAuth(req *http.Request, jwt string) {
+	if jwt != "" {
+		req.Header.Set("Authorization", security.BearerPrefix+string(jwt))
+	}
+}
+
+func Delete(url string, jwt string) error {
+	req, err := http.NewRequest(http.MethodDelete, url, nil)
+	if err != nil {
+		return err
+	}
+	maybeAddAuth(req, jwt)
+	resp, e := GetGlobalHttpClient().Do(req)
+	if e != nil {
+		return e
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	switch resp.StatusCode {
+	case http.StatusNotFound, http.StatusNoContent, http.StatusAccepted, http.StatusOK:
+		return nil
+	}
+	m := make(map[string]interface{})
+	if e := json.Unmarshal(body, &m); e == nil {
+		if s, ok := m["error"].(string); ok {
+			return errors.New(s)
+		}
+	}
+	return errors.New(string(body))
+}
+
+func DeleteProxied(url string, jwt string) (body []byte, httpStatus int, err error) {
+	req, err := http.NewRequest(http.MethodDelete, url, nil)
+	if err != nil {
+		return
+	}
+	maybeAddAuth(req, jwt)
+	resp, err := GetGlobalHttpClient().Do(req)
+	if err != nil {
+		return
+	}
+	defer resp.Body.Close()
+	body, err = io.ReadAll(resp.Body)
+	if err != nil {
+		return
+	}
+	httpStatus = resp.StatusCode
+	return
+}
+
+func GetBufferStream(url string, values url.Values, allocatedBytes []byte, eachBuffer func([]byte)) error {
+	r, err := GetGlobalHttpClient().PostForm(url, values)
+	if err != nil {
+		return err
+	}
+	defer CloseResponse(r)
+	if r.StatusCode != 200 {
+		return fmt.Errorf("%s: %s", url, r.Status)
+	}
+	for {
+		n, err := r.Body.Read(allocatedBytes)
+		if n > 0 {
+			eachBuffer(allocatedBytes[:n])
+		}
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+	}
+}
+
+func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) error {
+	r, err := GetGlobalHttpClient().PostForm(url, values)
+	if err != nil {
+		return err
+	}
+	defer CloseResponse(r)
+	if r.StatusCode != 200 {
+		return fmt.Errorf("%s: %s", url, r.Status)
+	}
+	return readFn(r.Body)
+}
+
+func DownloadFile(fileUrl string, jwt string, offset ...int64) (filename string, header http.Header, resp *http.Response, e error) {
+	return DownloadFileWithClient(GetGlobalHttpClient(), fileUrl, jwt, offset...)
+}
+
+// DownloadFileWithClient is like DownloadFile but uses the provided HTTP client
+// instead of the global one. This is used by filer.sync to download from
+// remote clusters that use different TLS certificates.
+func DownloadFileWithClient(client *util_http_client.HTTPClient, fileUrl string, jwt string, offset ...int64) (filename string, header http.Header, resp *http.Response, e error) {
+	if client == nil {
+		return "", nil, nil, fmt.Errorf("nil HTTP client in DownloadFileWithClient")
+	}
+	req, err := http.NewRequest(http.MethodGet, fileUrl, nil)
+	if err != nil {
+		return "", nil, nil, err
+	}
+
+	maybeAddAuth(req, jwt)
+
+	var rangeOffset int64
+	if len(offset) > 0 {
+		rangeOffset = offset[0]
+	}
+	if rangeOffset > 0 {
+		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", rangeOffset))
+	}
+
+	response, err := client.Do(req)
+	if err != nil {
+		return "", nil, nil, err
+	}
+
+	if rangeOffset > 0 {
+		expected := fmt.Sprintf("bytes %d-", rangeOffset)
+		if response.StatusCode != http.StatusPartialContent ||
+			!strings.HasPrefix(response.Header.Get("Content-Range"), expected) {
+			CloseResponse(response)
+			return "", nil, nil, fmt.Errorf("range request %q to %s returned %s with Content-Range %q",
+				req.Header.Get("Range"), fileUrl, response.Status, response.Header.Get("Content-Range"))
+		}
+	}
+
+	header = response.Header
+	contentDisposition := response.Header["Content-Disposition"]
+	if len(contentDisposition) > 0 {
+		idx := strings.Index(contentDisposition[0], "filename=")
+		if idx != -1 {
+			filename = contentDisposition[0][idx+len("filename="):]
+			filename = strings.Trim(filename, "\"")
+		}
+	}
+	resp = response
+	return
+}
+
+func Do(req *http.Request) (resp *http.Response, err error) {
+	return GetGlobalHttpClient().Do(req)
+}
+
+func NormalizeUrl(url string) (string, error) {
+	return GetGlobalHttpClient().NormalizeHttpScheme(url)
+}
+
+func ReadUrl(ctx context.Context, fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, buf []byte) (int64, error) {
+
+	if cipherKey != nil {
+		var n int
+		_, err := readEncryptedUrl(ctx, fileUrl, "", cipherKey, isContentCompressed, isFullChunk, offset, size, func(data []byte) {
+			n = copy(buf, data)
+		})
+		return int64(n), err
+	}
+
+	req, err := http.NewRequest(http.MethodGet, fileUrl, nil)
+	if err != nil {
+		return 0, err
+	}
+	if !isFullChunk {
+		req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1))
+	} else {
+		req.Header.Set("Accept-Encoding", "gzip")
+	}
+
+	r, err := GetGlobalHttpClient().Do(req)
+	if err != nil {
+		return 0, err
+	}
+	defer CloseResponse(r)
+
+	if r.StatusCode >= 400 {
+		return 0, fmt.Errorf("%s: %s", fileUrl, r.Status)
+	}
+
+	var reader io.ReadCloser
+	contentEncoding := r.Header.Get("Content-Encoding")
+	switch contentEncoding {
+	case "gzip":
+		reader, err = gzip.NewReader(r.Body)
+		if err != nil {
+			return 0, err
+		}
+		defer reader.Close()
+	default:
+		reader = r.Body
+	}
+
+	var (
+		i, m int
+		n    int64
+	)
+
+	// refers to https://github.com/golang/go/blob/master/src/bytes/buffer.go#L199
+	// commit id c170b14c2c1cfb2fd853a37add92a82fd6eb4318
+	for {
+		m, err = reader.Read(buf[i:])
+		i += m
+		n += int64(m)
+		if err == io.EOF {
+			return n, nil
+		}
+		if err != nil {
+			return n, err
+		}
+		if n == int64(len(buf)) {
+			break
+		}
+	}
+	// drains the response body to avoid memory leak
+	data, _ := io.ReadAll(reader)
+	if len(data) != 0 {
+		glog.V(1).InfofCtx(ctx, "%s reader has remaining %d bytes", contentEncoding, len(data))
+	}
+	return n, err
+}
+
+func ReadUrlAsStream(ctx context.Context, fileUrl, jwt string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error) {
+	if cipherKey != nil {
+		return readEncryptedUrl(ctx, fileUrl, jwt, cipherKey, isContentGzipped, isFullChunk, offset, size, fn)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileUrl, nil)
+	if err != nil {
+		return false, err
+	}
+	maybeAddAuth(req, jwt)
+
+	if isFullChunk {
+		req.Header.Add("Accept-Encoding", "gzip")
+	} else {
+		req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1))
+	}
+	request_id.InjectToRequest(ctx, req)
+
+	r, err := GetGlobalHttpClient().Do(req)
+	if err != nil {
+		if ctx.Err() == nil {
+			recordUnreachable(req.URL.Host)
+		}
+		return true, err
+	}
+	recordReachable(req.URL.Host)
+	defer CloseResponse(r)
+	if r.StatusCode >= 400 {
+		if r.StatusCode == http.StatusNotFound {
+			return true, fmt.Errorf("%s: %s: %w", fileUrl, r.Status, ErrNotFound)
+		}
+		if r.StatusCode == http.StatusTooManyRequests {
+			return false, fmt.Errorf("%s: %s: %w", fileUrl, r.Status, ErrTooManyRequests)
+		}
+		retryable = r.StatusCode >= 499
+		return retryable, fmt.Errorf("%s: %s", fileUrl, r.Status)
+	}
+
+	var reader io.ReadCloser
+	contentEncoding := r.Header.Get("Content-Encoding")
+	switch contentEncoding {
+	case "gzip":
+		reader, err = gzip.NewReader(r.Body)
+		if err != nil {
+			return true, err
+		}
+		defer reader.Close()
+	default:
+		reader = r.Body
+	}
+
+	var (
+		m int
+	)
+	buf := mem.Allocate(256 * 1024)
+	defer mem.Free(buf)
+
+	for {
+		// Check for context cancellation before each read
+		select {
+		case <-ctx.Done():
+			return false, ctx.Err()
+		default:
+		}
+
+		m, err = reader.Read(buf)
+		if m > 0 {
+			fn(buf[:m])
+		}
+		if err == io.EOF {
+			return false, nil
+		}
+		if err != nil {
+			return true, err
+		}
+	}
+
+}
+
+func readEncryptedUrl(ctx context.Context, fileUrl, jwt string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (bool, error) {
+	encryptedData, retryable, err := GetAuthenticated(ctx, fileUrl, jwt)
+	if err != nil {
+		return retryable, fmt.Errorf("fetch %s: %v", fileUrl, err)
+	}
+	decryptedData, err := util.Decrypt(encryptedData, util.CipherKey(cipherKey))
+	if err != nil {
+		return false, fmt.Errorf("decrypt %s: %v", fileUrl, err)
+	}
+	if isContentCompressed {
+		decryptedData, err = util.DecompressData(decryptedData)
+		if err != nil {
+			glog.V(0).InfofCtx(ctx, "unzip decrypt %s: %v", fileUrl, err)
+		}
+	}
+	if len(decryptedData) < int(offset)+size {
+		return false, fmt.Errorf("read decrypted %s size %d [%d, %d)", fileUrl, len(decryptedData), offset, int(offset)+size)
+	}
+	if isFullChunk {
+		fn(decryptedData)
+	} else {
+		sliceEnd := int(offset) + size
+		fn(decryptedData[int(offset):sliceEnd])
+	}
+	return false, nil
+}
+
+func ReadUrlAsReaderCloser(fileUrl string, jwt string, rangeHeader string) (*http.Response, io.ReadCloser, error) {
+
+	req, err := http.NewRequest(http.MethodGet, fileUrl, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	if rangeHeader != "" {
+		req.Header.Add("Range", rangeHeader)
+	} else {
+		req.Header.Add("Accept-Encoding", "gzip")
+	}
+
+	maybeAddAuth(req, jwt)
+
+	r, err := GetGlobalHttpClient().Do(req)
+	if err != nil {
+		return nil, nil, err
+	}
+	if r.StatusCode >= 400 {
+		CloseResponse(r)
+		return nil, nil, fmt.Errorf("%s: %s", fileUrl, r.Status)
+	}
+
+	var reader io.ReadCloser
+	contentEncoding := r.Header.Get("Content-Encoding")
+	switch contentEncoding {
+	case "gzip":
+		reader, err = gzip.NewReader(r.Body)
+		if err != nil {
+			return nil, nil, err
+		}
+	default:
+		reader = r.Body
+	}
+
+	return r, reader, nil
+}
+
+func CloseResponse(resp *http.Response) {
+	if resp == nil || resp.Body == nil {
+		return
+	}
+	reader := &CountingReader{reader: resp.Body}
+	io.Copy(io.Discard, reader)
+	resp.Body.Close()
+	if reader.BytesRead > 0 {
+		glog.V(1).Infof("response leftover %d bytes", reader.BytesRead)
+	}
+}
+
+func CloseRequest(req *http.Request) {
+	reader := &CountingReader{reader: req.Body}
+	io.Copy(io.Discard, reader)
+	req.Body.Close()
+	if reader.BytesRead > 0 {
+		glog.V(1).Infof("request leftover %d bytes", reader.BytesRead)
+	}
+}
+
+type CountingReader struct {
+	reader    io.Reader
+	BytesRead int
+}
+
+func (r *CountingReader) Read(p []byte) (n int, err error) {
+	n, err = r.reader.Read(p)
+	r.BytesRead += n
+	return n, err
+}
+
+// refreshedUrls asks for a fresh location list and reports whether it is worth
+// retrying on: a list that comes back empty, or identical to the one that just
+// failed everywhere, says the locations were never the problem.
+func refreshedUrls(ctx context.Context, refreshUrls RefreshUrlsFunc, current []string, fileId string) ([]string, bool) {
+	if refreshUrls == nil {
+		return nil, false
+	}
+	fresh := refreshUrls()
+	if len(fresh) == 0 || SameUrls(current, fresh) {
+		return nil, false
+	}
+	glog.V(0).InfofCtx(ctx, "chunk %s failed on every known location, retrying on %d fresh ones", fileId, len(fresh))
+	return fresh, true
+}
+
+// SameUrls reports whether two location lists hold the same URLs, regardless of
+// order: lookups shuffle the locations they return, so comparing positionally
+// would read a reshuffle of the very same replicas as a fresh set.
+func SameUrls(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	counts := make(map[string]int, len(a))
+	for _, url := range a {
+		counts[url]++
+	}
+	for _, url := range b {
+		if counts[url] == 0 {
+			return false
+		}
+		counts[url]--
+	}
+	return true
+}
+
+// RefreshUrlsFunc supplies a fresh location list for a chunk. The retry loops
+// call it at most once: after every location in the list failed, to retry on
+// the fresh list at once, or after a later location answered for one that
+// failed, so the next read starts from what the cluster knows now instead of
+// trying the same dead replica again. Returning nil or the same list leaves
+// the caller on the original locations.
+type RefreshUrlsFunc func() []string
+
+// RetriedFetchChunkData reads a chunk, trying every location before backing off
+// and trying them again. refreshUrls may be nil; when it is not, a pass in which
+// every location failed is treated as a stale list rather than a slow cluster,
+// and the fresh list is tried immediately instead of after the next backoff.
+// A pass that failed on one location and succeeded on another refreshes the
+// list for the reads that follow.
+func RetriedFetchChunkData(ctx context.Context, buffer []byte, urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, fileId string, refreshUrls RefreshUrlsFunc) (n int, err error) {
+
+	var jwt security.EncodedJwt
+	if len(urlStrings) > 0 && IsProxyChunkUrl(urlStrings[0]) {
+		jwt = security.EncodedJwt(JwtForFilerServer(false))
+	} else {
+		loadJwtConfigOnce.Do(loadJwtConfig)
+		if cfg := jwtSigningReadConfigPtr.Load(); cfg != nil && len(cfg.key) > 0 {
+			jwt = security.GenJwtForVolumeServer(cfg.key, cfg.expires, fileId)
+		}
+	}
+
+	// For unencrypted, non-gzipped full chunks, use direct buffer read
+	// This avoids the 64KB intermediate buffer and callback overhead
+	if cipherKey == nil && !isGzipped && isFullChunk {
+		return retriedFetchChunkDataDirect(ctx, buffer, urlStrings, string(jwt), fileId, refreshUrls)
+	}
+
+	var shouldRetry bool
+
+	for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
+		// Check for context cancellation before starting retry loop
+		select {
+		case <-ctx.Done():
+			return n, ctx.Err()
+		default:
+		}
+
+		var failed bool
+		for _, urlString := range ReachableFirst(urlStrings) {
+			// Check for context cancellation before each volume server request
+			select {
+			case <-ctx.Done():
+				return n, ctx.Err()
+			default:
+			}
+
+			n = 0
+			if strings.Contains(urlString, "%") {
+				urlString = url.PathEscape(urlString)
+			}
+			shouldRetry, err = ReadUrlAsStream(ctx, AppendQueryParameter(urlString, "readDeleted", "true"), string(jwt), cipherKey, isGzipped, isFullChunk, offset, len(buffer), func(data []byte) {
+				// Check for context cancellation during data processing
+				select {
+				case <-ctx.Done():
+					// Stop processing data when context is cancelled
+					return
+				default:
+				}
+
+				if n < len(buffer) {
+					x := copy(buffer[n:], data)
+					n += x
+				}
+			})
+			if !shouldRetry {
+				break
+			}
+			if err != nil {
+				failed = true
+				glog.V(0).InfofCtx(ctx, "read %s failed, err: %v", urlString, err)
+			} else {
+				break
+			}
+		}
+		if err == nil && failed && refreshUrls != nil {
+			refreshUrls()
+		}
+		if err != nil && shouldRetry {
+			if fresh, ok := refreshedUrls(ctx, refreshUrls, urlStrings, fileId); ok {
+				urlStrings, refreshUrls = fresh, nil
+				continue
+			}
+			refreshUrls = nil
+			glog.V(0).InfofCtx(ctx, "retry reading in %v", waitTime)
+			// Sleep with proper context cancellation and timer cleanup
+			timer := time.NewTimer(waitTime)
+			select {
+			case <-ctx.Done():
+				timer.Stop()
+				return n, ctx.Err()
+			case <-timer.C:
+				// Continue with retry
+			}
+		} else {
+			break
+		}
+	}
+
+	return n, err
+
+}
+
+// retriedFetchChunkDataDirect reads chunk data directly into the buffer without
+// intermediate buffering. This reduces memory copies and improves throughput
+// for large chunk reads.
+func retriedFetchChunkDataDirect(ctx context.Context, buffer []byte, urlStrings []string, jwt, fileId string, refreshUrls RefreshUrlsFunc) (n int, err error) {
+	var shouldRetry bool
+
+	for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
+		select {
+		case <-ctx.Done():
+			return 0, ctx.Err()
+		default:
+		}
+
+		var failed bool
+		for _, urlString := range ReachableFirst(urlStrings) {
+			select {
+			case <-ctx.Done():
+				return 0, ctx.Err()
+			default:
+			}
+
+			n, shouldRetry, err = readUrlDirectToBuffer(ctx, AppendQueryParameter(urlString, "readDeleted", "true"), jwt, buffer)
+			if err == nil {
+				if failed && refreshUrls != nil {
+					refreshUrls()
+				}
+				return n, nil
+			}
+			if !shouldRetry {
+				break
+			}
+			failed = true
+			glog.V(0).InfofCtx(ctx, "read %s failed, err: %v", urlString, err)
+		}
+
+		if err != nil && shouldRetry {
+			if fresh, ok := refreshedUrls(ctx, refreshUrls, urlStrings, fileId); ok {
+				urlStrings, refreshUrls = fresh, nil
+				continue
+			}
+			refreshUrls = nil
+			glog.V(0).InfofCtx(ctx, "retry reading in %v", waitTime)
+			timer := time.NewTimer(waitTime)
+			select {
+			case <-ctx.Done():
+				timer.Stop()
+				return 0, ctx.Err()
+			case <-timer.C:
+			}
+		} else {
+			break
+		}
+	}
+
+	return n, err
+}
+
+// readUrlDirectToBuffer reads HTTP response directly into the provided buffer,
+// avoiding intermediate buffer allocations and copies.
+func readUrlDirectToBuffer(ctx context.Context, fileUrl, jwt string, buffer []byte) (n int, retryable bool, err error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileUrl, nil)
+	if err != nil {
+		return 0, false, err
+	}
+	maybeAddAuth(req, jwt)
+	request_id.InjectToRequest(ctx, req)
+
+	r, err := GetGlobalHttpClient().Do(req)
+	if err != nil {
+		if ctx.Err() == nil {
+			recordUnreachable(req.URL.Host)
+		}
+		return 0, true, err
+	}
+	recordReachable(req.URL.Host)
+	defer CloseResponse(r)
+
+	if r.StatusCode >= 400 {
+		if r.StatusCode == http.StatusNotFound {
+			return 0, true, fmt.Errorf("%s: %s: %w", fileUrl, r.Status, ErrNotFound)
+		}
+		if r.StatusCode == http.StatusTooManyRequests {
+			return 0, false, fmt.Errorf("%s: %s: %w", fileUrl, r.Status, ErrTooManyRequests)
+		}
+		retryable = r.StatusCode >= 499
+		return 0, retryable, fmt.Errorf("%s: %s", fileUrl, r.Status)
+	}
+
+	// Read directly into the buffer without intermediate copying
+	// This is significantly faster for large chunks (16MB+)
+	var totalRead int
+	for totalRead < len(buffer) {
+		select {
+		case <-ctx.Done():
+			return totalRead, false, ctx.Err()
+		default:
+		}
+
+		m, readErr := r.Body.Read(buffer[totalRead:])
+		totalRead += m
+		if readErr != nil {
+			if readErr == io.EOF {
+				// Return io.ErrUnexpectedEOF if we haven't filled the buffer
+				// This prevents silent data corruption from truncated responses
+				if totalRead < len(buffer) {
+					return totalRead, true, io.ErrUnexpectedEOF
+				}
+				return totalRead, false, nil
+			}
+			return totalRead, true, readErr
+		}
+	}
+
+	return totalRead, false, nil
+}

weed/util/http/http_global_client_util_test.go

@@ -0,0 +1,259 @@
+package http
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestAppendQueryParameter(t *testing.T) {
+	testCases := []struct {
+		name     string
+		rawURL   string
+		key      string
+		value    string
+		expected string
+	}{
+		{
+			name:     "without existing query",
+			rawURL:   "http://example.com/3,abc",
+			key:      "readDeleted",
+			value:    "true",
+			expected: "http://example.com/3,abc?readDeleted=true",
+		},
+		{
+			name:     "with existing query",
+			rawURL:   "http://example.com/?proxyChunkId=3,abc",
+			key:      "readDeleted",
+			value:    "true",
+			expected: "http://example.com/?proxyChunkId=3,abc&readDeleted=true",
+		},
+		{
+			name:     "with trailing question mark",
+			rawURL:   "http://example.com/?",
+			key:      "readDeleted",
+			value:    "true",
+			expected: "http://example.com/?readDeleted=true",
+		},
+		{
+			name:     "with trailing ampersand",
+			rawURL:   "http://example.com/?proxyChunkId=3,abc&",
+			key:      "readDeleted",
+			value:    "true",
+			expected: "http://example.com/?proxyChunkId=3,abc&readDeleted=true",
+		},
+		{
+			name:     "encodes values",
+			rawURL:   "http://example.com/data",
+			key:      "note",
+			value:    "space value",
+			expected: "http://example.com/data?note=space+value",
+		},
+		{
+			name:     "preserves fragment",
+			rawURL:   "http://example.com/data#frag",
+			key:      "readDeleted",
+			value:    "true",
+			expected: "http://example.com/data?readDeleted=true#frag",
+		},
+		{
+			name:     "blank url",
+			rawURL:   "",
+			key:      "readDeleted",
+			value:    "true",
+			expected: "?readDeleted=true",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			actual := AppendQueryParameter(tc.rawURL, tc.key, tc.value)
+			if actual != tc.expected {
+				t.Fatalf("expected %q, got %q", tc.expected, actual)
+			}
+		})
+	}
+}
+func TestReadUrlAsStreamReturnsGzipReaderError(t *testing.T) {
+	InitGlobalHttpClient()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Encoding", "gzip")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("not gzip"))
+	}))
+	defer server.Close()
+
+	_, err := ReadUrlAsStream(context.Background(), server.URL, "", nil, false, true, 0, 0, func(data []byte) {})
+	if err == nil {
+		t.Fatal("ReadUrlAsStream returned nil error for invalid gzip response")
+	}
+}
+
+func TestDeleteReturnsInvalidRequestErrorBeforeAddingAuth(t *testing.T) {
+	defer func() {
+		if r := recover(); r != nil {
+			t.Fatalf("Delete panicked before returning the request error: %v", r)
+		}
+	}()
+
+	if err := Delete("http://[::1", "jwt"); err == nil {
+		t.Fatal("expected invalid request error")
+	}
+}
+
+func TestDeleteTreatsNoContentAsSuccess(t *testing.T) {
+	InitGlobalHttpClient()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodDelete {
+			t.Errorf("expected DELETE, got %s", r.Method)
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer server.Close()
+
+	if err := Delete(server.URL, ""); err != nil {
+		t.Fatalf("expected 204 DELETE to succeed, got %v", err)
+	}
+}
+
+func TestDeleteProxiedReturnsInvalidRequestErrorBeforeAddingAuth(t *testing.T) {
+	defer func() {
+		if r := recover(); r != nil {
+			t.Fatalf("DeleteProxied panicked before returning the request error: %v", r)
+		}
+	}()
+
+	if _, _, err := DeleteProxied("http://[::1", "jwt"); err == nil {
+		t.Fatal("expected invalid request error")
+	}
+}
+
+// TestRetriedFetchChunkDataRetriesFreshUrlsImmediately covers the case the
+// refresh hook exists for: every location the caller knew about is gone, and
+// the data is live somewhere the caller has not heard of yet. The read must
+// land on the fresh location without first sitting through the backoff ladder.
+func TestRetriedFetchChunkDataRetriesFreshUrlsImmediately(t *testing.T) {
+	payload := []byte("chunk contents")
+	live := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write(payload)
+	}))
+	defer live.Close()
+
+	// A port nothing listens on: the address is well formed, the dial fails.
+	dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	deadURL := dead.URL
+	dead.Close()
+
+	refreshed := 0
+	buffer := make([]byte, len(payload))
+	start := time.Now()
+	n, err := RetriedFetchChunkData(context.Background(), buffer, []string{deadURL + "/3,abc"}, nil, false, true, 0, "3,abc",
+		func() []string {
+			refreshed++
+			return []string{live.URL + "/3,abc"}
+		})
+	elapsed := time.Since(start)
+
+	if err != nil {
+		t.Fatalf("fetch with a refreshed location: %v", err)
+	}
+	if n != len(payload) || string(buffer[:n]) != string(payload) {
+		t.Fatalf("got %q, want %q", buffer[:n], payload)
+	}
+	if refreshed != 1 {
+		t.Fatalf("refresh called %d times, want exactly 1", refreshed)
+	}
+	// The first backoff is a full second; landing well under it is the point.
+	if elapsed > 500*time.Millisecond {
+		t.Fatalf("took %v, expected the retry to skip the backoff", elapsed)
+	}
+}
+
+// TestRetriedFetchChunkDataKeepsBackoffWhenLocationsAreUnchanged makes sure a
+// refresh that returns the same list is treated as "the locations were never
+// the problem" rather than as a reason to spin.
+func TestRetriedFetchChunkDataKeepsBackoffWhenLocationsAreUnchanged(t *testing.T) {
+	dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	deadURL := dead.URL
+	dead.Close()
+
+	urls := []string{deadURL + "/3,abc"}
+	refreshed := 0
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+
+	buffer := make([]byte, 4)
+	_, err := RetriedFetchChunkData(ctx, buffer, urls, nil, false, true, 0, "3,abc", func() []string {
+		refreshed++
+		return urls
+	})
+	if err == nil {
+		t.Fatal("expected the fetch to fail against a dead location")
+	}
+	if refreshed != 1 {
+		t.Fatalf("refresh called %d times, want exactly 1", refreshed)
+	}
+}
+
+// TestRetriedFetchChunkDataRefreshesLocationsAfterPartialFailure covers a read
+// that fails on a cached dead replica and succeeds on the next one: the read
+// itself is fine, but the list it came from is stale and must be refreshed so
+// later reads do not start with the dead replica again. Both the direct and
+// the streaming read paths take this branch.
+func TestRetriedFetchChunkDataRefreshesLocationsAfterPartialFailure(t *testing.T) {
+	forgetUnreachable(t)
+	payload := []byte("chunk contents")
+	live := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write(payload)
+	}))
+	defer live.Close()
+	dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	deadURL := dead.URL
+	dead.Close()
+
+	for _, isFullChunk := range []bool{true, false} {
+		forgetUnreachable(t)
+		refreshed := 0
+		refresh := func() []string {
+			refreshed++
+			return []string{live.URL + "/3,abc"}
+		}
+		buffer := make([]byte, len(payload))
+		n, err := RetriedFetchChunkData(context.Background(), buffer, []string{deadURL + "/3,abc", live.URL + "/3,abc"}, nil, false, isFullChunk, 0, "3,abc", refresh)
+		if err != nil || string(buffer[:n]) != string(payload) {
+			t.Fatalf("fullChunk=%v: got %q, %v; want %q", isFullChunk, buffer[:n], err, payload)
+		}
+		if refreshed != 1 {
+			t.Fatalf("fullChunk=%v: refresh called %d times after a partial failure, want exactly 1", isFullChunk, refreshed)
+		}
+
+		// a read answered by the first location leaves the list alone
+		refreshed = 0
+		n, err = RetriedFetchChunkData(context.Background(), buffer, []string{live.URL + "/3,abc", deadURL + "/3,abc"}, nil, false, isFullChunk, 0, "3,abc", refresh)
+		if err != nil || string(buffer[:n]) != string(payload) {
+			t.Fatalf("fullChunk=%v: got %q, %v; want %q", isFullChunk, buffer[:n], err, payload)
+		}
+		if refreshed != 0 {
+			t.Fatalf("fullChunk=%v: refresh called %d times without a failure, want none", isFullChunk, refreshed)
+		}
+	}
+}
+
+func TestSameUrlsIgnoresOrder(t *testing.T) {
+	a := []string{"http://a:8080/3,x", "http://b:8080/3,x"}
+	if !SameUrls(a, []string{a[1], a[0]}) {
+		t.Fatal("a reshuffle of the same locations should not count as fresh")
+	}
+	if SameUrls(a, []string{a[0], "http://c:8080/3,x"}) {
+		t.Fatal("a different location should count as fresh")
+	}
+	if SameUrls(a, a[:1]) {
+		t.Fatal("a shorter list should count as fresh")
+	}
+}

weed/util/http/query_middleware.go

@@ -0,0 +1,25 @@
+package http
+
+import (
+	"net/http"
+	"strings"
+)
+
+// EscapeSemicolonsInQuery re-encodes raw ';' in the query string. RFC 3986
+// allows ';' in a query and AWS treats it as data, but Go's url.ParseQuery
+// drops every key=value pair containing one. Presigned URLs carry
+// X-Amz-SignedHeaders=content-type%3Bhost; clients that decode the %3B would
+// otherwise lose the parameter entirely and fail signature verification.
+func EscapeSemicolonsInQuery(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.Contains(r.URL.RawQuery, ";") {
+			u := *r.URL
+			u.RawQuery = strings.ReplaceAll(u.RawQuery, ";", "%3B")
+			r2 := new(http.Request)
+			*r2 = *r
+			r2.URL = &u
+			r = r2
+		}
+		next.ServeHTTP(w, r)
+	})
+}

weed/util/http/reachability.go

@@ -0,0 +1,60 @@
+package http
+
+import (
+	"net/url"
+	"sync"
+	"time"
+)
+
+// unreachableRetryInterval is how long a server that failed to answer is tried
+// last before a read probes it again.
+const unreachableRetryInterval = 30 * time.Second
+
+// unreachable maps a host to when it last failed to answer a request. A dead
+// replica sits in the location list of every volume it held, so this is kept
+// per host rather than per volume.
+var unreachable sync.Map
+
+func recordUnreachable(host string) {
+	unreachable.Store(host, time.Now())
+}
+
+func recordReachable(host string) {
+	unreachable.Delete(host)
+}
+
+// ReachableFirst orders urls so that hosts which recently failed to answer
+// come last, keeping the order within each group. Once the retry interval has
+// passed, the first read to ask claims the probe and tries that host first;
+// the others keep it last until the probe settles. A read probes at most one
+// host, so a claim is always followed by an attempt and several expired hosts
+// are probed by successive reads.
+func ReachableFirst(urls []string) []string {
+	var probe, reachable, unanswered []string
+	for _, u := range urls {
+		host := hostOf(u)
+		failedAt, failed := unreachable.Load(host)
+		switch {
+		case !failed:
+			reachable = append(reachable, u)
+		case time.Since(failedAt.(time.Time)) < unreachableRetryInterval:
+			unanswered = append(unanswered, u)
+		case probe == nil && unreachable.CompareAndSwap(host, failedAt, time.Now()):
+			probe = append(probe, u)
+		default:
+			unanswered = append(unanswered, u)
+		}
+	}
+	if probe == nil && unanswered == nil {
+		return urls
+	}
+	return append(append(probe, reachable...), unanswered...)
+}
+
+func hostOf(rawUrl string) string {
+	parsed, err := url.Parse(rawUrl)
+	if err != nil {
+		return ""
+	}
+	return parsed.Host
+}

weed/util/http/reachability_test.go

@@ -0,0 +1,112 @@
+package http
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func forgetUnreachable(t *testing.T) {
+	t.Helper()
+	forget := func() {
+		unreachable.Range(func(host, _ any) bool {
+			unreachable.Delete(host)
+			return true
+		})
+	}
+	forget()
+	t.Cleanup(forget)
+}
+
+func assertOrder(t *testing.T, got []string, want ...string) {
+	t.Helper()
+	if len(got) != len(want) {
+		t.Fatalf("got %v, want %v", got, want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Fatalf("got %v, want %v", got, want)
+		}
+	}
+}
+
+func TestReachableFirstTriesUnansweringHostLast(t *testing.T) {
+	forgetUnreachable(t)
+	urls := []string{"http://a:8080/3,x", "http://b:8080/3,x", "http://c:8080/3,x"}
+
+	assertOrder(t, ReachableFirst(urls), urls...)
+	recordUnreachable("a:8080")
+	assertOrder(t, ReachableFirst(urls), urls[1], urls[2], urls[0])
+	recordReachable("a:8080")
+	assertOrder(t, ReachableFirst(urls), urls...)
+}
+
+func TestReachableFirstProbesOnceAfterRetryInterval(t *testing.T) {
+	forgetUnreachable(t)
+	urls := []string{"http://b:8080/3,x", "http://a:8080/3,x"}
+	unreachable.Store("a:8080", time.Now().Add(-unreachableRetryInterval))
+
+	// the first read to come by probes a, the next keeps it last until that settles
+	assertOrder(t, ReachableFirst(urls), urls[1], urls[0])
+	assertOrder(t, ReachableFirst(urls), urls...)
+	recordReachable("a:8080")
+	assertOrder(t, ReachableFirst(urls), urls...)
+}
+
+func TestReachableFirstProbesOneExpiredHostPerRead(t *testing.T) {
+	forgetUnreachable(t)
+	urls := []string{"http://a:8080/3,x", "http://b:8080/3,x", "http://c:8080/3,x"}
+	expired := time.Now().Add(-unreachableRetryInterval)
+	unreachable.Store("a:8080", expired)
+	unreachable.Store("c:8080", expired)
+
+	assertOrder(t, ReachableFirst(urls), urls[0], urls[1], urls[2])
+	assertOrder(t, ReachableFirst(urls), urls[2], urls[1], urls[0])
+	assertOrder(t, ReachableFirst(urls), urls[1], urls[0], urls[2])
+}
+
+// hangupServer accepts the connection and drops it without answering, the way
+// a replica behind a broken network does, and counts how often that happened.
+func hangupServer(t *testing.T) (*httptest.Server, *int32) {
+	t.Helper()
+	var hangups int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&hangups, 1)
+		conn, _, err := w.(http.Hijacker).Hijack()
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		conn.Close()
+	}))
+	t.Cleanup(srv.Close)
+	return srv, &hangups
+}
+
+func TestRetriedFetchChunkDataTriesUnansweringServerLast(t *testing.T) {
+	forgetUnreachable(t)
+	payload := []byte("chunk contents")
+	live := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write(payload)
+	}))
+	defer live.Close()
+	hangup, hangups := hangupServer(t)
+
+	urls := []string{hangup.URL + "/3,abc", live.URL + "/3,abc"}
+	for i := 0; i < 3; i++ {
+		buffer := make([]byte, len(payload))
+		n, err := RetriedFetchChunkData(context.Background(), buffer, urls, nil, false, true, 0, "3,abc", nil)
+		if err != nil {
+			t.Fatalf("read %d: %v", i, err)
+		}
+		if string(buffer[:n]) != string(payload) {
+			t.Fatalf("read %d got %q, want %q", i, buffer[:n], payload)
+		}
+	}
+	if got := atomic.LoadInt32(hangups); got != 1 {
+		t.Fatalf("the server that hung up was tried %d times, want once", got)
+	}
+}

weed/util/httpdown/http_down.go

@@ -0,0 +1,400 @@
+// Package httpdown provides http.ConnState enabled graceful termination of
+// http.Server.
+// based on github.com/facebookarchive/httpdown, who's licence is MIT-licence,
+// we add a feature of supporting for http TLS
+package httpdown
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/facebookgo/clock"
+	"github.com/facebookgo/stats"
+)
+
+const (
+	defaultStopTimeout = time.Minute
+	defaultKillTimeout = time.Minute
+)
+
+// A Server allows encapsulates the process of accepting new connections and
+// serving them, and gracefully shutting down the listener without dropping
+// active connections.
+type Server interface {
+	// Wait waits for the serving loop to finish. This will happen when Stop is
+	// called, at which point it returns no error, or if there is an error in the
+	// serving loop. You must call Wait after calling Serve or ListenAndServe.
+	Wait() error
+
+	// Stop stops the listener. It will block until all connections have been
+	// closed.
+	Stop() error
+}
+
+// HTTP defines the configuration for serving a http.Server. Multiple calls to
+// Serve or ListenAndServe can be made on the same HTTP instance. The default
+// timeouts of 1 minute each result in a maximum of 2 minutes before a Stop()
+// returns.
+type HTTP struct {
+	// StopTimeout is the duration before we begin force closing connections.
+	// Defaults to 1 minute.
+	StopTimeout time.Duration
+
+	// KillTimeout is the duration before which we completely give up and abort
+	// even though we still have connected clients. This is useful when a large
+	// number of client connections exist and closing them can take a long time.
+	// Note, this is in addition to the StopTimeout. Defaults to 1 minute.
+	KillTimeout time.Duration
+
+	// Stats is optional. If provided, it will be used to record various metrics.
+	Stats stats.Client
+
+	// Clock allows for testing timing related functionality. Do not specify this
+	// in production code.
+	Clock clock.Clock
+
+	// when set CertFile and KeyFile, the httpDown will start a http with TLS.
+	// Files containing a certificate and matching private key for the
+	// server must be provided if neither the Server's
+	// TLSConfig.Certificates nor TLSConfig.GetCertificate are populated.
+	// If the certificate is signed by a certificate authority, the
+	// certFile should be the concatenation of the server's certificate,
+	// any intermediates, and the CA's certificate.
+	CertFile, KeyFile string
+}
+
+// Serve provides the low-level API which is useful if you're creating your own
+// net.Listener.
+func (h HTTP) Serve(s *http.Server, l net.Listener) Server {
+	stopTimeout := h.StopTimeout
+	if stopTimeout == 0 {
+		stopTimeout = defaultStopTimeout
+	}
+	killTimeout := h.KillTimeout
+	if killTimeout == 0 {
+		killTimeout = defaultKillTimeout
+	}
+	klock := h.Clock
+	if klock == nil {
+		klock = clock.New()
+	}
+
+	ss := &server{
+		stopTimeout:  stopTimeout,
+		killTimeout:  killTimeout,
+		stats:        h.Stats,
+		clock:        klock,
+		oldConnState: s.ConnState,
+		listener:     l,
+		server:       s,
+		serveDone:    make(chan struct{}),
+		serveErr:     make(chan error, 1),
+		new:          make(chan net.Conn),
+		active:       make(chan net.Conn),
+		idle:         make(chan net.Conn),
+		closed:       make(chan net.Conn),
+		stop:         make(chan chan struct{}),
+		kill:         make(chan chan struct{}),
+		certFile:     h.CertFile,
+		keyFile:      h.KeyFile,
+	}
+	s.ConnState = ss.connState
+	go ss.manage()
+	go ss.serve()
+	return ss
+}
+
+// ListenAndServe returns a Server for the given http.Server. It is equivalent
+// to ListenAndServe from the standard library, but returns immediately.
+// Requests will be accepted in a background goroutine. If the http.Server has
+// a non-nil TLSConfig, a TLS enabled listener will be setup.
+func (h HTTP) ListenAndServe(s *http.Server) (Server, error) {
+	addr := s.Addr
+	if addr == "" {
+		if s.TLSConfig == nil {
+			addr = ":http"
+		} else {
+			addr = ":https"
+		}
+	}
+	l, err := net.Listen("tcp", addr)
+	if err != nil {
+		stats.BumpSum(h.Stats, "listen.error", 1)
+		return nil, err
+	}
+	if s.TLSConfig != nil {
+		l = tls.NewListener(l, s.TLSConfig)
+	}
+	return h.Serve(s, l), nil
+}
+
+// server manages the serving process and allows for gracefully stopping it.
+type server struct {
+	stopTimeout time.Duration
+	killTimeout time.Duration
+	stats       stats.Client
+	clock       clock.Clock
+
+	oldConnState func(net.Conn, http.ConnState)
+	server       *http.Server
+	serveDone    chan struct{}
+	serveErr     chan error
+	listener     net.Listener
+
+	new    chan net.Conn
+	active chan net.Conn
+	idle   chan net.Conn
+	closed chan net.Conn
+	stop   chan chan struct{}
+	kill   chan chan struct{}
+
+	stopOnce sync.Once
+	stopErr  error
+
+	certFile, keyFile string
+}
+
+func (s *server) connState(c net.Conn, cs http.ConnState) {
+	if s.oldConnState != nil {
+		s.oldConnState(c, cs)
+	}
+
+	switch cs {
+	case http.StateNew:
+		s.new <- c
+	case http.StateActive:
+		s.active <- c
+	case http.StateIdle:
+		s.idle <- c
+	case http.StateHijacked, http.StateClosed:
+		s.closed <- c
+	}
+}
+
+func (s *server) manage() {
+	defer func() {
+		close(s.new)
+		close(s.active)
+		close(s.idle)
+		close(s.closed)
+		close(s.stop)
+		close(s.kill)
+	}()
+
+	var stopDone chan struct{}
+
+	conns := map[net.Conn]http.ConnState{}
+	var countNew, countActive, countIdle float64
+
+	// decConn decrements the count associated with the current state of the
+	// given connection.
+	decConn := func(c net.Conn) {
+		switch conns[c] {
+		default:
+			panic(fmt.Errorf("unknown existing connection: %s", c))
+		case http.StateNew:
+			countNew--
+		case http.StateActive:
+			countActive--
+		case http.StateIdle:
+			countIdle--
+		}
+	}
+
+	// setup a ticker to report various values every minute. if we don't have a
+	// Stats implementation provided, we Stop it so it never ticks.
+	statsTicker := s.clock.Ticker(time.Minute)
+	if s.stats == nil {
+		statsTicker.Stop()
+	}
+
+	for {
+		select {
+		case <-statsTicker.C:
+			// we'll only get here when s.stats is not nil
+			s.stats.BumpAvg("http-state.new", countNew)
+			s.stats.BumpAvg("http-state.active", countActive)
+			s.stats.BumpAvg("http-state.idle", countIdle)
+			s.stats.BumpAvg("http-state.total", countNew+countActive+countIdle)
+		case c := <-s.new:
+			conns[c] = http.StateNew
+			countNew++
+		case c := <-s.active:
+			decConn(c)
+			countActive++
+
+			conns[c] = http.StateActive
+		case c := <-s.idle:
+			decConn(c)
+			countIdle++
+
+			conns[c] = http.StateIdle
+
+			// if we're already stopping, close it
+			if stopDone != nil {
+				c.Close()
+			}
+		case c := <-s.closed:
+			stats.BumpSum(s.stats, "conn.closed", 1)
+			decConn(c)
+			delete(conns, c)
+
+			// if we're waiting to stop and are all empty, we just closed the last
+			// connection and we're done.
+			if stopDone != nil && len(conns) == 0 {
+				close(stopDone)
+				return
+			}
+		case stopDone = <-s.stop:
+			// if we're already all empty, we're already done
+			if len(conns) == 0 {
+				close(stopDone)
+				return
+			}
+
+			// close current idle connections right away
+			for c, cs := range conns {
+				if cs == http.StateIdle {
+					c.Close()
+				}
+			}
+
+			// continue the loop and wait for all the ConnState updates which will
+			// eventually close(stopDone) and return from this goroutine.
+
+		case killDone := <-s.kill:
+			// force close all connections
+			stats.BumpSum(s.stats, "kill.conn.count", float64(len(conns)))
+			for c := range conns {
+				c.Close()
+			}
+
+			// don't block the kill.
+			close(killDone)
+
+			// continue the loop and we wait for all the ConnState updates and will
+			// return from this goroutine when we're all done. otherwise we'll try to
+			// send those ConnState updates on closed channels.
+
+		}
+	}
+}
+
+func (s *server) serve() {
+	stats.BumpSum(s.stats, "serve", 1)
+	switch {
+	case s.certFile != "" || s.keyFile != "":
+		s.serveErr <- s.server.ServeTLS(s.listener, s.certFile, s.keyFile)
+	case s.server.TLSConfig != nil && (len(s.server.TLSConfig.Certificates) > 0 || s.server.TLSConfig.GetCertificate != nil):
+		// TLSConfig carries the certificate source (e.g. a hot-reloading
+		// GetCertificate callback). Pass empty file args so ServeTLS uses it.
+		s.serveErr <- s.server.ServeTLS(s.listener, "", "")
+	default:
+		s.serveErr <- s.server.Serve(s.listener)
+	}
+	close(s.serveDone)
+	close(s.serveErr)
+}
+
+func (s *server) Wait() error {
+	if err := <-s.serveErr; !isUseOfClosedError(err) {
+		return err
+	}
+	return nil
+}
+
+func (s *server) Stop() error {
+	s.stopOnce.Do(func() {
+		defer stats.BumpTime(s.stats, "stop.time").End()
+		stats.BumpSum(s.stats, "stop", 1)
+
+		// first disable keep-alive for new connections
+		s.server.SetKeepAlivesEnabled(false)
+
+		// then close the listener so new connections can't connect come thru
+		closeErr := s.listener.Close()
+		<-s.serveDone
+
+		// then trigger the background goroutine to stop and wait for it
+		stopDone := make(chan struct{})
+		s.stop <- stopDone
+
+		// wait for stop
+		select {
+		case <-stopDone:
+		case <-s.clock.After(s.stopTimeout):
+			defer stats.BumpTime(s.stats, "kill.time").End()
+			stats.BumpSum(s.stats, "kill", 1)
+
+			// stop timed out, wait for kill
+			killDone := make(chan struct{})
+			s.kill <- killDone
+			select {
+			case <-killDone:
+			case <-s.clock.After(s.killTimeout):
+				// kill timed out, give up
+				stats.BumpSum(s.stats, "kill.timeout", 1)
+			}
+		}
+
+		if closeErr != nil && !isUseOfClosedError(closeErr) {
+			stats.BumpSum(s.stats, "listener.close.error", 1)
+			s.stopErr = closeErr
+		}
+	})
+	return s.stopErr
+}
+
+func isUseOfClosedError(err error) bool {
+	if err == nil {
+		return false
+	}
+	if opErr, ok := err.(*net.OpError); ok {
+		err = opErr.Err
+	}
+	return err.Error() == "use of closed network connection"
+}
+
+// ListenAndServe is a convenience function to serve and wait for a SIGTERM
+// or SIGINT before shutting down.
+func ListenAndServe(s *http.Server, hd *HTTP) error {
+	if hd == nil {
+		hd = &HTTP{}
+	}
+	hs, err := hd.ListenAndServe(s)
+	if err != nil {
+		return err
+	}
+
+	waiterr := make(chan error, 1)
+	go func() {
+		defer close(waiterr)
+		waiterr <- hs.Wait()
+	}()
+
+	signals := make(chan os.Signal, 10)
+	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
+
+	select {
+	case err := <-waiterr:
+		if err != nil {
+			return err
+		}
+	case <-signals:
+		signal.Stop(signals)
+		if err := hs.Stop(); err != nil {
+			return err
+		}
+		if err := <-waiterr; err != nil {
+			return err
+		}
+	}
+	return nil
+}

weed/util/inits.go

@@ -0,0 +1,52 @@
+package util
+
+import (
+	"fmt"
+	"sort"
+)
+
+// HumanReadableIntsMax joins a serials of inits into a smart one like 1-3 5 ... for human readable.
+func HumanReadableIntsMax(max int, ids ...int) string {
+	if len(ids) <= max {
+		return HumanReadableInts(ids...)
+	}
+
+	return HumanReadableInts(ids[:max]...) + " ..."
+}
+
+// HumanReadableInts joins a serials of inits into a smart one like 1-3 5 7-10 for human readable.
+func HumanReadableInts(ids ...int) string {
+	sort.Ints(ids)
+
+	s := ""
+	start := 0
+	last := 0
+
+	for i, v := range ids {
+		if i == 0 {
+			start = v
+			last = v
+			s = fmt.Sprintf("%d", v)
+			continue
+		}
+
+		if last+1 == v {
+			last = v
+			continue
+		}
+
+		if last > start {
+			s += fmt.Sprintf("-%d", last)
+		}
+
+		s += fmt.Sprintf(" %d", v)
+		start = v
+		last = v
+	}
+
+	if last != start {
+		s += fmt.Sprintf("-%d", last)
+	}
+
+	return s
+}

weed/util/inits_test.go

@@ -0,0 +1,20 @@
+package util
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHumanReadableIntsMax(t *testing.T) {
+	assert.Equal(t, "1-2 ...", HumanReadableIntsMax(2, 1, 2, 3))
+	assert.Equal(t, "1 3 ...", HumanReadableIntsMax(2, 1, 3, 5))
+}
+
+func TestHumanReadableInts(t *testing.T) {
+	assert.Equal(t, "1-3", HumanReadableInts(1, 2, 3))
+	assert.Equal(t, "1 3", HumanReadableInts(1, 3))
+	assert.Equal(t, "1 3 5", HumanReadableInts(5, 1, 3))
+	assert.Equal(t, "1-3 5", HumanReadableInts(1, 2, 3, 5))
+	assert.Equal(t, "1-3 5 7-9", HumanReadableInts(7, 9, 8, 1, 2, 3, 5))
+}

weed/util/limited_executor.go

@@ -0,0 +1,40 @@
+package util
+
+// initial version comes from https://github.com/korovkin/limiter/blob/master/limiter.go
+
+// LimitedConcurrentExecutor object
+type LimitedConcurrentExecutor struct {
+	limit     int
+	tokenChan chan int
+}
+
+func NewLimitedConcurrentExecutor(limit int) *LimitedConcurrentExecutor {
+
+	// allocate a limiter instance
+	c := &LimitedConcurrentExecutor{
+		limit:     limit,
+		tokenChan: make(chan int, limit),
+	}
+
+	// allocate the tokenChan:
+	for i := 0; i < c.limit; i++ {
+		c.tokenChan <- i
+	}
+
+	return c
+}
+
+// Execute adds a function to the execution queue.
+// if num of go routines allocated by this instance is < limit
+// launch a new go routine to execute job
+// else wait until a go routine becomes available
+func (c *LimitedConcurrentExecutor) Execute(job func()) {
+	token := <-c.tokenChan
+	go func() {
+		defer func() {
+			c.tokenChan <- token
+		}()
+		// run the job
+		job()
+	}()
+}

weed/util/lock_table.go

@@ -0,0 +1,177 @@
+package util
+
+import (
+	"fmt"
+	"sync"
+	"sync/atomic"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+// LockTable is a table of locks that can be acquired.
+// Locks are acquired in order of request.
+type LockTable[T comparable] struct {
+	lockIdSeq     int64
+	mu            sync.Mutex
+	locks         map[T]*LockEntry
+	locksInFlight map[T]int
+}
+
+type LockEntry struct {
+	mu                            sync.Mutex
+	waiters                       []*ActiveLock // ordered waiters that are blocked by exclusive locks
+	activeSharedLockOwnerCount    int32
+	activeExclusiveLockOwnerCount int32
+	cond                          *sync.Cond
+}
+
+type LockType int
+
+const (
+	SharedLock LockType = iota
+	ExclusiveLock
+)
+
+type ActiveLock struct {
+	ID        int64
+	isDeleted bool
+	intention string // for debugging
+	lockType  LockType
+}
+
+func NewLockTable[T comparable]() *LockTable[T] {
+	return &LockTable[T]{
+		locks:         make(map[T]*LockEntry),
+		locksInFlight: make(map[T]int),
+	}
+}
+
+func (lt *LockTable[T]) NewActiveLock(intention string, lockType LockType) *ActiveLock {
+	id := atomic.AddInt64(&lt.lockIdSeq, 1)
+	l := &ActiveLock{ID: id, intention: intention, lockType: lockType}
+	return l
+}
+
+func isNotFirstWaiter(lock *ActiveLock, entry *LockEntry) bool {
+	return len(entry.waiters) > 0 && lock.ID != entry.waiters[0].ID
+}
+
+func shouldWaitForExclusiveLock(lock *ActiveLock, entry *LockEntry) bool {
+	return !lock.isDeleted && (isNotFirstWaiter(lock, entry) || entry.activeExclusiveLockOwnerCount > 0 || entry.activeSharedLockOwnerCount > 0)
+}
+
+func shouldWaitForSharedLock(lock *ActiveLock, entry *LockEntry) bool {
+	return !lock.isDeleted && (isNotFirstWaiter(lock, entry) || entry.activeExclusiveLockOwnerCount > 0)
+}
+
+func (lt *LockTable[T]) AcquireLock(intention string, key T, lockType LockType) (lock *ActiveLock) {
+	lt.mu.Lock()
+	// Get or create the lock entry for the key
+	entry, exists := lt.locks[key]
+	if !exists {
+		entry = &LockEntry{}
+		entry.cond = sync.NewCond(&entry.mu)
+		lt.locks[key] = entry
+		lt.locksInFlight[key] = 0
+	}
+	lt.locksInFlight[key]++
+	lt.mu.Unlock()
+
+	lock = lt.NewActiveLock(intention, lockType)
+
+	// If the lock is held exclusively, wait
+	entry.mu.Lock()
+	if len(entry.waiters) > 0 || lockType == ExclusiveLock || entry.activeExclusiveLockOwnerCount > 0 {
+		if glog.V(4) {
+			fmt.Printf("ActiveLock %d %s wait for %+v type=%v with waiters %d active r%d w%d.\n", lock.ID, lock.intention, key, lockType, len(entry.waiters), entry.activeSharedLockOwnerCount, entry.activeExclusiveLockOwnerCount)
+			if len(entry.waiters) > 0 {
+				for _, waiter := range entry.waiters {
+					fmt.Printf(" %d", waiter.ID)
+				}
+				fmt.Printf("\n")
+			}
+		}
+		entry.waiters = append(entry.waiters, lock)
+		if lockType == ExclusiveLock {
+			for shouldWaitForExclusiveLock(lock, entry) {
+				entry.cond.Wait()
+			}
+		} else {
+			for shouldWaitForSharedLock(lock, entry) {
+				entry.cond.Wait()
+			}
+		}
+		// Remove the transaction from the waiters list
+		if len(entry.waiters) > 0 && lock.ID == entry.waiters[0].ID {
+			entry.waiters = entry.waiters[1:]
+			entry.cond.Broadcast()
+		}
+	}
+
+	// Otherwise, grant the lock
+	if glog.V(4) {
+		fmt.Printf("ActiveLock %d %s locked %+v type=%v with waiters %d active r%d w%d.\n", lock.ID, lock.intention, key, lockType, len(entry.waiters), entry.activeSharedLockOwnerCount, entry.activeExclusiveLockOwnerCount)
+		if len(entry.waiters) > 0 {
+			for _, waiter := range entry.waiters {
+				fmt.Printf(" %d", waiter.ID)
+			}
+			fmt.Printf("\n")
+		}
+	}
+	if lock.lockType == ExclusiveLock {
+		entry.activeExclusiveLockOwnerCount++
+	} else {
+		entry.activeSharedLockOwnerCount++
+	}
+	entry.mu.Unlock()
+
+	return lock
+}
+
+func (lt *LockTable[T]) ReleaseLock(key T, lock *ActiveLock) {
+	lt.mu.Lock()
+	defer lt.mu.Unlock()
+
+	entry, exists := lt.locks[key]
+	if !exists {
+		return
+	}
+
+	lt.locksInFlight[key]--
+	entry.mu.Lock()
+	defer entry.mu.Unlock()
+
+	// Remove the transaction from the waiters list
+	for i, waiter := range entry.waiters {
+		if waiter == lock {
+			waiter.isDeleted = true
+			entry.waiters = append(entry.waiters[:i], entry.waiters[i+1:]...)
+			break
+		}
+	}
+
+	if lock.lockType == ExclusiveLock {
+		entry.activeExclusiveLockOwnerCount--
+	} else {
+		entry.activeSharedLockOwnerCount--
+	}
+
+	// If there are no waiters, release the lock
+	if len(entry.waiters) == 0 && lt.locksInFlight[key] <= 0 && entry.activeExclusiveLockOwnerCount <= 0 && entry.activeSharedLockOwnerCount <= 0 {
+		delete(lt.locks, key)
+		delete(lt.locksInFlight, key)
+	}
+
+	if glog.V(4) {
+		fmt.Printf("ActiveLock %d %s unlocked %+v type=%v with waiters %d active r%d w%d.\n", lock.ID, lock.intention, key, lock.lockType, len(entry.waiters), entry.activeSharedLockOwnerCount, entry.activeExclusiveLockOwnerCount)
+		if len(entry.waiters) > 0 {
+			for _, waiter := range entry.waiters {
+				fmt.Printf(" %d", waiter.ID)
+			}
+			fmt.Printf("\n")
+		}
+	}
+
+	// Notify the next waiter
+	entry.cond.Broadcast()
+}

weed/util/lock_table_test.go

@@ -0,0 +1,86 @@
+package util
+
+import (
+	"fmt"
+	"math/rand"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestOrderedLock(t *testing.T) {
+	lt := NewLockTable[string]()
+
+	var wg sync.WaitGroup
+	// Simulate transactions requesting locks
+	for i := 1; i <= 50; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			key := "resource"
+			lockType := SharedLock
+			if i%5 == 0 {
+				lockType = ExclusiveLock
+			}
+
+			// Simulate attempting to acquire the lock
+			lock := lt.AcquireLock("", key, lockType)
+
+			// Lock acquired, perform some work
+			fmt.Printf("ActiveLock %d acquired lock %v\n", lock.ID, lockType)
+
+			// Simulate some work
+			time.Sleep(time.Duration(rand.Int31n(10)*10) * time.Millisecond)
+
+			// Release the lock
+			lt.ReleaseLock(key, lock)
+			fmt.Printf("ActiveLock %d released lock %v\n", lock.ID, lockType)
+		}(i)
+	}
+
+	wg.Wait()
+}
+
+func TestShouldWaitForSharedLock(t *testing.T) {
+	lock := &ActiveLock{ID: 2, lockType: SharedLock}
+	entry := &LockEntry{
+		waiters: []*ActiveLock{{ID: 1}, lock},
+	}
+
+	if !shouldWaitForSharedLock(lock, entry) {
+		t.Fatal("shared lock should wait behind earlier waiters")
+	}
+
+	entry.waiters = []*ActiveLock{lock}
+	entry.activeExclusiveLockOwnerCount = 1
+	if !shouldWaitForSharedLock(lock, entry) {
+		t.Fatal("shared lock should wait while an exclusive owner is active")
+	}
+
+	lock.isDeleted = true
+	if shouldWaitForSharedLock(lock, entry) {
+		t.Fatal("deleted shared lock should stop waiting even if an exclusive owner is active")
+	}
+}
+
+func TestShouldWaitForExclusiveLock(t *testing.T) {
+	lock := &ActiveLock{ID: 2, lockType: ExclusiveLock}
+	entry := &LockEntry{
+		waiters: []*ActiveLock{{ID: 1}, lock},
+	}
+
+	if !shouldWaitForExclusiveLock(lock, entry) {
+		t.Fatal("exclusive lock should wait behind earlier waiters")
+	}
+
+	entry.waiters = []*ActiveLock{lock}
+	entry.activeSharedLockOwnerCount = 1
+	if !shouldWaitForExclusiveLock(lock, entry) {
+		t.Fatal("exclusive lock should wait while shared owners are active")
+	}
+
+	lock.isDeleted = true
+	if shouldWaitForExclusiveLock(lock, entry) {
+		t.Fatal("deleted exclusive lock should stop waiting even if shared owners are active")
+	}
+}

weed/util/log_buffer/disk_buffer_cache.go

@@ -0,0 +1,195 @@
+package log_buffer
+
+import (
+	"container/list"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+// DiskBufferCache is a small LRU cache for recently-read historical data buffers
+// This reduces Filer load when multiple consumers are catching up on historical messages
+type DiskBufferCache struct {
+	maxSize   int
+	ttl       time.Duration
+	cache     map[string]*cacheEntry
+	lruList   *list.List
+	mu        sync.RWMutex
+	hits      int64
+	misses    int64
+	evictions int64
+}
+
+type cacheEntry struct {
+	key        string
+	data       []byte
+	offset     int64
+	timestamp  time.Time
+	lruElement *list.Element
+	isNegative bool // true if this is a negative cache entry (data not found)
+}
+
+// NewDiskBufferCache creates a new cache with the specified size and TTL
+// Recommended size: 3-5 buffers (each ~8MB)
+// Recommended TTL: 30-60 seconds
+func NewDiskBufferCache(maxSize int, ttl time.Duration) *DiskBufferCache {
+	cache := &DiskBufferCache{
+		maxSize: maxSize,
+		ttl:     ttl,
+		cache:   make(map[string]*cacheEntry),
+		lruList: list.New(),
+	}
+
+	// Start background cleanup goroutine
+	go cache.cleanupLoop()
+
+	return cache
+}
+
+// Get retrieves a buffer from the cache
+// Returns (data, offset, found)
+// If found=true and data=nil, this is a negative cache entry (data doesn't exist)
+func (c *DiskBufferCache) Get(key string) ([]byte, int64, bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	entry, exists := c.cache[key]
+	if !exists {
+		c.misses++
+		return nil, 0, false
+	}
+
+	// Check if entry has expired
+	if time.Since(entry.timestamp) > c.ttl {
+		c.evict(entry)
+		c.misses++
+		return nil, 0, false
+	}
+
+	// Move to front of LRU list (most recently used)
+	c.lruList.MoveToFront(entry.lruElement)
+	c.hits++
+
+	if entry.isNegative {
+		glog.V(4).Infof("📦 CACHE HIT (NEGATIVE): key=%s - data not found (hits=%d misses=%d)",
+			key, c.hits, c.misses)
+	} else {
+		glog.V(4).Infof("📦 CACHE HIT: key=%s offset=%d size=%d (hits=%d misses=%d)",
+			key, entry.offset, len(entry.data), c.hits, c.misses)
+	}
+
+	return entry.data, entry.offset, true
+}
+
+// Put adds a buffer to the cache
+// If data is nil, this creates a negative cache entry (data doesn't exist)
+func (c *DiskBufferCache) Put(key string, data []byte, offset int64) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	isNegative := data == nil
+
+	// Check if entry already exists
+	if entry, exists := c.cache[key]; exists {
+		// Update existing entry
+		entry.data = data
+		entry.offset = offset
+		entry.timestamp = time.Now()
+		entry.isNegative = isNegative
+		c.lruList.MoveToFront(entry.lruElement)
+		if isNegative {
+			glog.V(4).Infof("📦 CACHE UPDATE (NEGATIVE): key=%s - data not found", key)
+		} else {
+			glog.V(4).Infof("📦 CACHE UPDATE: key=%s offset=%d size=%d", key, offset, len(data))
+		}
+		return
+	}
+
+	// Evict oldest entry if cache is full
+	if c.lruList.Len() >= c.maxSize {
+		oldest := c.lruList.Back()
+		if oldest != nil {
+			c.evict(oldest.Value.(*cacheEntry))
+		}
+	}
+
+	// Add new entry
+	entry := &cacheEntry{
+		key:        key,
+		data:       data,
+		offset:     offset,
+		timestamp:  time.Now(),
+		isNegative: isNegative,
+	}
+	entry.lruElement = c.lruList.PushFront(entry)
+	c.cache[key] = entry
+
+	if isNegative {
+		glog.V(4).Infof("📦 CACHE PUT (NEGATIVE): key=%s - data not found (cache_size=%d/%d)",
+			key, c.lruList.Len(), c.maxSize)
+	} else {
+		glog.V(4).Infof("📦 CACHE PUT: key=%s offset=%d size=%d (cache_size=%d/%d)",
+			key, offset, len(data), c.lruList.Len(), c.maxSize)
+	}
+}
+
+// evict removes an entry from the cache (must be called with lock held)
+func (c *DiskBufferCache) evict(entry *cacheEntry) {
+	delete(c.cache, entry.key)
+	c.lruList.Remove(entry.lruElement)
+	c.evictions++
+	glog.V(4).Infof("📦 CACHE EVICT: key=%s (evictions=%d)", entry.key, c.evictions)
+}
+
+// cleanupLoop periodically removes expired entries
+func (c *DiskBufferCache) cleanupLoop() {
+	ticker := time.NewTicker(c.ttl / 2)
+	defer ticker.Stop()
+
+	for range ticker.C {
+		c.cleanup()
+	}
+}
+
+// cleanup removes expired entries
+func (c *DiskBufferCache) cleanup() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	now := time.Now()
+	var toEvict []*cacheEntry
+
+	// Find expired entries
+	for _, entry := range c.cache {
+		if now.Sub(entry.timestamp) > c.ttl {
+			toEvict = append(toEvict, entry)
+		}
+	}
+
+	// Evict expired entries
+	for _, entry := range toEvict {
+		c.evict(entry)
+	}
+
+	if len(toEvict) > 0 {
+		glog.V(3).Infof("📦 CACHE CLEANUP: evicted %d expired entries", len(toEvict))
+	}
+}
+
+// Stats returns cache statistics
+func (c *DiskBufferCache) Stats() (hits, misses, evictions int64, size int) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.hits, c.misses, c.evictions, c.lruList.Len()
+}
+
+// Clear removes all entries from the cache
+func (c *DiskBufferCache) Clear() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.cache = make(map[string]*cacheEntry)
+	c.lruList = list.New()
+	glog.V(2).Infof("📦 CACHE CLEARED")
+}

weed/util/log_buffer/flush_budget_test.go

@@ -0,0 +1,273 @@
+package log_buffer
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util/mem"
+)
+
+func TestFlushBudgetBoundsQueuedBytes(t *testing.T) {
+	b := newFlushBudget(100)
+
+	if got := b.reserve(0, 60); got != 60 {
+		t.Fatalf("first reserve returned %d, want 60", got)
+	}
+
+	// 60 + 60 is over the limit, so the second producer has to wait.
+	reserved := make(chan int)
+	go func() { reserved <- b.reserve(1, 60) }()
+
+	select {
+	case <-reserved:
+		t.Fatal("second reserve went through while the budget was full")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	b.release(60)
+	select {
+	case got := <-reserved:
+		if got != 60 {
+			t.Errorf("second reserve returned %d, want 60", got)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("second reserve never woke up after the release")
+	}
+}
+
+// A window bigger than the whole budget still has to get through, or an
+// oversized entry would wedge the flush loop instead of merely spiking it.
+func TestFlushBudgetAdmitsOversizedWindow(t *testing.T) {
+	b := newFlushBudget(100)
+
+	if got := b.reserve(0, 50); got != 50 {
+		t.Fatalf("reserve returned %d, want 50", got)
+	}
+
+	reserved := make(chan int)
+	go func() { reserved <- b.reserve(1, 500) }()
+
+	select {
+	case <-reserved:
+		t.Fatal("oversized reserve went through before the queue drained")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	b.release(50)
+	select {
+	case got := <-reserved:
+		if got != 100 {
+			t.Errorf("oversized reserve booked %d, want the whole %d budget", got, 100)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("oversized reserve never got through on an empty queue")
+	}
+}
+
+// Windows have to reach the flush loop in the order they were sealed. A
+// producer can park here for seconds, so letting a later window overtake an
+// earlier one would persist them out of order and walk the flushed watermarks
+// backwards. Sized so only one window fits at a time, which is what makes the
+// admission order observable rather than a race between the woken goroutines.
+func TestFlushBudgetAdmitsInSealOrder(t *testing.T) {
+	const windows = 5
+	b := newFlushBudget(100)
+	b.reserve(0, 100) // fills the budget, so every window below has to wait
+
+	// Park them in reverse seal order, so arrival order cannot be what produces
+	// the right answer. Each window needs the whole budget, so exactly one is
+	// admitted per release and the order is observable.
+	admitted := make(chan uint64, windows)
+	for seq := uint64(windows); seq >= 1; seq-- {
+		go func(seq uint64) {
+			b.reserve(seq, 100)
+			admitted <- seq
+		}(seq)
+		time.Sleep(20 * time.Millisecond) // let it reach the wait
+	}
+
+	for want := uint64(1); want <= windows; want++ {
+		select {
+		case seq := <-admitted:
+			t.Fatalf("window %d was admitted while the budget was still held", seq)
+		case <-time.After(20 * time.Millisecond):
+		}
+
+		b.release(100)
+		select {
+		case got := <-admitted:
+			if got != want {
+				t.Fatalf("admitted window %d, want %d", got, want)
+			}
+		case <-time.After(2 * time.Second):
+			t.Fatalf("window %d never admitted", want)
+		}
+	}
+}
+
+// Shutdown must not be held up by a producer parked on the budget.
+func TestFlushBudgetCloseReleasesWaiters(t *testing.T) {
+	b := newFlushBudget(100)
+	b.reserve(0, 100)
+
+	done := make(chan struct{})
+	go func() {
+		b.reserve(1, 100)
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		t.Fatal("reserve went through while the budget was full")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	b.close()
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("close did not release the parked producer")
+	}
+}
+
+// What the queue actually holds is the pooled slab, which mem.Allocate rounds
+// up to a size class. Charging the window length instead would let the queue
+// retain roughly twice the ceiling.
+func TestQueueFlushChargesTheSlabNotTheWindow(t *testing.T) {
+	stall := make(chan struct{})
+	lb := NewLogBuffer("slab", time.Hour, func(_ *LogBuffer, _, _ time.Time, _ []byte, _, _ int64) {
+		<-stall
+	}, nil, func() {})
+	defer func() { close(stall); lb.ShutdownLogBuffer() }()
+
+	// 5 MiB rounds up to an 8 MiB slot.
+	const windowSize = 5 << 20
+	data := mem.Allocate(windowSize)
+	if cap(data) == len(data) {
+		t.Skipf("allocator returned an exact fit (%d bytes), nothing to distinguish", cap(data))
+	}
+
+	if !lb.queueFlush(&dataToFlush{data: data}) {
+		t.Fatal("queueFlush dropped the window")
+	}
+
+	lb.flushBudget.mu.Lock()
+	queued := lb.flushBudget.queued
+	lb.flushBudget.mu.Unlock()
+
+	if queued != cap(data) {
+		t.Errorf("charged %d bytes for a window holding a %d byte slab (len %d)", queued, cap(data), len(data))
+	}
+}
+
+// The window is already sealed by the time queueFlush runs, so dropping it
+// loses records the caller was told were accepted. A shutdown racing the
+// hand-off must not cost data while the queue still has room for it.
+func TestQueueFlushKeepsSealedWindowWhenQueueHasRoom(t *testing.T) {
+	stall := make(chan struct{})
+	var flushed atomic.Int64
+	lb := NewLogBuffer("shutdown-race", time.Hour, func(_ *LogBuffer, _, _ time.Time, _ []byte, _, _ int64) {
+		<-stall
+		flushed.Add(1)
+	}, nil, func() {})
+	defer func() { close(stall) }()
+
+	// Simulate a shutdown landing between the seal and the hand-off.
+	close(lb.shutdownCh)
+	lb.isStopping.Store(true)
+
+	// Stay inside the channel's capacity so room is guaranteed for every one of
+	// them; loopFlush is parked in the stalled flushFn and drains nothing.
+	for i := 0; i < flushQueueDepth; i++ {
+		if !lb.queueFlush(&dataToFlush{data: mem.Allocate(1024), seq: uint64(i)}) {
+			t.Fatalf("window %d was dropped even though the queue had room", i)
+		}
+	}
+}
+
+// The point of the budget: a stalled flush must stop producers rather than let
+// them keep handing over copies. Asserted on how many producers get through,
+// which is independent of the counter reserve maintains.
+func TestStalledFlushBlocksOversizedProducers(t *testing.T) {
+	stall := make(chan struct{})
+	var completed atomic.Int64
+
+	lb := NewLogBuffer("stalled", time.Hour, func(_ *LogBuffer, _, _ time.Time, _ []byte, _, _ int64) {
+		<-stall
+	}, nil, func() {})
+
+	// Just over BufferSize is enough to get a window per entry; the package
+	// leaves plenty of other LogBuffers alive, so keep the footprint small
+	// enough for a 32-bit runner.
+	entry := make([]byte, BufferSize+1)
+	const producers = flushQueueDepth
+
+	var wg sync.WaitGroup
+	for i := 0; i < producers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := lb.AddDataToBuffer(nil, entry, 0); err == nil {
+				completed.Add(1)
+			}
+		}()
+	}
+
+	time.Sleep(2 * time.Second)
+
+	got := completed.Load()
+	// Some have to get through -- a budget that admits nobody is a deadlock,
+	// not a bound -- and the rest have to be parked.
+	if got == 0 {
+		t.Error("no producer got through a stalled flush; the budget deadlocked")
+	}
+	if got >= producers {
+		t.Errorf("all %d producers got through a stalled flush; the budget did not bind", producers)
+	}
+
+	close(stall)
+	lb.ShutdownLogBuffer()
+	wg.Wait()
+}
+
+// ...and once the flush drains, everyone gets through: the bound must not
+// deadlock the producers it parks.
+func TestBlockedProducersDrainOnceFlushResumes(t *testing.T) {
+	stall := make(chan struct{})
+	var completed atomic.Int64
+
+	lb := NewLogBuffer("drain", time.Hour, func(_ *LogBuffer, _, _ time.Time, _ []byte, _, _ int64) {
+		<-stall
+	}, nil, func() {})
+
+	entry := make([]byte, BufferSize+1)
+	const producers = 6
+
+	var wg sync.WaitGroup
+	for i := 0; i < producers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := lb.AddDataToBuffer(nil, entry, 0); err == nil {
+				completed.Add(1)
+			}
+		}()
+	}
+
+	time.Sleep(500 * time.Millisecond)
+	close(stall)
+
+	done := make(chan struct{})
+	go func() { wg.Wait(); close(done) }()
+	select {
+	case <-done:
+	case <-time.After(30 * time.Second):
+		t.Fatal("producers never drained after the flush resumed")
+	}
+	if got := completed.Load(); got != producers {
+		t.Errorf("%d of %d producers completed", got, producers)
+	}
+	lb.ShutdownLogBuffer()
+}

weed/util/log_buffer/log_buffer_corruption_test.go

@@ -0,0 +1,224 @@
+package log_buffer
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// TestReadTsCorruptedBuffer tests that readTs properly returns an error for corrupted data
+func TestReadTsCorruptedBuffer(t *testing.T) {
+	// Create a corrupted buffer with invalid protobuf data
+	buf := make([]byte, 100)
+
+	// Set size field to 10 bytes (using proper encoding)
+	util.Uint32toBytes(buf[0:4], 10)
+
+	// Fill with garbage data that won't unmarshal as LogEntry
+	for i := 4; i < 14; i++ {
+		buf[i] = 0xFF
+	}
+
+	// Attempt to read timestamp
+	size, ts, err := readTs(buf, 0)
+
+	// Should return an error
+	if err == nil {
+		t.Error("Expected error for corrupted buffer, got nil")
+	}
+
+	// Size and ts should be zero on error
+	if size != 0 {
+		t.Errorf("Expected size=0 on error, got %d", size)
+	}
+
+	if ts != 0 {
+		t.Errorf("Expected ts=0 on error, got %d", ts)
+	}
+
+	// Error should indicate corruption
+	if !errors.Is(err, ErrBufferCorrupted) {
+		t.Logf("Error message: %v", err)
+		// Check if error message contains expected text
+		if err.Error() == "" || len(err.Error()) == 0 {
+			t.Error("Expected non-empty error message")
+		}
+	}
+
+	t.Logf("✓ readTs correctly returned error for corrupted buffer: %v", err)
+}
+
+// TestReadTsValidBuffer tests that readTs works correctly for valid data
+func TestReadTsValidBuffer(t *testing.T) {
+	// Create a valid LogEntry
+	logEntry := &filer_pb.LogEntry{
+		TsNs: 123456789,
+		Key:  []byte("test-key"),
+	}
+
+	// Marshal it
+	data, err := proto.Marshal(logEntry)
+	if err != nil {
+		t.Fatalf("Failed to marshal LogEntry: %v", err)
+	}
+
+	// Create buffer with size prefix using util function
+	buf := make([]byte, 4+len(data))
+	util.Uint32toBytes(buf[0:4], uint32(len(data)))
+	copy(buf[4:], data)
+
+	// Read timestamp
+	size, ts, err := readTs(buf, 0)
+
+	// Should succeed
+	if err != nil {
+		t.Fatalf("Expected no error for valid buffer, got: %v", err)
+	}
+
+	// Should return correct values
+	if size != len(data) {
+		t.Errorf("Expected size=%d, got %d", len(data), size)
+	}
+
+	if ts != logEntry.TsNs {
+		t.Errorf("Expected ts=%d, got %d", logEntry.TsNs, ts)
+	}
+
+	t.Logf("✓ readTs correctly parsed valid buffer: size=%d, ts=%d", size, ts)
+}
+
+// TestReadFromBufferCorruption tests that ReadFromBuffer propagates corruption errors
+func TestReadFromBufferCorruption(t *testing.T) {
+	lb := NewLogBuffer("test-corruption", time.Second, nil, nil, func() {})
+
+	// Add a valid entry first using AddDataToBuffer
+	validKey := []byte("valid")
+	validData, _ := proto.Marshal(&filer_pb.LogEntry{
+		TsNs: 1000,
+		Key:  validKey,
+	})
+	if err := lb.AddDataToBuffer(validKey, validData, 1000); err != nil {
+		t.Fatalf("Failed to add data to buffer: %v", err)
+	}
+
+	// Manually corrupt the buffer by writing garbage
+	// This simulates a corruption scenario
+	if len(lb.idx) > 0 {
+		pos := lb.idx[0]
+		// Overwrite the protobuf data with garbage
+		for i := pos + 4; i < pos+8 && i < len(lb.buf); i++ {
+			lb.buf[i] = 0xFF
+		}
+	}
+
+	// Try to read - should detect corruption
+	startPos := MessagePosition{Time: lb.startTime}
+	buf, offset, _, err := lb.ReadFromBuffer(startPos)
+
+	// Should return corruption error
+	if err == nil {
+		t.Error("Expected corruption error, got nil")
+		if buf != nil {
+			t.Logf("Unexpected success: got buffer with %d bytes", buf.Len())
+		}
+	} else {
+		// Verify it's a corruption error
+		if !errors.Is(err, ErrBufferCorrupted) {
+			t.Logf("Got error (not ErrBufferCorrupted sentinel, but still an error): %v", err)
+		}
+		t.Logf("✓ ReadFromBuffer correctly detected corruption: %v", err)
+	}
+
+	t.Logf("ReadFromBuffer result: buf=%v, offset=%d, err=%v", buf != nil, offset, err)
+}
+
+// TestLocateByTsCorruption tests that locateByTs propagates corruption errors
+func TestLocateByTsCorruption(t *testing.T) {
+	// Create a MemBuffer with corrupted data
+	mb := &MemBuffer{
+		buf:  make([]byte, 100),
+		size: 14,
+	}
+
+	// Set size field (using proper encoding)
+	util.Uint32toBytes(mb.buf[0:4], 10)
+
+	// Fill with garbage
+	for i := 4; i < 14; i++ {
+		mb.buf[i] = 0xFF
+	}
+
+	// Try to locate by timestamp
+	pos, err := mb.locateByTs(mb.startTime)
+
+	// Should return error
+	if err == nil {
+		t.Errorf("Expected corruption error, got nil (pos=%d)", pos)
+	} else {
+		t.Logf("✓ locateByTs correctly detected corruption: %v", err)
+	}
+}
+
+// TestErrorPropagationChain tests the complete error propagation from readTs -> locateByTs -> ReadFromBuffer
+func TestErrorPropagationChain(t *testing.T) {
+	t.Run("Corruption in readTs", func(t *testing.T) {
+		// Already covered by TestReadTsCorruptedBuffer
+		t.Log("✓ readTs error propagation tested")
+	})
+
+	t.Run("Corruption in locateByTs", func(t *testing.T) {
+		// Already covered by TestLocateByTsCorruption
+		t.Log("✓ locateByTs error propagation tested")
+	})
+
+	t.Run("Corruption in ReadFromBuffer binary search", func(t *testing.T) {
+		// Already covered by TestReadFromBufferCorruption
+		t.Log("✓ ReadFromBuffer error propagation tested")
+	})
+
+	t.Log("✓ Complete error propagation chain verified")
+}
+
+// TestNoSilentCorruption verifies that corruption never returns (0, 0) silently
+func TestNoSilentCorruption(t *testing.T) {
+	// Create various corrupted buffers
+	testCases := []struct {
+		name string
+		buf  []byte
+		pos  int
+	}{
+		{
+			name: "Invalid protobuf",
+			buf:  []byte{10, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
+			pos:  0,
+		},
+		{
+			name: "Truncated data",
+			buf:  []byte{100, 0, 0, 0, 1, 2, 3}, // Size says 100 but only 3 bytes available
+			pos:  0,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			size, ts, err := readTs(tc.buf, tc.pos)
+
+			// CRITICAL: Must return error, never silent (0, 0)
+			if err == nil {
+				t.Errorf("CRITICAL: readTs returned (%d, %d, nil) for corrupted buffer - this causes silent data corruption!", size, ts)
+			} else {
+				t.Logf("✓ Correctly returned error instead of silent (0, 0): %v", err)
+			}
+
+			// On error, size and ts should be 0
+			if size != 0 || ts != 0 {
+				t.Errorf("On error, expected (0, 0), got (%d, %d)", size, ts)
+			}
+		})
+	}
+}

weed/util/log_buffer/log_buffer_eviction_gate_test.go

@@ -0,0 +1,390 @@
+package log_buffer
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+// TestEvictionWatermarkTracksSealedRing drives the watermark through the real
+// eviction path instead of writing the field: only copyToFlushInternal sees the
+// window about to be dropped, and it reads it one statement before SealBuffer
+// shifts it out of slot 0. The buffer here never flushes, matching the
+// aggregated meta ring, so the watermark is its only emptiness proof.
+func TestEvictionWatermarkTracksSealedRing(t *testing.T) {
+	lb := NewLogBuffer("evict-watermark", time.Minute, nil, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	// Each timestamp is further past the previous window's start than the flush
+	// interval, so every append seals the window before it.
+	base := time.Now().Add(-30 * time.Minute).Truncate(time.Second)
+	step := 2 * time.Minute
+	at := func(i int) time.Time { return base.Add(time.Duration(i) * step) }
+
+	if got := lb.GetLastEvictedTsNs(); got != 0 {
+		t.Fatalf("fresh buffer evicted through %v, want 0", time.Unix(0, got))
+	}
+
+	// PreviousBufferCount seals only fill the ring; the next one drops window 0.
+	for i := 0; i <= PreviousBufferCount; i++ {
+		if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: at(i).UnixNano(), Data: []byte("x"), Key: []byte("k")}); err != nil {
+			t.Fatalf("add %d: %v", i, err)
+		}
+		if got := lb.GetLastEvictedTsNs(); got != 0 {
+			t.Fatalf("after %d appends evicted through %v, want nothing evicted yet", i+1, time.Unix(0, got))
+		}
+	}
+
+	if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: at(PreviousBufferCount + 1).UnixNano(), Data: []byte("x"), Key: []byte("k")}); err != nil {
+		t.Fatalf("add evicting entry: %v", err)
+	}
+	// Window 0 held only the first entry, so its stopTime is that entry's ts.
+	if got, want := lb.GetLastEvictedTsNs(), at(0).UnixNano(); got != want {
+		t.Fatalf("evicted through %v, want the dropped window's stop %v", time.Unix(0, got), time.Unix(0, want))
+	}
+
+	// One more eviction advances the watermark; it never regresses.
+	if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: at(PreviousBufferCount + 2).UnixNano(), Data: []byte("x"), Key: []byte("k")}); err != nil {
+		t.Fatalf("add second evicting entry: %v", err)
+	}
+	if got, want := lb.GetLastEvictedTsNs(), at(1).UnixNano(); got != want {
+		t.Fatalf("evicted through %v, want %v", time.Unix(0, got), time.Unix(0, want))
+	}
+}
+
+// TestGapResumeCursorReadsSingleEntrySealedWindow pins the resume cursor
+// against the shape that actually breaks: a sealed window holding one entry,
+// where startTime == stopTime. The sealed-buffer lookup only enters a window
+// whose stopTime is strictly after the cursor, so a cursor sitting exactly on
+// earliest walks straight past such a window and its sole event is never
+// delivered. Low-volume metadata windows are routinely one entry.
+func TestGapResumeCursorReadsSingleEntrySealedWindow(t *testing.T) {
+	lb := NewLogBuffer("gap-resume-sealed", time.Minute, nil, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	// Timestamps far enough apart that every append seals the window before it,
+	// so each sealed window holds exactly one entry.
+	base := time.Now().Add(-time.Hour).Truncate(time.Second)
+	for i := 0; i < 3; i++ {
+		if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{
+			TsNs: base.Add(time.Duration(i) * 2 * time.Minute).UnixNano(), Data: []byte("x"), Key: []byte("k"),
+		}); err != nil {
+			t.Fatalf("add %d: %v", i, err)
+		}
+	}
+	earliest := lb.GetEarliestTime()
+	if earliest.IsZero() {
+		t.Fatal("expected in-memory data")
+	}
+
+	// firstTsFrom reports the timestamp of the first entry a read hands back.
+	firstTsFrom := func(cursorTsNs int64) int64 {
+		buf, _, pooled, err := lb.ReadFromBuffer(NewMessagePosition(cursorTsNs, gapResumeTestOffset))
+		if err != nil {
+			t.Fatalf("read at %v: %v", time.Unix(0, cursorTsNs), err)
+		}
+		if buf == nil {
+			t.Fatalf("read at %v returned no data", time.Unix(0, cursorTsNs))
+		}
+		if pooled {
+			defer lb.ReleaseMemory(buf)
+		}
+		_, ts, readErr := readTs(buf.Bytes(), 0)
+		if readErr != nil {
+			t.Fatalf("decode first entry: %v", readErr)
+		}
+		return ts
+	}
+
+	// The resume the resolvers issue must deliver the earliest entry itself.
+	if got := firstTsFrom(earliest.UnixNano() - 1); got != earliest.UnixNano() {
+		t.Fatalf("resume just below earliest starts at %v, want the earliest entry %v",
+			time.Unix(0, got), earliest)
+	}
+	// Resuming exactly on earliest is the shape that loses it.
+	if got := firstTsFrom(earliest.UnixNano()); got == earliest.UnixNano() {
+		t.Fatal("expected a cursor exactly on earliest to skip the single-entry window (precondition)")
+	}
+}
+
+// gapResumeTestOffset mirrors the sentinel the filer's gap resume carries.
+const gapResumeTestOffset = -2
+
+// TestGapResumeCursorReadsFromMemory pins the contract the filer's gap resolver
+// depends on: the cursor it resumes with must be one this read actually serves.
+// ReadFromBuffer only falls through to memory for a cursor below the in-memory
+// window when the offset is a sentinel, and answers ResumeFromDiskError for
+// every positive one -- a resume carrying a positive offset would bounce
+// straight back to the resolver, which sees no progress and parks a subscriber
+// whose data is sitting in the ring.
+func TestGapResumeCursorReadsFromMemory(t *testing.T) {
+	lb := NewLogBuffer("gap-resume", time.Minute, nil, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	base := time.Now().Add(-time.Hour).Truncate(time.Second)
+	for i := 0; i < 3; i++ {
+		if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{
+			TsNs: base.Add(time.Duration(i) * time.Second).UnixNano(), Data: []byte("x"), Key: []byte("k"),
+		}); err != nil {
+			t.Fatalf("add %d: %v", i, err)
+		}
+	}
+	earliest := lb.GetEarliestTime()
+	if earliest.IsZero() {
+		t.Fatal("expected in-memory data")
+	}
+
+	// The resolver resumes at earliest itself, read inclusively.
+	if buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(earliest.UnixNano(), -2)); err != nil || buf == nil {
+		t.Fatalf("resume at earliest: buf=%v err=%v", buf != nil, err)
+	}
+	// A sentinel just below it is served too, so the exact boundary is not load-bearing.
+	if buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(earliest.UnixNano()-1, -2)); err != nil || buf == nil {
+		t.Fatalf("resume just below earliest: buf=%v err=%v", buf != nil, err)
+	}
+	// A positive offset below the window is refused: this is the shape a gap
+	// resume must never take.
+	if _, _, _, err := lb.ReadFromBuffer(NewMessagePosition(earliest.UnixNano()-1, 1)); err != ResumeFromDiskError {
+		t.Fatalf("positive-offset cursor below the window: want ResumeFromDiskError, got %v", err)
+	}
+}
+
+// TestFlushSubscriberContract pins what the filer's gap parks rest on: a flush
+// subscriber is woken when a flush lands and the flush watermark is already
+// visible at that moment - loopFlush stores lastFlushTsNs before notifying, so
+// a waiter that re-checks the watermark on wake-up cannot miss the flush that
+// woke it. Appends alone never signal this channel, and unregistering closes
+// it so an abandoned waiter is not stranded.
+func TestFlushSubscriberContract(t *testing.T) {
+	flushed := make(chan struct{}, 16)
+	// A flush interval far longer than the test, so appends never auto-seal a
+	// window and ForceFlush is the only source of flushes: each round then has
+	// exactly one flush, and the token received is known to belong to it.
+	lb := NewLogBuffer("flush-sub", time.Hour,
+		func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+			flushed <- struct{}{}
+		}, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	ch := lb.RegisterFlushSubscriber("s")
+
+	// An append is not a flush: nothing may arrive on the channel. The probe
+	// sits just below the first round timestamp: below, so no round entry gets
+	// collision-bumped and its stored stopTime stays comparable to the local
+	// value each round asserts against; just below, because a gap wider than
+	// the flush interval would make round 0's own append auto-seal the probe
+	// window - a second flush whose token throws every round off by one.
+	if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: time.Now().Add(-time.Hour - time.Millisecond).UnixNano(), Data: []byte("x"), Key: []byte("k")}); err != nil {
+		t.Fatalf("add: %v", err)
+	}
+	select {
+	case <-ch:
+		t.Fatal("an append must not wake a flush subscriber")
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	// A flush wakes it with the watermark already stored - the contract the gap
+	// parks re-check on. The store-before-notify ordering itself is pinned by a
+	// load-bearing comment at the loopFlush site (a reorder's window is
+	// nanoseconds on that goroutine, untestable from here); these round-trips
+	// verify the observable contract and catch a notify with no store at all.
+	base := time.Now().Add(-time.Hour)
+	for i := 0; i < 8; i++ {
+		ts := base.Add(time.Duration(i) * time.Millisecond).UnixNano()
+		// The round entry must be above the buffer head: a bumped timestamp
+		// flushes a stopTime far above the local ts compared below, making the
+		// round's assertion vacuously true.
+		if last := lb.LastTsNs.Load(); last >= ts {
+			t.Fatalf("round %d: ts not above buffer head; the assertion would be vacuous", i)
+		}
+		if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: ts, Data: []byte("x"), Key: []byte("k")}); err != nil {
+			t.Fatalf("add round %d: %v", i, err)
+		}
+		go lb.ForceFlush()
+		select {
+		case <-ch:
+			if got := lb.GetLastFlushTsNs(); got < ts {
+				t.Fatalf("round %d: woken with watermark %v short of the flushed entry %v; a waiter re-checking it on wake-up misses the flush that woke it",
+					i, time.Unix(0, got), time.Unix(0, ts))
+			}
+		case <-time.After(5 * time.Second):
+			t.Fatalf("round %d: a flush must wake the flush subscriber", i)
+		}
+		select {
+		case <-flushed:
+		case <-time.After(5 * time.Second):
+			t.Fatalf("round %d: flushFn did not run", i)
+		}
+	}
+
+	// Unregistering closes the channel so an abandoned waiter unblocks.
+	lb.UnregisterFlushSubscriber("s")
+	select {
+	case _, ok := <-ch:
+		if ok {
+			t.Fatal("unregister must close the channel, not send on it")
+		}
+	case <-time.After(time.Second):
+		t.Fatal("unregister must close the channel")
+	}
+
+	// Unknown ids and double unregisters are harmless.
+	lb.UnregisterFlushSubscriber("s")
+	lb.UnregisterFlushSubscriber("never-registered")
+}
+
+// TestEvictionGatedCursor pins the gated sentinel: below the eviction watermark
+// it is refused to disk under the read's own lock - the only place the check is
+// atomic with the serve decision - while the plain -2 sentinel keeps master's
+// serve-from-earliest behavior for the message queue's readers.
+func TestEvictionGatedCursor(t *testing.T) {
+	lb := NewLogBuffer("gated-cursor", time.Minute, nil, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	// Entries far enough apart that every append seals the window before it;
+	// wrapping the ring moves the watermark for real.
+	base := time.Now().Add(-time.Hour).Truncate(time.Second)
+	for i := 0; i < PreviousBufferCount+2; i++ {
+		if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{
+			TsNs: base.Add(time.Duration(i) * 2 * time.Minute).UnixNano(), Data: []byte("x"), Key: []byte("k"),
+		}); err != nil {
+			t.Fatalf("add %d: %v", i, err)
+		}
+	}
+	evicted := lb.GetLastEvictedTsNs()
+	if evicted == 0 {
+		t.Fatal("precondition: the ring evicted nothing")
+	}
+
+	// Below the watermark: gated goes to disk, -2 keeps serving (MQ contract).
+	if _, _, _, err := lb.ReadFromBuffer(NewMessagePosition(evicted-1, EvictionGatedOffset)); err != ResumeFromDiskError {
+		t.Fatalf("gated below watermark: want ResumeFromDiskError, got %v", err)
+	}
+	if buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(evicted-1, -2)); err != nil || buf == nil {
+		t.Fatalf("plain sentinel below watermark: buf=%v err=%v, master behavior must hold", buf != nil, err)
+	}
+	// At and above the watermark the gated cursor serves normally.
+	if buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(evicted, EvictionGatedOffset)); err != nil || buf == nil {
+		t.Fatalf("gated at watermark: buf=%v err=%v", buf != nil, err)
+	}
+	if buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(evicted+1, EvictionGatedOffset)); err != nil || buf == nil {
+		t.Fatalf("gated above watermark: buf=%v err=%v", buf != nil, err)
+	}
+}
+
+// TestMarkEvictedThroughGatesYoungRing pins the young-ring gate: with the mark
+// set, a below-window gated cursor goes to disk before the first real eviction
+// instead of being served the earliest retained entry.
+func TestMarkEvictedThroughGatesYoungRing(t *testing.T) {
+	lb := NewLogBuffer("young-ring", time.Minute, nil, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	seed := time.Now().Add(-time.Minute).UnixNano()
+	lb.MarkEvictedThrough(seed)
+	if got := lb.GetLastEvictedTsNs(); got != seed {
+		t.Fatalf("evicted through %v, want the mark %v", time.Unix(0, got), time.Unix(0, seed))
+	}
+	if got := lb.GetLastEvictedOriginalTsNs(); got != seed {
+		t.Fatalf("original watermark %v, want the mark %v", time.Unix(0, got), time.Unix(0, seed))
+	}
+	// A lower mark must not regress an established boundary.
+	lb.MarkEvictedThrough(seed - int64(time.Hour))
+	if got := lb.GetLastEvictedTsNs(); got != seed {
+		t.Fatalf("lower mark regressed the watermark to %v", time.Unix(0, got))
+	}
+
+	// One post-mark arrival: the ring's window starts well above the mark.
+	if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{
+		TsNs: time.Now().UnixNano(), Data: []byte("x"), Key: []byte("k"),
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Below the mark: gated goes to disk, -2 keeps serving (MQ contract).
+	if _, _, _, err := lb.ReadFromBuffer(NewMessagePosition(seed-1, EvictionGatedOffset)); err != ResumeFromDiskError {
+		t.Fatalf("gated below the mark: want ResumeFromDiskError, got %v", err)
+	}
+	if buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(seed-1, -2)); err != nil || buf == nil {
+		t.Fatalf("plain sentinel below the mark: buf=%v err=%v", buf != nil, err)
+	}
+	if buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(seed, EvictionGatedOffset)); err != nil || buf == nil {
+		t.Fatalf("gated at the mark: buf=%v err=%v", buf != nil, err)
+	}
+}
+
+// TestEvictionOriginalWatermark pins the second timestamp space. The ring bumps
+// an out-of-order arrival past its head, so a bump-heavy interval (peer history
+// replay) leaves stopTimes above anything on any peer's disk; a gap gate
+// comparing disk cursors against those would park a subscriber that drained
+// every peer's log. The original watermark tracks what was actually received.
+func TestEvictionOriginalWatermark(t *testing.T) {
+	lb := NewLogBuffer("orig-watermark", time.Minute, nil, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	// Newest original first: it sets the ring head, so every later arrival is
+	// bumped. Seals go by bumped time, which advances by nanoseconds, so each
+	// window is sealed explicitly; wrapping the ring evicts.
+	newest := time.Now().Add(-time.Hour).Truncate(time.Second).UnixNano()
+	for i := 0; i < PreviousBufferCount+2; i++ {
+		orig := newest - int64(i)*int64(time.Minute)
+		if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: orig, Data: []byte("x"), Key: []byte("k")}); err != nil {
+			t.Fatalf("add %d: %v", i, err)
+		}
+		lb.ForceFlush()
+	}
+
+	bumped := lb.GetLastEvictedTsNs()
+	original := lb.GetLastEvictedOriginalTsNs()
+	if bumped == 0 || original == 0 {
+		t.Fatalf("precondition: nothing evicted (bumped=%d original=%d)", bumped, original)
+	}
+	if original > newest {
+		t.Fatalf("original watermark %v exceeds the highest received timestamp %v", time.Unix(0, original), time.Unix(0, newest))
+	}
+	if bumped <= newest {
+		t.Fatalf("bumped watermark %v not past the ring head %v: the spaces did not diverge", time.Unix(0, bumped), time.Unix(0, newest))
+	}
+	// The punchline: a disk cursor that drained every peer's log sits at the
+	// highest received timestamp - clearing the original watermark while still
+	// below the bumped one, which no disk timestamp can ever reach.
+	diskHead := newest
+	if diskHead < original {
+		t.Fatalf("disk head %v short of the original watermark %v", time.Unix(0, diskHead), time.Unix(0, original))
+	}
+	if diskHead >= bumped {
+		t.Fatal("disk head reached the bumped watermark; the gate would not have parked and this test proves nothing")
+	}
+}
+
+// TestOriginalWatermarkWindowAttribution pins which window an entry's received
+// timestamp is credited to. An append that rolls the window over seals the
+// previous one first; crediting the incoming timestamp before that hands it to
+// the sealed window (inflating its watermark: spurious parks) and loses it
+// from the new one (deflating: gaps proven empty that are not).
+func TestOriginalWatermarkWindowAttribution(t *testing.T) {
+	lb := NewLogBuffer("orig-attribution", time.Minute, nil, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	base := time.Now().Add(-time.Hour).Truncate(time.Second)
+	t1 := base.UnixNano()
+	t2 := base.Add(2 * time.Minute).UnixNano() // past the flush interval: seals window A
+
+	if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: t1, Data: []byte("x"), Key: []byte("k")}); err != nil {
+		t.Fatalf("add t1: %v", err)
+	}
+	if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: t2, Data: []byte("x"), Key: []byte("k")}); err != nil {
+		t.Fatalf("add t2: %v", err)
+	}
+
+	sealed := lb.prevBuffers.buffers[len(lb.prevBuffers.buffers)-1]
+	if sealed.size == 0 {
+		t.Fatal("precondition: the second append did not seal the first window")
+	}
+	if got := sealed.maxOriginalTsNs; got != t1 {
+		t.Fatalf("sealed window credited with %v, want its own entry %v (t2 belongs to the open window)", time.Unix(0, got), time.Unix(0, t1))
+	}
+	if got := lb.curWindowMaxOriginalTsNs; got != t2 {
+		t.Fatalf("open window holds %v, want the entry that rolled it over %v", time.Unix(0, got), time.Unix(0, t2))
+	}
+}

weed/util/log_buffer/log_buffer_flush_gap_test.go

@@ -0,0 +1,697 @@
+package log_buffer
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"google.golang.org/protobuf/proto"
+)
+
+// TestFlushOffsetGap_ReproduceDataLoss reproduces the critical bug where messages
+// are lost in the gap between flushed disk data and in-memory buffer.
+//
+// OBSERVED BEHAVIOR FROM LOGS:
+//
+//	Request offset: 1764
+//	Disk contains: 1000-1763 (764 messages)
+//	Memory buffer starts at: 1800
+//	Gap: 1764-1799 (36 messages) ← MISSING!
+//
+// This test verifies:
+// 1. All messages sent to buffer are accounted for
+// 2. No gaps exist between disk and memory offsets
+// 3. Flushed data and in-memory data have continuous offset ranges
+func TestFlushOffsetGap_ReproduceDataLoss(t *testing.T) {
+	var flushedMessages []*filer_pb.LogEntry
+	var flushMu sync.Mutex
+
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+		t.Logf("FLUSH: minOffset=%d maxOffset=%d size=%d bytes", minOffset, maxOffset, len(buf))
+
+		// Parse and store flushed messages
+		flushMu.Lock()
+		defer flushMu.Unlock()
+
+		// Parse buffer to extract messages
+		parsedCount := 0
+		for pos := 0; pos+4 < len(buf); {
+			if pos+4 > len(buf) {
+				break
+			}
+
+			size := uint32(buf[pos])<<24 | uint32(buf[pos+1])<<16 | uint32(buf[pos+2])<<8 | uint32(buf[pos+3])
+			if pos+4+int(size) > len(buf) {
+				break
+			}
+
+			entryData := buf[pos+4 : pos+4+int(size)]
+			logEntry := &filer_pb.LogEntry{}
+			if err := proto.Unmarshal(entryData, logEntry); err == nil {
+				flushedMessages = append(flushedMessages, logEntry)
+				parsedCount++
+			}
+
+			pos += 4 + int(size)
+		}
+
+		t.Logf("  Parsed %d messages from flush buffer", parsedCount)
+	}
+
+	logBuffer := NewLogBuffer("test", 100*time.Millisecond, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Send 100 messages
+	messageCount := 100
+	t.Logf("Sending %d messages...", messageCount)
+
+	for i := 0; i < messageCount; i++ {
+		if err := logBuffer.AddToBuffer(&mq_pb.DataMessage{
+			Key:   []byte(fmt.Sprintf("key-%d", i)),
+			Value: []byte(fmt.Sprintf("message-%d", i)),
+			TsNs:  time.Now().UnixNano(),
+		}); err != nil {
+			t.Fatalf("Failed to add buffer: %v", err)
+		}
+	}
+
+	// Force flush multiple times to simulate real workload
+	t.Logf("Forcing flush...")
+	logBuffer.ForceFlush()
+
+	// Add more messages after flush
+	for i := messageCount; i < messageCount+50; i++ {
+		if err := logBuffer.AddToBuffer(&mq_pb.DataMessage{
+			Key:   []byte(fmt.Sprintf("key-%d", i)),
+			Value: []byte(fmt.Sprintf("message-%d", i)),
+			TsNs:  time.Now().UnixNano(),
+		}); err != nil {
+			t.Fatalf("Failed to add buffer: %v", err)
+		}
+	}
+
+	// Force another flush
+	logBuffer.ForceFlush()
+	time.Sleep(200 * time.Millisecond) // Wait for flush to complete
+
+	// Now check the buffer state
+	logBuffer.RLock()
+	bufferStartOffset := logBuffer.bufferStartOffset
+	currentOffset := logBuffer.offset
+	pos := logBuffer.pos
+	logBuffer.RUnlock()
+
+	flushMu.Lock()
+	flushedCount := len(flushedMessages)
+	var maxFlushedOffset int64 = -1
+	var minFlushedOffset int64 = -1
+	if flushedCount > 0 {
+		minFlushedOffset = flushedMessages[0].Offset
+		maxFlushedOffset = flushedMessages[flushedCount-1].Offset
+	}
+	flushMu.Unlock()
+
+	t.Logf("\nBUFFER STATE AFTER FLUSH:")
+	t.Logf("  bufferStartOffset: %d", bufferStartOffset)
+	t.Logf("  currentOffset (HWM): %d", currentOffset)
+	t.Logf("  pos (bytes in buffer): %d", pos)
+	t.Logf("  Messages sent: %d (offsets 0-%d)", messageCount+50, messageCount+49)
+	t.Logf("  Messages flushed to disk: %d (offsets %d-%d)", flushedCount, minFlushedOffset, maxFlushedOffset)
+
+	// CRITICAL CHECK: Is there a gap between flushed data and memory buffer?
+	if flushedCount > 0 && maxFlushedOffset >= 0 {
+		gap := bufferStartOffset - (maxFlushedOffset + 1)
+
+		t.Logf("\nOFFSET CONTINUITY CHECK:")
+		t.Logf("  Last flushed offset: %d", maxFlushedOffset)
+		t.Logf("  Buffer starts at: %d", bufferStartOffset)
+		t.Logf("  Gap: %d offsets", gap)
+
+		if gap > 0 {
+			t.Errorf("CRITICAL BUG REPRODUCED: OFFSET GAP DETECTED!")
+			t.Errorf("   Disk has offsets %d-%d", minFlushedOffset, maxFlushedOffset)
+			t.Errorf("   Memory buffer starts at: %d", bufferStartOffset)
+			t.Errorf("   MISSING OFFSETS: %d-%d (%d messages)", maxFlushedOffset+1, bufferStartOffset-1, gap)
+			t.Errorf("   These messages are LOST - neither on disk nor in memory!")
+		} else if gap < 0 {
+			t.Errorf("OFFSET OVERLAP: Memory buffer starts BEFORE last flushed offset!")
+			t.Errorf("   This indicates data corruption or race condition")
+		} else {
+			t.Logf("PASS: No gap detected - offsets are continuous")
+		}
+
+		// Check if we can read all expected offsets
+		t.Logf("\nREADABILITY CHECK:")
+		for testOffset := int64(0); testOffset < currentOffset; testOffset += 10 {
+			// Try to read from buffer
+			requestPosition := NewMessagePositionFromOffset(testOffset)
+			buf, _, _, err := logBuffer.ReadFromBuffer(requestPosition)
+
+			isReadable := (buf != nil && len(buf.Bytes()) > 0) || err == ResumeFromDiskError
+			status := "OK"
+			if !isReadable && err == nil {
+				status = "NOT READABLE"
+			}
+
+			t.Logf("  Offset %d: %s (buf=%v, err=%v)", testOffset, status, buf != nil, err)
+
+			// If offset is in the gap, it should fail to read
+			if flushedCount > 0 && testOffset > maxFlushedOffset && testOffset < bufferStartOffset {
+				if isReadable {
+					t.Errorf("   Unexpected: Offset %d in gap range should NOT be readable!", testOffset)
+				} else {
+					t.Logf("   Expected: Offset %d in gap is not readable (data lost)", testOffset)
+				}
+			}
+		}
+	}
+
+	// Check that all sent messages are accounted for
+	expectedMessageCount := messageCount + 50
+	messagesInMemory := int(currentOffset - bufferStartOffset)
+	totalAccountedFor := flushedCount + messagesInMemory
+
+	t.Logf("\nMESSAGE ACCOUNTING:")
+	t.Logf("  Expected: %d messages", expectedMessageCount)
+	t.Logf("  Flushed to disk: %d", flushedCount)
+	t.Logf("  In memory buffer: %d (offset range %d-%d)", messagesInMemory, bufferStartOffset, currentOffset-1)
+	t.Logf("  Total accounted for: %d", totalAccountedFor)
+	t.Logf("  Missing: %d messages", expectedMessageCount-totalAccountedFor)
+
+	if totalAccountedFor < expectedMessageCount {
+		t.Errorf("DATA LOSS CONFIRMED: %d messages are missing!", expectedMessageCount-totalAccountedFor)
+	} else {
+		t.Logf("All messages accounted for")
+	}
+}
+
+// TestFlushOffsetGap_CheckPrevBuffers tests if messages might be stuck in prevBuffers
+// instead of being properly flushed to disk.
+func TestFlushOffsetGap_CheckPrevBuffers(t *testing.T) {
+	var flushCount int
+	var flushMu sync.Mutex
+
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+		flushMu.Lock()
+		flushCount++
+		count := flushCount
+		flushMu.Unlock()
+
+		t.Logf("FLUSH #%d: minOffset=%d maxOffset=%d size=%d bytes", count, minOffset, maxOffset, len(buf))
+	}
+
+	logBuffer := NewLogBuffer("test", 100*time.Millisecond, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Send messages in batches with flushes in between
+	for batch := 0; batch < 5; batch++ {
+		t.Logf("\nBatch %d:", batch)
+
+		// Send 20 messages
+		for i := 0; i < 20; i++ {
+			offset := int64(batch*20 + i)
+			if err := logBuffer.AddToBuffer(&mq_pb.DataMessage{
+				Key:   []byte(fmt.Sprintf("key-%d", offset)),
+				Value: []byte(fmt.Sprintf("message-%d", offset)),
+				TsNs:  time.Now().UnixNano(),
+			}); err != nil {
+				t.Fatalf("Failed to add buffer: %v", err)
+			}
+		}
+
+		// Check state before flush
+		logBuffer.RLock()
+		beforeFlushOffset := logBuffer.offset
+		beforeFlushStart := logBuffer.bufferStartOffset
+		logBuffer.RUnlock()
+
+		// Force flush
+		logBuffer.ForceFlush()
+		time.Sleep(50 * time.Millisecond)
+
+		// Check state after flush
+		logBuffer.RLock()
+		afterFlushOffset := logBuffer.offset
+		afterFlushStart := logBuffer.bufferStartOffset
+		prevBufferCount := len(logBuffer.prevBuffers.buffers)
+
+		// Check prevBuffers state
+		t.Logf("  Before flush: offset=%d, bufferStartOffset=%d", beforeFlushOffset, beforeFlushStart)
+		t.Logf("  After flush: offset=%d, bufferStartOffset=%d, prevBuffers=%d",
+			afterFlushOffset, afterFlushStart, prevBufferCount)
+
+		// Check each prevBuffer
+		for i, prevBuf := range logBuffer.prevBuffers.buffers {
+			if prevBuf.size > 0 {
+				t.Logf("    prevBuffer[%d]: offsets %d-%d, size=%d bytes (NOT FLUSHED!)",
+					i, prevBuf.startOffset, prevBuf.offset, prevBuf.size)
+			}
+		}
+		logBuffer.RUnlock()
+
+		// CRITICAL: Check if bufferStartOffset advanced correctly
+		expectedNewStart := beforeFlushOffset
+		if afterFlushStart != expectedNewStart {
+			t.Errorf("  bufferStartOffset mismatch!")
+			t.Errorf("     Expected: %d (= offset before flush)", expectedNewStart)
+			t.Errorf("     Actual: %d", afterFlushStart)
+			t.Errorf("     Gap: %d offsets", expectedNewStart-afterFlushStart)
+		}
+	}
+}
+
+// TestFlushOffsetGap_ConcurrentWriteAndFlush tests for race conditions
+// between writing new messages and flushing old ones.
+func TestFlushOffsetGap_ConcurrentWriteAndFlush(t *testing.T) {
+	var allFlushedOffsets []int64
+	var flushMu sync.Mutex
+
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+		t.Logf("FLUSH: offsets %d-%d (%d bytes)", minOffset, maxOffset, len(buf))
+
+		flushMu.Lock()
+		// Record the offset range that was flushed
+		for offset := minOffset; offset <= maxOffset; offset++ {
+			allFlushedOffsets = append(allFlushedOffsets, offset)
+		}
+		flushMu.Unlock()
+	}
+
+	logBuffer := NewLogBuffer("test", 50*time.Millisecond, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Concurrently write messages and force flushes
+	var wg sync.WaitGroup
+
+	// Writer goroutine
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 200; i++ {
+			if err := logBuffer.AddToBuffer(&mq_pb.DataMessage{
+				Key:   []byte(fmt.Sprintf("key-%d", i)),
+				Value: []byte(fmt.Sprintf("message-%d", i)),
+				TsNs:  time.Now().UnixNano(),
+			}); err != nil {
+				t.Errorf("Failed to add buffer: %v", err)
+				return
+			}
+			if i%50 == 0 {
+				time.Sleep(10 * time.Millisecond)
+			}
+		}
+	}()
+
+	// Flusher goroutine
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 5; i++ {
+			time.Sleep(30 * time.Millisecond)
+			logBuffer.ForceFlush()
+		}
+	}()
+
+	wg.Wait()
+	time.Sleep(200 * time.Millisecond) // Wait for final flush
+
+	// Check final state
+	logBuffer.RLock()
+	finalOffset := logBuffer.offset
+	finalBufferStart := logBuffer.bufferStartOffset
+	logBuffer.RUnlock()
+
+	flushMu.Lock()
+	flushedCount := len(allFlushedOffsets)
+	flushMu.Unlock()
+
+	expectedCount := int(finalOffset)
+	inMemory := int(finalOffset - finalBufferStart)
+	totalAccountedFor := flushedCount + inMemory
+
+	t.Logf("\nFINAL STATE:")
+	t.Logf("  Total messages sent: %d (offsets 0-%d)", expectedCount, expectedCount-1)
+	t.Logf("  Flushed to disk: %d", flushedCount)
+	t.Logf("  In memory: %d (offsets %d-%d)", inMemory, finalBufferStart, finalOffset-1)
+	t.Logf("  Total accounted: %d", totalAccountedFor)
+	t.Logf("  Missing: %d", expectedCount-totalAccountedFor)
+
+	if totalAccountedFor < expectedCount {
+		t.Errorf("DATA LOSS in concurrent scenario: %d messages missing!", expectedCount-totalAccountedFor)
+	}
+}
+
+// TestFlushOffsetGap_ProductionScenario reproduces the actual production scenario
+// where the broker uses AddLogEntryToBuffer with explicit Kafka offsets.
+// This simulates leader publishing with offset assignment.
+func TestFlushOffsetGap_ProductionScenario(t *testing.T) {
+	var flushedData []struct {
+		minOffset int64
+		maxOffset int64
+		messages  []*filer_pb.LogEntry
+	}
+	var flushMu sync.Mutex
+
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+		// Parse messages from buffer
+		messages := []*filer_pb.LogEntry{}
+		for pos := 0; pos+4 < len(buf); {
+			size := uint32(buf[pos])<<24 | uint32(buf[pos+1])<<16 | uint32(buf[pos+2])<<8 | uint32(buf[pos+3])
+			if pos+4+int(size) > len(buf) {
+				break
+			}
+			entryData := buf[pos+4 : pos+4+int(size)]
+			logEntry := &filer_pb.LogEntry{}
+			if err := proto.Unmarshal(entryData, logEntry); err == nil {
+				messages = append(messages, logEntry)
+			}
+			pos += 4 + int(size)
+		}
+
+		flushMu.Lock()
+		flushedData = append(flushedData, struct {
+			minOffset int64
+			maxOffset int64
+			messages  []*filer_pb.LogEntry
+		}{minOffset, maxOffset, messages})
+		flushMu.Unlock()
+
+		t.Logf("FLUSH: minOffset=%d maxOffset=%d, parsed %d messages", minOffset, maxOffset, len(messages))
+	}
+
+	logBuffer := NewLogBuffer("test", time.Hour, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Simulate broker behavior: assign Kafka offsets and add to buffer
+	// This is what PublishWithOffset() does
+	nextKafkaOffset := int64(0)
+
+	// Round 1: Add 50 messages with Kafka offsets 0-49
+	t.Logf("\n=== ROUND 1: Adding messages 0-49 ===")
+	for i := 0; i < 50; i++ {
+		logEntry := &filer_pb.LogEntry{
+			Key:    []byte(fmt.Sprintf("key-%d", i)),
+			Data:   []byte(fmt.Sprintf("message-%d", i)),
+			TsNs:   time.Now().UnixNano(),
+			Offset: nextKafkaOffset, // Explicit Kafka offset
+		}
+		if err := logBuffer.AddLogEntryToBuffer(logEntry); err != nil {
+			t.Fatalf("Failed to add log entry: %v", err)
+		}
+		nextKafkaOffset++
+	}
+
+	// Check buffer state before flush
+	logBuffer.RLock()
+	beforeFlushOffset := logBuffer.offset
+	beforeFlushStart := logBuffer.bufferStartOffset
+	logBuffer.RUnlock()
+	t.Logf("Before flush: logBuffer.offset=%d, bufferStartOffset=%d, nextKafkaOffset=%d",
+		beforeFlushOffset, beforeFlushStart, nextKafkaOffset)
+
+	// Flush
+	logBuffer.ForceFlush()
+	time.Sleep(100 * time.Millisecond)
+
+	// Check buffer state after flush
+	logBuffer.RLock()
+	afterFlushOffset := logBuffer.offset
+	afterFlushStart := logBuffer.bufferStartOffset
+	logBuffer.RUnlock()
+	t.Logf("After flush: logBuffer.offset=%d, bufferStartOffset=%d",
+		afterFlushOffset, afterFlushStart)
+
+	// Round 2: Add another 50 messages with Kafka offsets 50-99
+	t.Logf("\n=== ROUND 2: Adding messages 50-99 ===")
+	for i := 0; i < 50; i++ {
+		logEntry := &filer_pb.LogEntry{
+			Key:    []byte(fmt.Sprintf("key-%d", 50+i)),
+			Data:   []byte(fmt.Sprintf("message-%d", 50+i)),
+			TsNs:   time.Now().UnixNano(),
+			Offset: nextKafkaOffset,
+		}
+		if err := logBuffer.AddLogEntryToBuffer(logEntry); err != nil {
+			t.Fatalf("Failed to add log entry: %v", err)
+		}
+		nextKafkaOffset++
+	}
+
+	logBuffer.ForceFlush()
+	time.Sleep(100 * time.Millisecond)
+
+	// Verification: Check if all Kafka offsets are accounted for
+	flushMu.Lock()
+	t.Logf("\n=== VERIFICATION ===")
+	t.Logf("Expected Kafka offsets: 0-%d", nextKafkaOffset-1)
+
+	allOffsets := make(map[int64]bool)
+	for flushIdx, flush := range flushedData {
+		t.Logf("Flush #%d: minOffset=%d, maxOffset=%d, messages=%d",
+			flushIdx, flush.minOffset, flush.maxOffset, len(flush.messages))
+
+		for _, msg := range flush.messages {
+			if allOffsets[msg.Offset] {
+				t.Errorf("  DUPLICATE: Offset %d appears multiple times!", msg.Offset)
+			}
+			allOffsets[msg.Offset] = true
+		}
+	}
+	flushMu.Unlock()
+
+	// Check for missing offsets
+	missingOffsets := []int64{}
+	for expectedOffset := int64(0); expectedOffset < nextKafkaOffset; expectedOffset++ {
+		if !allOffsets[expectedOffset] {
+			missingOffsets = append(missingOffsets, expectedOffset)
+		}
+	}
+
+	if len(missingOffsets) > 0 {
+		t.Errorf("\nMISSING OFFSETS DETECTED: %d offsets missing", len(missingOffsets))
+		if len(missingOffsets) <= 20 {
+			t.Errorf("Missing: %v", missingOffsets)
+		} else {
+			t.Errorf("Missing: %v ... and %d more", missingOffsets[:20], len(missingOffsets)-20)
+		}
+		t.Errorf("\nThis reproduces the production bug!")
+	} else {
+		t.Logf("\nSUCCESS: All %d Kafka offsets accounted for (0-%d)", nextKafkaOffset, nextKafkaOffset-1)
+	}
+
+	// Check buffer offset consistency
+	logBuffer.RLock()
+	finalOffset := logBuffer.offset
+	finalBufferStart := logBuffer.bufferStartOffset
+	logBuffer.RUnlock()
+
+	t.Logf("\nFinal buffer state:")
+	t.Logf("  logBuffer.offset: %d", finalOffset)
+	t.Logf("  bufferStartOffset: %d", finalBufferStart)
+	t.Logf("  Expected (nextKafkaOffset): %d", nextKafkaOffset)
+
+	if finalOffset != nextKafkaOffset {
+		t.Errorf("logBuffer.offset mismatch: expected %d, got %d", nextKafkaOffset, finalOffset)
+	}
+}
+
+// TestFlushOffsetGap_ConcurrentReadDuringFlush tests if concurrent reads
+// during flush can cause messages to be missed.
+func TestFlushOffsetGap_ConcurrentReadDuringFlush(t *testing.T) {
+	var flushedOffsets []int64
+	var flushMu sync.Mutex
+
+	readFromDiskFn := func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+		// Simulate reading from disk - return flushed offsets
+		flushMu.Lock()
+		defer flushMu.Unlock()
+
+		for _, offset := range flushedOffsets {
+			if offset >= startPosition.Offset {
+				logEntry := &filer_pb.LogEntry{
+					Key:    []byte(fmt.Sprintf("key-%d", offset)),
+					Data:   []byte(fmt.Sprintf("message-%d", offset)),
+					TsNs:   time.Now().UnixNano(),
+					Offset: offset,
+				}
+				isDone, err := eachLogEntryFn(logEntry)
+				if err != nil || isDone {
+					return NewMessagePositionFromOffset(offset + 1), isDone, err
+				}
+			}
+		}
+		return startPosition, false, nil
+	}
+
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+		// Parse and store flushed offsets
+		flushMu.Lock()
+		defer flushMu.Unlock()
+
+		for pos := 0; pos+4 < len(buf); {
+			size := uint32(buf[pos])<<24 | uint32(buf[pos+1])<<16 | uint32(buf[pos+2])<<8 | uint32(buf[pos+3])
+			if pos+4+int(size) > len(buf) {
+				break
+			}
+			entryData := buf[pos+4 : pos+4+int(size)]
+			logEntry := &filer_pb.LogEntry{}
+			if err := proto.Unmarshal(entryData, logEntry); err == nil {
+				flushedOffsets = append(flushedOffsets, logEntry.Offset)
+			}
+			pos += 4 + int(size)
+		}
+
+		t.Logf("FLUSH: Stored %d offsets to disk (minOffset=%d, maxOffset=%d)",
+			len(flushedOffsets), minOffset, maxOffset)
+	}
+
+	logBuffer := NewLogBuffer("test", time.Hour, flushFn, readFromDiskFn, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Add 100 messages
+	t.Logf("Adding 100 messages...")
+	for i := int64(0); i < 100; i++ {
+		logEntry := &filer_pb.LogEntry{
+			Key:    []byte(fmt.Sprintf("key-%d", i)),
+			Data:   []byte(fmt.Sprintf("message-%d", i)),
+			TsNs:   time.Now().UnixNano(),
+			Offset: i,
+		}
+		if err := logBuffer.AddLogEntryToBuffer(logEntry); err != nil {
+			t.Fatalf("Failed to add log entry: %v", err)
+		}
+	}
+
+	// Flush (moves data to disk)
+	t.Logf("Flushing...")
+	logBuffer.ForceFlush()
+	time.Sleep(100 * time.Millisecond)
+
+	// Now try to read all messages using ReadMessagesAtOffset
+	t.Logf("\nReading messages from offset 0...")
+	messages, nextOffset, hwm, endOfPartition, err := logBuffer.ReadMessagesAtOffset(0, 1000, 1024*1024)
+
+	t.Logf("Read result: messages=%d, nextOffset=%d, hwm=%d, endOfPartition=%v, err=%v",
+		len(messages), nextOffset, hwm, endOfPartition, err)
+
+	// Verify all offsets can be read
+	readOffsets := make(map[int64]bool)
+	for _, msg := range messages {
+		readOffsets[msg.Offset] = true
+	}
+
+	missingOffsets := []int64{}
+	for expectedOffset := int64(0); expectedOffset < 100; expectedOffset++ {
+		if !readOffsets[expectedOffset] {
+			missingOffsets = append(missingOffsets, expectedOffset)
+		}
+	}
+
+	if len(missingOffsets) > 0 {
+		t.Errorf("MISSING OFFSETS after flush: %d offsets cannot be read", len(missingOffsets))
+		if len(missingOffsets) <= 20 {
+			t.Errorf("Missing: %v", missingOffsets)
+		} else {
+			t.Errorf("Missing: %v ... and %d more", missingOffsets[:20], len(missingOffsets)-20)
+		}
+	} else {
+		t.Logf("All 100 offsets can be read after flush")
+	}
+}
+
+// TestFlushOffsetGap_ForceFlushAdvancesBuffer tests if ForceFlush
+// properly advances bufferStartOffset after flushing.
+func TestFlushOffsetGap_ForceFlushAdvancesBuffer(t *testing.T) {
+	flushedRanges := []struct{ min, max int64 }{}
+	var flushMu sync.Mutex
+
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+		flushMu.Lock()
+		flushedRanges = append(flushedRanges, struct{ min, max int64 }{minOffset, maxOffset})
+		flushMu.Unlock()
+		t.Logf("FLUSH: offsets %d-%d", minOffset, maxOffset)
+	}
+
+	logBuffer := NewLogBuffer("test", time.Hour, flushFn, nil, nil) // Long interval, manual flush only
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Send messages, flush, check state - repeat
+	for round := 0; round < 3; round++ {
+		t.Logf("\n=== ROUND %d ===", round)
+
+		// Check state before adding messages
+		logBuffer.RLock()
+		beforeOffset := logBuffer.offset
+		beforeStart := logBuffer.bufferStartOffset
+		logBuffer.RUnlock()
+
+		t.Logf("Before adding: offset=%d, bufferStartOffset=%d", beforeOffset, beforeStart)
+
+		// Add 10 messages
+		for i := 0; i < 10; i++ {
+			if err := logBuffer.AddToBuffer(&mq_pb.DataMessage{
+				Key:   []byte(fmt.Sprintf("round-%d-msg-%d", round, i)),
+				Value: []byte(fmt.Sprintf("data-%d-%d", round, i)),
+				TsNs:  time.Now().UnixNano(),
+			}); err != nil {
+				t.Fatalf("Failed to add buffer: %v", err)
+			}
+		}
+
+		// Check state after adding
+		logBuffer.RLock()
+		afterAddOffset := logBuffer.offset
+		afterAddStart := logBuffer.bufferStartOffset
+		logBuffer.RUnlock()
+
+		t.Logf("After adding: offset=%d, bufferStartOffset=%d", afterAddOffset, afterAddStart)
+
+		// Force flush
+		t.Logf("Forcing flush...")
+		logBuffer.ForceFlush()
+		time.Sleep(100 * time.Millisecond)
+
+		// Check state after flush
+		logBuffer.RLock()
+		afterFlushOffset := logBuffer.offset
+		afterFlushStart := logBuffer.bufferStartOffset
+		logBuffer.RUnlock()
+
+		t.Logf("After flush: offset=%d, bufferStartOffset=%d", afterFlushOffset, afterFlushStart)
+
+		// CRITICAL CHECK: bufferStartOffset should advance to where offset was before flush
+		if afterFlushStart != afterAddOffset {
+			t.Errorf("FLUSH BUG: bufferStartOffset did NOT advance correctly!")
+			t.Errorf("   Expected bufferStartOffset=%d (= offset after add)", afterAddOffset)
+			t.Errorf("   Actual bufferStartOffset=%d", afterFlushStart)
+			t.Errorf("   Gap: %d offsets WILL BE LOST", afterAddOffset-afterFlushStart)
+		} else {
+			t.Logf("bufferStartOffset correctly advanced to %d", afterFlushStart)
+		}
+	}
+
+	// Final verification: check all offset ranges are continuous
+	flushMu.Lock()
+	t.Logf("\n=== FLUSHED RANGES ===")
+	for i, r := range flushedRanges {
+		t.Logf("Flush #%d: offsets %d-%d", i, r.min, r.max)
+
+		// Check continuity with previous flush
+		if i > 0 {
+			prevMax := flushedRanges[i-1].max
+			currentMin := r.min
+			gap := currentMin - (prevMax + 1)
+
+			if gap > 0 {
+				t.Errorf("GAP between flush #%d and #%d: %d offsets missing!", i-1, i, gap)
+			} else if gap < 0 {
+				t.Errorf("OVERLAP between flush #%d and #%d: %d offsets duplicated!", i-1, i, -gap)
+			} else {
+				t.Logf("  Continuous with previous flush")
+			}
+		}
+	}
+	flushMu.Unlock()
+}

weed/util/log_buffer/log_buffer_oversized_test.go

@@ -0,0 +1,66 @@
+package log_buffer
+
+import (
+	"testing"
+	"time"
+)
+
+// An entry larger than BufferSize grows the window array to hold it. Window
+// arrays cycle through SealBuffer rather than being freed, so the grown one has
+// to be let go once it comes back around -- otherwise every later window is
+// allocated and snapshotted at the oversized width.
+func TestOversizedEntryDoesNotInflateBufferForever(t *testing.T) {
+	lb := NewLogBuffer("oversized", time.Hour, func(_ *LogBuffer, _, _ time.Time, _ []byte, _, _ int64) {}, nil, func() {})
+	defer lb.ShutdownLogBuffer()
+
+	if err := lb.AddDataToBuffer(nil, make([]byte, BufferSize+1), time.Now().UnixNano()); err != nil {
+		t.Fatalf("add oversized entry: %v", err)
+	}
+	if len(lb.buf) <= BufferSize {
+		t.Fatalf("oversized entry did not grow the buffer: %d bytes", len(lb.buf))
+	}
+
+	// One seal per window: the grown array lands in the newest sealed slot and
+	// needs a full trip through the rotation to come back as the current buffer.
+	for i := 0; i < PreviousBufferCount+2; i++ {
+		lb.ForceFlush()
+		if err := lb.AddDataToBuffer(nil, []byte("small"), time.Now().UnixNano()); err != nil {
+			t.Fatalf("add small entry: %v", err)
+		}
+	}
+
+	if len(lb.buf) > BufferSize {
+		t.Errorf("current buffer still %d bytes, want at most %d", len(lb.buf), BufferSize)
+	}
+	for i, sealed := range lb.prevBuffers.buffers {
+		if len(sealed.buf) > BufferSize {
+			t.Errorf("sealed buffer %d still %d bytes, want at most %d", i, len(sealed.buf), BufferSize)
+		}
+	}
+}
+
+// The oversized entry itself still has to survive the round trip.
+func TestOversizedEntryStillFlushes(t *testing.T) {
+	var flushed [][]byte
+	lb := NewLogBuffer("oversized-flush", time.Hour, func(_ *LogBuffer, _, _ time.Time, buf []byte, _, _ int64) {
+		flushed = append(flushed, append([]byte(nil), buf...))
+	}, nil, func() {})
+	defer lb.ShutdownLogBuffer()
+
+	payload := make([]byte, BufferSize+1)
+	for i := range payload {
+		payload[i] = byte(i)
+	}
+	if err := lb.AddDataToBuffer(nil, payload, time.Now().UnixNano()); err != nil {
+		t.Fatalf("add oversized entry: %v", err)
+	}
+	lb.ForceFlush()
+
+	var total int
+	for _, buf := range flushed {
+		total += len(buf)
+	}
+	if total < len(payload) {
+		t.Errorf("flushed %d bytes, want at least the %d byte payload", total, len(payload))
+	}
+}

weed/util/log_buffer/log_buffer_queryability_test.go

@@ -0,0 +1,303 @@
+package log_buffer
+
+import (
+	"bytes"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/proto"
+)
+
+// TestBufferQueryability tests that data written to the buffer can be immediately queried
+func TestBufferQueryability(t *testing.T) {
+	// Create a log buffer with a long flush interval to prevent premature flushing
+	logBuffer := NewLogBuffer("test-buffer", 10*time.Minute,
+		func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+			// Mock flush function - do nothing to keep data in memory
+		},
+		func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+			// Mock read from disk function
+			return startPosition, false, nil
+		},
+		func() {
+			// Mock notify function
+		})
+
+	// Test data similar to schema registry messages
+	testKey := []byte(`{"keytype":"SCHEMA","subject":"test-topic-value","version":1,"magic":1}`)
+	testValue := []byte(`{"subject":"test-topic-value","version":1,"id":1,"schemaType":"AVRO","schema":"\"string\"","deleted":false}`)
+
+	// Create a LogEntry with offset (simulating the schema registry scenario)
+	logEntry := &filer_pb.LogEntry{
+		TsNs:             time.Now().UnixNano(),
+		PartitionKeyHash: 12345,
+		Data:             testValue,
+		Key:              testKey,
+		Offset:           1,
+	}
+
+	// Add the entry to the buffer
+	if err := logBuffer.AddLogEntryToBuffer(logEntry); err != nil {
+		t.Fatalf("Failed to add log entry: %v", err)
+	}
+
+	// Verify the buffer has data
+	if logBuffer.pos == 0 {
+		t.Fatal("Buffer should have data after adding entry")
+	}
+
+	// Test immediate queryability - read from buffer starting from beginning
+	startPosition := NewMessagePosition(0, 0) // Start from beginning
+	bufferCopy, batchIndex, _, err := logBuffer.ReadFromBuffer(startPosition)
+
+	if err != nil {
+		t.Fatalf("ReadFromBuffer failed: %v", err)
+	}
+
+	if bufferCopy == nil {
+		t.Fatal("ReadFromBuffer returned nil buffer - data should be queryable immediately")
+	}
+
+	if batchIndex != 1 {
+		t.Errorf("Expected batchIndex=1, got %d", batchIndex)
+	}
+
+	// Verify we can read the data back
+	buf := bufferCopy.Bytes()
+	if len(buf) == 0 {
+		t.Fatal("Buffer copy is empty")
+	}
+
+	// Parse the first entry from the buffer
+	if len(buf) < 4 {
+		t.Fatal("Buffer too small to contain entry size")
+	}
+
+	size := util.BytesToUint32(buf[0:4])
+	if len(buf) < 4+int(size) {
+		t.Fatalf("Buffer too small to contain entry data: need %d, have %d", 4+int(size), len(buf))
+	}
+
+	entryData := buf[4 : 4+int(size)]
+
+	// Unmarshal and verify the entry
+	retrievedEntry := &filer_pb.LogEntry{}
+	if err := proto.Unmarshal(entryData, retrievedEntry); err != nil {
+		t.Fatalf("Failed to unmarshal retrieved entry: %v", err)
+	}
+
+	// Verify the data matches
+	if !bytes.Equal(retrievedEntry.Key, testKey) {
+		t.Errorf("Key mismatch: expected %s, got %s", string(testKey), string(retrievedEntry.Key))
+	}
+
+	if !bytes.Equal(retrievedEntry.Data, testValue) {
+		t.Errorf("Value mismatch: expected %s, got %s", string(testValue), string(retrievedEntry.Data))
+	}
+
+	if retrievedEntry.Offset != 1 {
+		t.Errorf("Offset mismatch: expected 1, got %d", retrievedEntry.Offset)
+	}
+
+	t.Logf("Buffer queryability test passed - data is immediately readable")
+}
+
+// TestMultipleEntriesQueryability tests querying multiple entries from buffer
+func TestMultipleEntriesQueryability(t *testing.T) {
+	logBuffer := NewLogBuffer("test-multi-buffer", 10*time.Minute,
+		func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+			// Mock flush function
+		},
+		func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+			return startPosition, false, nil
+		},
+		func() {})
+
+	// Add multiple entries
+	for i := 1; i <= 3; i++ {
+		logEntry := &filer_pb.LogEntry{
+			TsNs:             time.Now().UnixNano() + int64(i*1000), // Ensure different timestamps
+			PartitionKeyHash: int32(i),
+			Data:             []byte("test-data-" + string(rune('0'+i))),
+			Key:              []byte("test-key-" + string(rune('0'+i))),
+			Offset:           int64(i),
+		}
+		if err := logBuffer.AddLogEntryToBuffer(logEntry); err != nil {
+			t.Fatalf("Failed to add log entry: %v", err)
+		}
+	}
+
+	// Read all entries
+	startPosition := NewMessagePosition(0, 0)
+	bufferCopy, batchIndex, _, err := logBuffer.ReadFromBuffer(startPosition)
+
+	if err != nil {
+		t.Fatalf("ReadFromBuffer failed: %v", err)
+	}
+
+	if bufferCopy == nil {
+		t.Fatal("ReadFromBuffer returned nil buffer")
+	}
+
+	if batchIndex != 3 {
+		t.Errorf("Expected batchIndex=3, got %d", batchIndex)
+	}
+
+	// Count entries in buffer
+	buf := bufferCopy.Bytes()
+	entryCount := 0
+	pos := 0
+
+	for pos+4 < len(buf) {
+		size := util.BytesToUint32(buf[pos : pos+4])
+		if pos+4+int(size) > len(buf) {
+			break
+		}
+
+		entryData := buf[pos+4 : pos+4+int(size)]
+		entry := &filer_pb.LogEntry{}
+		if err := proto.Unmarshal(entryData, entry); err != nil {
+			t.Fatalf("Failed to unmarshal entry %d: %v", entryCount+1, err)
+		}
+
+		entryCount++
+		pos += 4 + int(size)
+
+		t.Logf("Entry %d: Key=%s, Data=%s, Offset=%d", entryCount, string(entry.Key), string(entry.Data), entry.Offset)
+	}
+
+	if entryCount != 3 {
+		t.Errorf("Expected 3 entries, found %d", entryCount)
+	}
+
+	t.Logf("Multiple entries queryability test passed - found %d entries", entryCount)
+}
+
+// TestSchemaRegistryScenario tests the specific scenario that was failing
+func TestSchemaRegistryScenario(t *testing.T) {
+	logBuffer := NewLogBuffer("_schemas", 10*time.Minute,
+		func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+			// Mock flush function - simulate what happens in real scenario
+			t.Logf("FLUSH: startTime=%v, stopTime=%v, bufSize=%d, minOffset=%d, maxOffset=%d",
+				startTime, stopTime, len(buf), minOffset, maxOffset)
+		},
+		func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+			return startPosition, false, nil
+		},
+		func() {})
+
+	// Simulate schema registry message
+	schemaKey := []byte(`{"keytype":"SCHEMA","subject":"test-schema-value","version":1,"magic":1}`)
+	schemaValue := []byte(`{"subject":"test-schema-value","version":1,"id":12,"schemaType":"AVRO","schema":"\"string\"","deleted":false}`)
+
+	logEntry := &filer_pb.LogEntry{
+		TsNs:             time.Now().UnixNano(),
+		PartitionKeyHash: 12345,
+		Data:             schemaValue,
+		Key:              schemaKey,
+		Offset:           0, // First message
+	}
+
+	// Add to buffer
+	if err := logBuffer.AddLogEntryToBuffer(logEntry); err != nil {
+		t.Fatalf("Failed to add log entry: %v", err)
+	}
+
+	// Simulate the SQL query scenario - read from offset 0
+	startPosition := NewMessagePosition(0, 0)
+	bufferCopy, _, _, err := logBuffer.ReadFromBuffer(startPosition)
+
+	if err != nil {
+		t.Fatalf("Schema registry scenario failed: %v", err)
+	}
+
+	if bufferCopy == nil {
+		t.Fatal("Schema registry scenario: ReadFromBuffer returned nil - this is the bug!")
+	}
+
+	// Verify schema data is readable
+	buf := bufferCopy.Bytes()
+	if len(buf) < 4 {
+		t.Fatal("Buffer too small")
+	}
+
+	size := util.BytesToUint32(buf[0:4])
+	entryData := buf[4 : 4+int(size)]
+
+	retrievedEntry := &filer_pb.LogEntry{}
+	if err := proto.Unmarshal(entryData, retrievedEntry); err != nil {
+		t.Fatalf("Failed to unmarshal schema entry: %v", err)
+	}
+
+	// Verify schema value is preserved
+	if !bytes.Equal(retrievedEntry.Data, schemaValue) {
+		t.Errorf("Schema value lost! Expected: %s, Got: %s", string(schemaValue), string(retrievedEntry.Data))
+	}
+
+	if len(retrievedEntry.Data) != len(schemaValue) {
+		t.Errorf("Schema value length mismatch! Expected: %d, Got: %d", len(schemaValue), len(retrievedEntry.Data))
+	}
+
+	t.Logf("Schema registry scenario test passed - schema value preserved: %d bytes", len(retrievedEntry.Data))
+}
+
+// TestTimeBasedFirstReadBeforeEarliest ensures starting slightly before earliest memory
+// does not force a disk resume and returns in-memory data (regression test)
+func TestTimeBasedFirstReadBeforeEarliest(t *testing.T) {
+	flushed := false
+	logBuffer := NewLogBuffer("local", 10*time.Minute,
+		func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+			// keep in memory; we just want earliest time populated
+			_ = buf
+		},
+		func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+			// disk should not be consulted in this regression path
+			return startPosition, false, nil
+		},
+		func() {})
+
+	// Seed one entry so earliestTime is set
+	baseTs := time.Now().Add(-time.Second)
+	entry := &filer_pb.LogEntry{TsNs: baseTs.UnixNano(), Data: []byte("x"), Key: []byte("k"), Offset: 0}
+	if err := logBuffer.AddLogEntryToBuffer(entry); err != nil {
+		t.Fatalf("Failed to add log entry: %v", err)
+	}
+	_ = flushed
+
+	// Start read 1ns before earliest memory, with offset sentinel (-2)
+	startPos := NewMessagePosition(baseTs.Add(-time.Nanosecond).UnixNano(), -2)
+	buf, _, _, err := logBuffer.ReadFromBuffer(startPos)
+	if err != nil {
+		t.Fatalf("ReadFromBuffer returned err: %v", err)
+	}
+	if buf == nil {
+		t.Fatalf("Expected in-memory data, got nil buffer")
+	}
+}
+
+// TestEarliestTimeExactRead ensures starting exactly at earliest time returns first entry (no skip)
+func TestEarliestTimeExactRead(t *testing.T) {
+	logBuffer := NewLogBuffer("local", 10*time.Minute,
+		func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {},
+		func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+			return startPosition, false, nil
+		},
+		func() {})
+
+	ts := time.Now()
+	entry := &filer_pb.LogEntry{TsNs: ts.UnixNano(), Data: []byte("a"), Key: []byte("k"), Offset: 0}
+	if err := logBuffer.AddLogEntryToBuffer(entry); err != nil {
+		t.Fatalf("Failed to add log entry: %v", err)
+	}
+
+	startPos := NewMessagePosition(ts.UnixNano(), -2)
+	buf, _, _, err := logBuffer.ReadFromBuffer(startPos)
+	if err != nil {
+		t.Fatalf("ReadFromBuffer err: %v", err)
+	}
+	if buf == nil || buf.Len() == 0 {
+		t.Fatalf("Expected data at earliest time, got nil/empty")
+	}
+}

weed/util/log_buffer/log_buffer_test.go

@@ -0,0 +1,599 @@
+package log_buffer
+
+import (
+	"bytes"
+	"crypto/rand"
+	"fmt"
+	"io"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func TestNewLogBufferFirstBuffer(t *testing.T) {
+	flushInterval := time.Second
+	lb := NewLogBuffer("test", flushInterval, func(logBuffer *LogBuffer, startTime time.Time, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+		fmt.Printf("flush from %v to %v %d bytes\n", startTime, stopTime, len(buf))
+	}, nil, func() {
+	})
+
+	startTime := MessagePosition{Time: time.Now()}
+
+	messageSize := 1024
+	messageCount := 5000
+
+	receivedMessageCount := 0
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		lastProcessedTime, isDone, err := lb.LoopProcessLogData("test", startTime, 0, func() bool {
+			// stop if no more messages
+			return receivedMessageCount < messageCount
+		}, func(logEntry *filer_pb.LogEntry) (isDone bool, err error) {
+			receivedMessageCount++
+			if receivedMessageCount >= messageCount {
+				println("processed all messages")
+				return true, io.EOF
+			}
+			return false, nil
+		})
+
+		fmt.Printf("before flush: sent %d received %d\n", messageCount, receivedMessageCount)
+		fmt.Printf("lastProcessedTime %v isDone %v err: %v\n", lastProcessedTime, isDone, err)
+		if err != nil && err != io.EOF {
+			t.Errorf("unexpected error %v", err)
+		}
+	}()
+
+	var buf = make([]byte, messageSize)
+	for i := 0; i < messageCount; i++ {
+		rand.Read(buf)
+		if err := lb.AddToBuffer(&mq_pb.DataMessage{
+			Key:   nil,
+			Value: buf,
+			TsNs:  0,
+		}); err != nil {
+			t.Fatalf("Failed to add buffer: %v", err)
+		}
+	}
+	wg.Wait()
+
+	if receivedMessageCount != messageCount {
+		t.Errorf("expect %d messages, but got %d", messageCount, receivedMessageCount)
+	}
+}
+
+func TestReadFromBufferTimestampBased_AfterFlushReturnsNewerData(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	payload := bytes.Repeat([]byte("x"), 4096)
+	var sealed *MemBuffer
+
+	for i := 0; i < 5000; i++ {
+		if err := lb.AddDataToBuffer([]byte("k"), payload, int64(i+1)); err != nil {
+			t.Fatalf("AddDataToBuffer(%d): %v", i, err)
+		}
+		candidate := lb.prevBuffers.buffers[len(lb.prevBuffers.buffers)-1]
+		if candidate.size > 0 {
+			sealed = &MemBuffer{
+				size:      candidate.size,
+				startTime: candidate.startTime,
+				stopTime:  candidate.stopTime,
+				offset:    candidate.offset,
+			}
+			break
+		}
+	}
+
+	if sealed == nil {
+		t.Fatal("expected first buffer flush to produce a sealed buffer")
+	}
+
+	for i := 5000; i < 5100; i++ {
+		if err := lb.AddDataToBuffer([]byte("k"), payload, int64(i+1)); err != nil {
+			t.Fatalf("AddDataToBuffer(%d): %v", i, err)
+		}
+	}
+
+	buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(sealed.stopTime.UnixNano(), sealed.offset))
+	if err != nil {
+		t.Fatalf("ReadFromBuffer returned error: %v", err)
+	}
+	if buf == nil || buf.Len() == 0 {
+		t.Fatalf("expected newer data after the first sealed buffer, got %v", buf)
+	}
+}
+
+// TestReadFromBuffer_OldOffsetReturnsResumeFromDiskError tests that requesting an old offset
+// that has been flushed to disk properly returns ResumeFromDiskError instead of hanging forever.
+// This reproduces the bug where Schema Registry couldn't read the _schemas topic.
+func TestReadFromBuffer_OldOffsetReturnsResumeFromDiskError(t *testing.T) {
+	tests := []struct {
+		name              string
+		bufferStartOffset int64
+		currentOffset     int64
+		requestedOffset   int64
+		hasData           bool
+		expectError       error
+		description       string
+	}{
+		{
+			name:              "Request offset 0 when buffer starts at 4 (Schema Registry bug scenario)",
+			bufferStartOffset: 4,
+			currentOffset:     10,
+			requestedOffset:   0,
+			hasData:           true,
+			expectError:       ResumeFromDiskError,
+			description:       "When Schema Registry tries to read from offset 0, but data has been flushed to disk",
+		},
+		{
+			name:              "Request offset before buffer start with empty buffer",
+			bufferStartOffset: 10,
+			currentOffset:     10,
+			requestedOffset:   5,
+			hasData:           false,
+			expectError:       ResumeFromDiskError,
+			description:       "Old offset with no data in memory should trigger disk read",
+		},
+		{
+			name:              "Request offset before buffer start with data",
+			bufferStartOffset: 100,
+			currentOffset:     150,
+			requestedOffset:   50,
+			hasData:           true,
+			expectError:       ResumeFromDiskError,
+			description:       "Old offset with current data in memory should still trigger disk read",
+		},
+		{
+			name:              "Request current offset (no disk read needed)",
+			bufferStartOffset: 4,
+			currentOffset:     10,
+			requestedOffset:   10,
+			hasData:           true,
+			expectError:       nil,
+			description:       "Current offset should return data from memory without error",
+		},
+		{
+			name:              "Request offset within buffer range",
+			bufferStartOffset: 4,
+			currentOffset:     10,
+			requestedOffset:   7,
+			hasData:           true,
+			expectError:       nil,
+			description:       "Offset within buffer range should return data without error",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Create a LogBuffer with minimal configuration
+			lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+
+			// Simulate data that has been flushed to disk by setting bufferStartOffset
+			lb.bufferStartOffset = tt.bufferStartOffset
+			lb.offset = tt.currentOffset
+
+			// CRITICAL: Mark this as an offset-based buffer
+			lb.hasOffsets = true
+
+			// Add some data to the buffer if needed (at current offset position)
+			if tt.hasData {
+				testData := []byte("test message")
+				// Use AddLogEntryToBuffer to preserve offset information
+				if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{
+					TsNs:   time.Now().UnixNano(),
+					Key:    []byte("key"),
+					Data:   testData,
+					Offset: tt.currentOffset, // Add data at current offset
+				}); err != nil {
+					t.Fatalf("Failed to add log entry: %v", err)
+				}
+			}
+
+			// Create an offset-based position for the requested offset
+			requestPosition := NewMessagePositionFromOffset(tt.requestedOffset)
+
+			// Try to read from the buffer
+			buf, batchIdx, _, err := lb.ReadFromBuffer(requestPosition)
+
+			// Verify the error matches expectations
+			if tt.expectError != nil {
+				if err != tt.expectError {
+					t.Errorf("%s\nExpected error: %v\nGot error: %v\nbuf=%v, batchIdx=%d",
+						tt.description, tt.expectError, err, buf != nil, batchIdx)
+				} else {
+					t.Logf("✓ %s: correctly returned %v", tt.description, err)
+				}
+			} else {
+				if err != nil {
+					t.Errorf("%s\nExpected no error but got: %v\nbuf=%v, batchIdx=%d",
+						tt.description, err, buf != nil, batchIdx)
+				} else {
+					t.Logf("✓ %s: correctly returned data without error", tt.description)
+				}
+			}
+		})
+	}
+}
+
+// TestReadFromBuffer_OldOffsetWithNoPrevBuffers specifically tests the bug fix
+// where requesting an old offset would return nil instead of ResumeFromDiskError
+func TestReadFromBuffer_OldOffsetWithNoPrevBuffers(t *testing.T) {
+	// This is the exact scenario that caused the Schema Registry to hang:
+	// 1. Data was published to _schemas topic (offsets 0, 1, 2, 3)
+	// 2. Data was flushed to disk
+	// 3. LogBuffer's bufferStartOffset was updated to 4
+	// 4. Schema Registry tried to read from offset 0
+	// 5. ReadFromBuffer would return (nil, offset, nil) instead of ResumeFromDiskError
+	// 6. The subscriber would wait forever for data that would never come from memory
+
+	lb := NewLogBuffer("_schemas", time.Hour, nil, nil, func() {})
+
+	// Simulate the state after data has been flushed to disk:
+	// - bufferStartOffset = 10 (data 0-9 has been flushed)
+	// - offset = 15 (next offset to assign, current buffer has 10-14)
+	// - pos = 100 (some data in current buffer)
+	// Set prevBuffers to have non-overlapping ranges to avoid the safety check at line 420-428
+	lb.bufferStartOffset = 10
+	lb.offset = 15
+	lb.pos = 100
+
+	// Modify prevBuffers to have non-zero offset ranges that DON'T include the requested offset
+	// This bypasses the safety check and exposes the real bug
+	for i := range lb.prevBuffers.buffers {
+		lb.prevBuffers.buffers[i].startOffset = 20 + int64(i)*10 // 20, 30, 40, etc.
+		lb.prevBuffers.buffers[i].offset = 25 + int64(i)*10      // 25, 35, 45, etc.
+		lb.prevBuffers.buffers[i].size = 0                       // Empty (flushed)
+	}
+
+	// Schema Registry requests offset 5 (which is before bufferStartOffset=10)
+	requestPosition := NewMessagePositionFromOffset(5)
+
+	// Before the fix, this would return (nil, offset, nil) causing an infinite wait
+	// After the fix, this should return ResumeFromDiskError
+	buf, batchIdx, _, err := lb.ReadFromBuffer(requestPosition)
+
+	t.Logf("DEBUG: ReadFromBuffer returned: buf=%v, batchIdx=%d, err=%v", buf != nil, batchIdx, err)
+	t.Logf("DEBUG: Buffer state: bufferStartOffset=%d, offset=%d, pos=%d",
+		lb.bufferStartOffset, lb.offset, lb.pos)
+	t.Logf("DEBUG: Requested offset 5, prevBuffers[0] range: [%d-%d]",
+		lb.prevBuffers.buffers[0].startOffset, lb.prevBuffers.buffers[0].offset)
+
+	if err != ResumeFromDiskError {
+		t.Errorf("CRITICAL BUG REPRODUCED: Expected ResumeFromDiskError but got err=%v, buf=%v, batchIdx=%d\n"+
+			"This causes Schema Registry to hang indefinitely waiting for data that's on disk!",
+			err, buf != nil, batchIdx)
+		t.Errorf("The buggy code falls through without returning ResumeFromDiskError!")
+	} else {
+		t.Logf("✓ BUG FIX VERIFIED: Correctly returns ResumeFromDiskError when requesting old offset 5")
+		t.Logf("  This allows the subscriber to read from disk instead of waiting forever")
+	}
+}
+
+// TestReadFromBuffer_EmptyBufferAtCurrentOffset tests Bug #2
+// where an empty buffer at the current offset would return empty data instead of ResumeFromDiskError
+func TestReadFromBuffer_EmptyBufferAtCurrentOffset(t *testing.T) {
+	lb := NewLogBuffer("_schemas", time.Hour, nil, nil, func() {})
+
+	// Simulate buffer state where data 0-3 was published and flushed, but buffer NOT advanced yet:
+	// - bufferStartOffset = 0 (buffer hasn't been advanced after flush)
+	// - offset = 4 (next offset to assign - data 0-3 exists)
+	// - pos = 0 (buffer is empty after flush)
+	// This happens in the window between flush and buffer advancement
+	lb.bufferStartOffset = 0
+	lb.offset = 4
+	lb.pos = 0
+
+	// Schema Registry requests offset 0 (which appears to be in range [0, 4])
+	requestPosition := NewMessagePositionFromOffset(0)
+
+	// BUG: Without fix, this returns empty buffer instead of checking disk
+	// FIX: Should return ResumeFromDiskError because buffer is empty (pos=0) despite valid range
+	buf, batchIdx, _, err := lb.ReadFromBuffer(requestPosition)
+
+	t.Logf("DEBUG: ReadFromBuffer returned: buf=%v, batchIdx=%d, err=%v", buf != nil, batchIdx, err)
+	t.Logf("DEBUG: Buffer state: bufferStartOffset=%d, offset=%d, pos=%d",
+		lb.bufferStartOffset, lb.offset, lb.pos)
+
+	if err != ResumeFromDiskError {
+		if buf == nil || len(buf.Bytes()) == 0 {
+			t.Errorf("CRITICAL BUG #2 REPRODUCED: Empty buffer should return ResumeFromDiskError, got err=%v, buf=%v\n"+
+				"Without the fix, Schema Registry gets empty data instead of reading from disk!",
+				err, buf != nil)
+		}
+	} else {
+		t.Logf("✓ BUG #2 FIX VERIFIED: Empty buffer correctly returns ResumeFromDiskError to check disk")
+	}
+}
+
+// TestReadFromBuffer_OffsetRanges tests various offset range scenarios
+func TestReadFromBuffer_OffsetRanges(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+
+	// Setup: buffer contains offsets 10-20
+	lb.bufferStartOffset = 10
+	lb.offset = 20
+	lb.pos = 100 // some data in buffer
+
+	testCases := []struct {
+		name            string
+		requestedOffset int64
+		expectedError   error
+		description     string
+	}{
+		{
+			name:            "Before buffer start",
+			requestedOffset: 5,
+			expectedError:   ResumeFromDiskError,
+			description:     "Offset 5 < bufferStartOffset 10 → read from disk",
+		},
+		{
+			name:            "At buffer start",
+			requestedOffset: 10,
+			expectedError:   nil,
+			description:     "Offset 10 == bufferStartOffset 10 → read from buffer",
+		},
+		{
+			name:            "Within buffer range",
+			requestedOffset: 15,
+			expectedError:   nil,
+			description:     "Offset 15 is within [10, 20] → read from buffer",
+		},
+		{
+			name:            "At buffer end",
+			requestedOffset: 20,
+			expectedError:   nil,
+			description:     "Offset 20 == offset 20 → read from buffer",
+		},
+		{
+			name:            "After buffer end",
+			requestedOffset: 25,
+			expectedError:   nil,
+			description:     "Offset 25 > offset 20 → future data, return nil without error",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			requestPosition := NewMessagePositionFromOffset(tc.requestedOffset)
+			_, _, _, err := lb.ReadFromBuffer(requestPosition)
+
+			if tc.expectedError != nil {
+				if err != tc.expectedError {
+					t.Errorf("%s\nExpected error: %v, got: %v", tc.description, tc.expectedError, err)
+				} else {
+					t.Logf("✓ %s", tc.description)
+				}
+			} else {
+				// For nil expectedError, we accept either nil or no error condition
+				// (future offsets return nil without error)
+				if err != nil && err != ResumeFromDiskError {
+					t.Errorf("%s\nExpected no ResumeFromDiskError, got: %v", tc.description, err)
+				} else {
+					t.Logf("✓ %s", tc.description)
+				}
+			}
+		})
+	}
+}
+
+// TestReadFromBuffer_InitializedFromDisk tests Bug #3
+// where bufferStartOffset was incorrectly set to 0 after InitializeOffsetFromExistingData,
+// causing reads for old offsets to return new data instead of triggering a disk read.
+func TestReadFromBuffer_InitializedFromDisk(t *testing.T) {
+	// This reproduces the real Schema Registry bug scenario:
+	// 1. Broker restarts, finds 4 messages on disk (offsets 0-3)
+	// 2. InitializeOffsetFromExistingData sets offset=4
+	//    - BUG: bufferStartOffset=0 (wrong!)
+	//    - FIX: bufferStartOffset=4 (correct!)
+	// 3. First new message is written (offset 4)
+	// 4. Schema Registry reads offset 0
+	// 5. With FIX: requestedOffset=0 < bufferStartOffset=4 → ResumeFromDiskError (correct!)
+	// 6. Without FIX: requestedOffset=0 in range [0, 5] → returns wrong data (bug!)
+
+	lb := NewLogBuffer("_schemas", time.Hour, nil, nil, func() {})
+
+	// Use the actual InitializeOffsetFromExistingData to test the fix
+	err := lb.InitializeOffsetFromExistingData(func() (int64, error) {
+		return 3, nil // Simulate 4 messages on disk (offsets 0-3, highest=3)
+	})
+	if err != nil {
+		t.Fatalf("InitializeOffsetFromExistingData failed: %v", err)
+	}
+
+	t.Logf("After InitializeOffsetFromExistingData(highestOffset=3):")
+	t.Logf("  offset=%d (should be 4), bufferStartOffset=%d (FIX: should be 4, not 0)",
+		lb.offset, lb.bufferStartOffset)
+
+	// Now write a new message at offset 4
+	if err := lb.AddToBuffer(&mq_pb.DataMessage{
+		Key:   []byte("new-key"),
+		Value: []byte("new-message-at-offset-4"),
+		TsNs:  time.Now().UnixNano(),
+	}); err != nil {
+		t.Fatalf("Failed to add buffer: %v", err)
+	}
+	// After AddToBuffer: offset=5, pos>0
+
+	// Schema Registry tries to read offset 0 (should be on disk)
+	requestPosition := NewMessagePositionFromOffset(0)
+
+	buf, batchIdx, _, err := lb.ReadFromBuffer(requestPosition)
+
+	t.Logf("After writing new message:")
+	t.Logf("  bufferStartOffset=%d, offset=%d, pos=%d", lb.bufferStartOffset, lb.offset, lb.pos)
+	t.Logf("  Requested offset 0, got: buf=%v, batchIdx=%d, err=%v", buf != nil, batchIdx, err)
+
+	// EXPECTED BEHAVIOR (with fix):
+	// bufferStartOffset=4 after initialization, so requestedOffset=0 < bufferStartOffset=4
+	// → returns ResumeFromDiskError
+
+	// BUGGY BEHAVIOR (without fix):
+	// bufferStartOffset=0 after initialization, so requestedOffset=0 is in range [0, 5]
+	// → returns the NEW message (offset 4) instead of reading from disk!
+
+	if err != ResumeFromDiskError {
+		t.Errorf("CRITICAL BUG #3 REPRODUCED: Reading offset 0 after initialization from disk should return ResumeFromDiskError\n"+
+			"Instead got: err=%v, buf=%v, batchIdx=%d\n"+
+			"This means Schema Registry would receive WRONG data (offset 4) when requesting offset 0!",
+			err, buf != nil, batchIdx)
+		t.Errorf("Root cause: bufferStartOffset=%d should be 4 after InitializeOffsetFromExistingData(highestOffset=3)",
+			lb.bufferStartOffset)
+	} else {
+		t.Logf("✓ BUG #3 FIX VERIFIED: Reading old offset 0 correctly returns ResumeFromDiskError")
+		t.Logf("  This ensures Schema Registry reads correct data from disk instead of getting new messages")
+	}
+}
+
+// TestLoopProcessLogDataWithOffset_DiskReadRetry tests that when a subscriber
+// reads from disk before data is available, it continues to retry disk reads
+// and eventually finds the data once it appears.  This reproduces the Schema
+// Registry timeout where the first read happened before the data landed on
+// disk and the loop never retried.
+//
+// The data is delivered exclusively through the disk-read path (no in-memory
+// AddToBuffer) so the test deterministically asserts that retries happen,
+// rather than racing the in-memory delivery path.
+func TestLoopProcessLogDataWithOffset_DiskReadRetry(t *testing.T) {
+	diskReadCallCount := 0
+	diskReadMu := sync.Mutex{}
+	dataReady := false
+	mockEntry := &filer_pb.LogEntry{
+		Key:    []byte("key-0"),
+		Data:   []byte("message-0"),
+		TsNs:   time.Now().UnixNano(),
+		Offset: 0,
+	}
+
+	readFromDiskFn := func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+		diskReadMu.Lock()
+		diskReadCallCount++
+		callNum := diskReadCallCount
+		ready := dataReady
+		diskReadMu.Unlock()
+
+		t.Logf("DISK READ #%d: startOffset=%d, dataReady=%v", callNum, startPosition.Offset, ready)
+
+		if !ready {
+			t.Logf("  → No data on disk yet")
+			return startPosition, false, nil
+		}
+
+		if mockEntry.Offset >= startPosition.Offset {
+			isDone, err := eachLogEntryFn(mockEntry)
+			if err != nil || isDone {
+				return NewMessagePositionFromOffset(mockEntry.Offset + 1), isDone, err
+			}
+		}
+		return NewMessagePositionFromOffset(mockEntry.Offset + 1), false, nil
+	}
+
+	logBuffer := NewLogBuffer("test", 1*time.Minute, nil, readFromDiskFn, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	receivedMessages := 0
+	mu := sync.Mutex{}
+	maxIterations := 50
+	iterationCount := 0
+
+	waitForDataFn := func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		iterationCount++
+		return receivedMessages == 0 && iterationCount < maxIterations
+	}
+
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+		mu.Lock()
+		receivedMessages++
+		mu.Unlock()
+		t.Logf("✉️  RECEIVED: offset=%d key=%s", offset, string(logEntry.Key))
+		return true, nil
+	}
+
+	var readerWg sync.WaitGroup
+	readerWg.Add(1)
+	go func() {
+		defer readerWg.Done()
+		startPosition := NewMessagePositionFromOffset(0)
+		_, isDone, err := logBuffer.LoopProcessLogDataWithOffset("test-subscriber", startPosition, 0, waitForDataFn, eachLogEntryFn)
+		t.Logf("📋 Reader finished: isDone=%v, err=%v", isDone, err)
+	}()
+
+	// Wait until the reader has issued at least one no-op disk read so we
+	// know it has entered the retry loop, then publish the data.
+	deadline := time.Now().Add(2 * time.Second)
+	for {
+		diskReadMu.Lock()
+		seen := diskReadCallCount
+		diskReadMu.Unlock()
+		if seen >= 1 {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("reader never called readFromDiskFn")
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+
+	t.Logf("➕ Marking data as ready on disk")
+	diskReadMu.Lock()
+	dataReady = true
+	diskReadMu.Unlock()
+
+	readerWg.Wait()
+
+	diskReadMu.Lock()
+	finalDiskReadCount := diskReadCallCount
+	diskReadMu.Unlock()
+
+	mu.Lock()
+	finalReceivedMessages := receivedMessages
+	finalIterations := iterationCount
+	mu.Unlock()
+
+	t.Logf("\nRESULTS:")
+	t.Logf("  Disk reads: %d", finalDiskReadCount)
+	t.Logf("  Received messages: %d", finalReceivedMessages)
+	t.Logf("  Loop iterations: %d", finalIterations)
+
+	if finalDiskReadCount < 2 {
+		t.Errorf("CRITICAL BUG REPRODUCED: Disk read was only called %d time(s)", finalDiskReadCount)
+		t.Errorf("Expected: Multiple disk reads as the loop continues after data lands")
+		t.Errorf("This is why Schema Registry times out - it reads once before data is available, never re-reads")
+	}
+
+	if finalReceivedMessages == 0 {
+		t.Errorf("SCHEMA REGISTRY TIMEOUT REPRODUCED: No messages received even after data landed")
+		t.Errorf("The subscriber is stuck because disk reads are not retried")
+	} else {
+		t.Logf("✓ SUCCESS: Message received after %d disk read attempts", finalDiskReadCount)
+	}
+}
+
+// A buffer shut down long before its flush interval elapses must not keep its
+// loops - and the tens of megabytes of slabs they reach - alive until the next
+// tick would have fired.
+func TestShutdownStopsGoroutinesPromptly(t *testing.T) {
+	lb := NewLogBuffer("shutdown", time.Hour, nil, nil, nil)
+	// let both loops start and park in their waits before shutting down
+	time.Sleep(100 * time.Millisecond)
+	lb.ShutdownLogBuffer()
+
+	done := make(chan struct{})
+	go func() {
+		lb.loopsDone.Wait()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("log buffer goroutines still running 5s after shutdown")
+	}
+}

weed/util/log_buffer/log_read.go

@@ -0,0 +1,597 @@
+package log_buffer
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var (
+	ResumeError         = fmt.Errorf("resume")
+	ResumeFromDiskError = fmt.Errorf("resumeFromDisk")
+	// StopReadingError, wrapped by an eachLogDataFn's error, marks a read the
+	// callback ended on purpose - control flow, not a failure - so the loop
+	// hands it back to the caller without logging it.
+	StopReadingError = fmt.Errorf("stopReading")
+)
+
+// notificationHealthCheckInterval bounds how long an idle subscriber blocks
+// on the notification channel before re-checking state (client disconnect via
+// waitForDataFn, LogBuffer shutdown, timestamp advancement). notifyChan is the
+// primary wakeup path when new data arrives or a flush lands; this timeout is
+// the safety net for any missed notification and also caps the latency to
+// notice that the subscriber should exit. Balances idle CPU and log noise
+// against client-disconnect detection latency.
+const notificationHealthCheckInterval = 250 * time.Millisecond
+
+// caughtUpDiskPollInterval is the (longer) timeout used once a subscriber has
+// already proven via ReadFromDiskFn that there is no data past its current
+// position on disk. In that state, notifyChan is the primary wakeup path for
+// in-memory writes; disk may still gain data from external writers (e.g.
+// Schema Registry's external disk-write path), so we still re-probe on
+// timeout, just at a much lower cadence than the 250ms health tick.
+const caughtUpDiskPollInterval = 2 * time.Second
+
+type MessagePosition struct {
+	Time          time.Time // timestamp of the message
+	Offset        int64     // Kafka offset for offset-based positioning, or batch index for timestamp-based
+	IsOffsetBased bool      // true if this position is offset-based, false if timestamp-based
+}
+
+func NewMessagePosition(tsNs int64, offset int64) MessagePosition {
+	return MessagePosition{
+		Time:          time.Unix(0, tsNs).UTC(),
+		Offset:        offset,
+		IsOffsetBased: false, // timestamp-based by default
+	}
+}
+
+// NewMessagePositionFromOffset creates a MessagePosition that represents a specific offset
+func NewMessagePositionFromOffset(offset int64) MessagePosition {
+	return MessagePosition{
+		Time:          time.Time{}, // Zero time for offset-based positions
+		Offset:        offset,
+		IsOffsetBased: true,
+	}
+}
+
+// GetOffset extracts the offset from an offset-based MessagePosition
+func (mp MessagePosition) GetOffset() int64 {
+	if !mp.IsOffsetBased {
+		return -1 // Not an offset-based position
+	}
+	return mp.Offset // Offset is stored directly
+}
+
+// awaitNotificationOrTimeout blocks until one of:
+//   - a new-data / flush notification arrives on notifyChan (returns true)
+//   - the LogBuffer is shut down via ShutdownLogBuffer (returns true; callers
+//     re-check IsStopping() and exit)
+//   - notificationHealthCheckInterval elapses (returns false; caller
+//     re-checks client-disconnect and other state)
+func (logBuffer *LogBuffer) awaitNotificationOrTimeout(notifyChan <-chan struct{}) bool {
+	return logBuffer.awaitNotificationOrTimeoutFor(notifyChan, notificationHealthCheckInterval)
+}
+
+// awaitNotificationOrTimeoutFor is awaitNotificationOrTimeout with a caller-supplied
+// timeout. Used by the ResumeFromDiskError path to back off to
+// caughtUpDiskPollInterval once the disk has been proven empty past the
+// subscriber's current position.
+func (logBuffer *LogBuffer) awaitNotificationOrTimeoutFor(notifyChan <-chan struct{}, timeout time.Duration) bool {
+	timer := time.NewTimer(timeout)
+	defer timer.Stop()
+
+	select {
+	case <-notifyChan:
+		return true
+	case <-logBuffer.shutdownCh:
+		return true
+	case <-timer.C:
+		return false
+	}
+}
+
+func (logBuffer *LogBuffer) LoopProcessLogData(readerName string, startPosition MessagePosition, stopTsNs int64,
+	waitForDataFn func() bool, eachLogDataFn EachLogEntryFuncType) (lastReadPosition MessagePosition, isDone bool, err error) {
+
+	// Register for instant notifications (<1ms latency)
+	notifyChan := logBuffer.RegisterSubscriber(readerName)
+	defer logBuffer.UnregisterSubscriber(readerName)
+
+	// loop through all messages
+	var bytesBuf *bytes.Buffer
+	var bytesBufPooled bool
+	var batchIndex int64
+	lastReadPosition = startPosition
+	var entryCounter int64
+	// caughtUpToDiskHead is set when ReadFromDiskFn last returned without
+	// advancing lastReadPosition, i.e. the disk has no data past our current
+	// position. In that steady state, the loop still re-probes buffer and disk
+	// (external writers can land data on disk without notifying subscribers),
+	// but at caughtUpDiskPollInterval instead of the 250ms health-check tick.
+	// Any progress (disk advance or notification) clears the flag, so the
+	// responsive 250ms cadence resumes for active readers.
+	caughtUpToDiskHead := false
+	defer func() {
+		if bytesBuf != nil && bytesBufPooled {
+			logBuffer.ReleaseMemory(bytesBuf)
+		}
+		// println("LoopProcessLogData", readerName, "sent messages total", entryCounter)
+	}()
+
+	for {
+
+		if bytesBuf != nil && bytesBufPooled {
+			logBuffer.ReleaseMemory(bytesBuf)
+			bytesBuf = nil // keep the deferred release from double-freeing if ReadFromBuffer panics
+		}
+		bytesBuf, batchIndex, bytesBufPooled, err = logBuffer.ReadFromBuffer(lastReadPosition)
+		if err == ResumeFromDiskError {
+			// Try to read from disk if readFromDiskFn is available
+			if logBuffer.ReadFromDiskFn != nil {
+				prevReadPosition := lastReadPosition
+				lastReadPosition, isDone, err = logBuffer.ReadFromDiskFn(lastReadPosition, stopTsNs, eachLogDataFn)
+				if err != nil {
+					return lastReadPosition, isDone, err
+				}
+				if isDone {
+					return lastReadPosition, isDone, nil
+				}
+				if lastReadPosition != prevReadPosition {
+					caughtUpToDiskHead = false
+					continue
+				}
+				if !caughtUpToDiskHead {
+					caughtUpToDiskHead = true
+					glog.V(4).Infof("%s: Caught up to disk head, backing off to %s poll", readerName, caughtUpDiskPollInterval)
+				}
+			} else if logBuffer.HasData() {
+				// HasData() and ReadFromBuffer lock separately, so a racing write can make
+				// HasData() see data the empty-buffer read missed. Re-read; only bail if the
+				// position is genuinely behind the in-memory window (flushed to disk).
+				reBuf, _, rePooled, reErr := logBuffer.ReadFromBuffer(lastReadPosition)
+				if reErr == ResumeFromDiskError {
+					return lastReadPosition, isDone, ResumeFromDiskError
+				}
+				if reBuf != nil && rePooled {
+					logBuffer.ReleaseMemory(reBuf)
+				}
+				continue
+			}
+
+			// Nothing readable anywhere for this cursor: a bounded subscription is
+			// done, same as the caught-up path below. Without this a bounded
+			// subscriber on a buffer that never took a write (empty since process
+			// start, ReadFromDiskFn nil) parks here forever while heartbeats keep
+			// the stream looking alive. The nil error matters: err still holds
+			// ResumeFromDiskError, which callers treat as retry-the-disk-pass.
+			if stopTsNs != 0 {
+				return lastReadPosition, true, nil
+			}
+
+			// CRITICAL: Check if client is still connected
+			if !waitForDataFn() {
+				// Client disconnected - exit cleanly
+				glog.V(4).Infof("%s: Client disconnected after disk read attempt", readerName)
+				return lastReadPosition, true, nil
+			}
+
+			// Wait for notification or timeout. While caught up to disk head,
+			// use the longer poll interval so idle readers don't spin every
+			// 250ms re-probing an empty buffer and disk.
+			waitTimeout := notificationHealthCheckInterval
+			if caughtUpToDiskHead {
+				waitTimeout = caughtUpDiskPollInterval
+			}
+			if logBuffer.awaitNotificationOrTimeoutFor(notifyChan, waitTimeout) {
+				glog.V(3).Infof("%s: Woke up from notification after ResumeFromDiskError", readerName)
+				caughtUpToDiskHead = false
+			}
+			// Silent on timeout while caught up: the re-probe cadence alone is
+			// the signal; no new log line per tick.
+
+			// If the LogBuffer is shutting down, exit cleanly instead of looping
+			// on ResumeFromDiskError. awaitNotificationOrTimeout returns true on
+			// shutdown (shutdownCh closed), which would otherwise spin here
+			// because ReadFromBuffer keeps returning ResumeFromDiskError.
+			if logBuffer.IsStopping() {
+				isDone = true
+				return
+			}
+
+			// Continue to next iteration (don't return ResumeFromDiskError)
+			continue
+		}
+		if err != nil {
+			// Check for buffer corruption error
+			if errors.Is(err, ErrBufferCorrupted) {
+				glog.Errorf("%s: Buffer corruption detected: %v", readerName, err)
+				return lastReadPosition, true, fmt.Errorf("buffer corruption: %w", err)
+			}
+			// Other errors
+			glog.Errorf("%s: ReadFromBuffer error: %v", readerName, err)
+			return lastReadPosition, true, err
+		}
+		readSize := 0
+		if bytesBuf != nil {
+			readSize = bytesBuf.Len()
+		}
+		glog.V(4).Infof("%s ReadFromBuffer at %v offset %d. Read bytes %v batchIndex %d", readerName, lastReadPosition, lastReadPosition.Offset, readSize, batchIndex)
+		if bytesBuf == nil {
+			if batchIndex >= 0 {
+				lastReadPosition = NewMessagePosition(lastReadPosition.Time.UnixNano(), batchIndex)
+			}
+			if stopTsNs != 0 {
+				isDone = true
+				return
+			}
+			lastTsNs := logBuffer.LastTsNs.Load()
+
+			for lastTsNs == logBuffer.LastTsNs.Load() {
+				if !waitForDataFn() {
+					isDone = true
+					return
+				}
+				// Wait for notification or timeout (instant wake-up when data arrives)
+				if logBuffer.awaitNotificationOrTimeout(notifyChan) {
+					glog.V(3).Infof("%s: Woke up from notification (LoopProcessLogData)", readerName)
+				} else if lastTsNs != logBuffer.LastTsNs.Load() {
+					break
+				} else {
+					glog.V(4).Infof("%s: Notification timeout (LoopProcessLogData), rechecking state", readerName)
+				}
+				// Exit the wait loop on shutdown so we don't spin against a
+				// closed shutdownCh.
+				if logBuffer.IsStopping() {
+					isDone = true
+					return
+				}
+			}
+			if logBuffer.IsStopping() {
+				isDone = true
+				return
+			}
+			continue
+		}
+
+		buf := bytesBuf.Bytes()
+		// fmt.Printf("ReadFromBuffer %s by %v size %d\n", readerName, lastReadPosition, len(buf))
+
+		batchSize := 0
+
+		for pos := 0; pos+4 < len(buf); {
+
+			size := util.BytesToUint32(buf[pos : pos+4])
+			if pos+4+int(size) > len(buf) {
+				err = ResumeError
+				glog.Errorf("LoopProcessLogData: %s read buffer %v read %d entries [%d,%d) from [0,%d)", readerName, lastReadPosition, batchSize, pos, pos+int(size)+4, len(buf))
+				return
+			}
+			entryData := buf[pos+4 : pos+4+int(size)]
+
+			logEntry := &filer_pb.LogEntry{}
+			// data/key alias entryData; valid only within eachLogDataFn below.
+			if err = unmarshalLogEntryAliased(entryData, logEntry); err != nil {
+				glog.Errorf("unexpected unmarshal mq_pb.Message: %v", err)
+				pos += 4 + int(size)
+				continue
+			}
+
+			// Handle offset-based filtering for offset-based start positions
+			if startPosition.IsOffsetBased {
+				startOffset := startPosition.GetOffset()
+				if logEntry.Offset < startOffset {
+					// Skip entries before the starting offset
+					pos += 4 + int(size)
+					batchSize++
+					continue
+				}
+			}
+
+			if stopTsNs != 0 && logEntry.TsNs > stopTsNs {
+				isDone = true
+				// println("stopTsNs", stopTsNs, "logEntry.TsNs", logEntry.TsNs)
+				return
+			}
+			lastReadPosition = NewMessagePosition(logEntry.TsNs, batchIndex)
+
+			if isDone, err = eachLogDataFn(logEntry); err != nil {
+				if !errors.Is(err, StopReadingError) {
+					glog.Errorf("LoopProcessLogData: %s process log entry %d key:%q ts_ns:%d offset:%d size:%d: %v", readerName, batchSize+1, logEntry.Key, logEntry.TsNs, logEntry.Offset, len(logEntry.Data), err)
+				}
+				return
+			}
+			if isDone {
+				glog.V(0).Infof("LoopProcessLogData2: %s process log entry %d", readerName, batchSize+1)
+				return
+			}
+
+			pos += 4 + int(size)
+			batchSize++
+			entryCounter++
+
+		}
+
+		glog.V(4).Infof("%s sent messages ts[%+v,%+v] size %d\n", readerName, startPosition, lastReadPosition, batchSize)
+	}
+
+}
+
+// LoopProcessLogDataWithOffset is similar to LoopProcessLogData but provides offset to the callback
+func (logBuffer *LogBuffer) LoopProcessLogDataWithOffset(readerName string, startPosition MessagePosition, stopTsNs int64,
+	waitForDataFn func() bool, eachLogDataFn EachLogEntryWithOffsetFuncType) (lastReadPosition MessagePosition, isDone bool, err error) {
+	glog.V(4).Infof("LoopProcessLogDataWithOffset started for %s, startPosition=%v", readerName, startPosition)
+
+	// Register for instant notifications (<1ms latency)
+	notifyChan := logBuffer.RegisterSubscriber(readerName)
+	defer logBuffer.UnregisterSubscriber(readerName)
+
+	// loop through all messages
+	var bytesBuf *bytes.Buffer
+	var bytesBufPooled bool
+	var offset int64
+	lastReadPosition = startPosition
+	var entryCounter int64
+	// See LoopProcessLogData for the caughtUpToDiskHead invariant.
+	caughtUpToDiskHead := false
+	defer func() {
+		if bytesBuf != nil && bytesBufPooled {
+			logBuffer.ReleaseMemory(bytesBuf)
+		}
+		// println("LoopProcessLogDataWithOffset", readerName, "sent messages total", entryCounter)
+	}()
+
+	for {
+		// Check stopTsNs at the beginning of each iteration
+		// This ensures we exit immediately if the stop time is in the past
+		if stopTsNs != 0 && time.Now().UnixNano() > stopTsNs {
+			isDone = true
+			return
+		}
+
+		if bytesBuf != nil && bytesBufPooled {
+			logBuffer.ReleaseMemory(bytesBuf)
+			bytesBuf = nil // keep the deferred release from double-freeing if ReadFromBuffer panics
+		}
+		bytesBuf, offset, bytesBufPooled, err = logBuffer.ReadFromBuffer(lastReadPosition)
+		glog.V(4).Infof("ReadFromBuffer for %s returned bytesBuf=%v, offset=%d, err=%v", readerName, bytesBuf != nil, offset, err)
+
+		// Check for buffer corruption error before other error handling
+		if err != nil && errors.Is(err, ErrBufferCorrupted) {
+			glog.Errorf("%s: Buffer corruption detected: %v", readerName, err)
+			return lastReadPosition, true, fmt.Errorf("buffer corruption: %w", err)
+		}
+
+		if err == ResumeFromDiskError {
+			// Try to read from disk if readFromDiskFn is available
+			if logBuffer.ReadFromDiskFn != nil {
+				prevReadPosition := lastReadPosition
+				// Wrap eachLogDataFn to match the expected signature
+				diskReadFn := func(logEntry *filer_pb.LogEntry) (bool, error) {
+					return eachLogDataFn(logEntry, logEntry.Offset)
+				}
+				lastReadPosition, isDone, err = logBuffer.ReadFromDiskFn(lastReadPosition, stopTsNs, diskReadFn)
+				if err != nil {
+					return lastReadPosition, isDone, err
+				}
+				if isDone {
+					return lastReadPosition, isDone, nil
+				}
+				if lastReadPosition != prevReadPosition {
+					caughtUpToDiskHead = false
+					continue
+				}
+				if !caughtUpToDiskHead {
+					caughtUpToDiskHead = true
+					glog.V(4).Infof("%s: Caught up to disk head, backing off to %s poll", readerName, caughtUpDiskPollInterval)
+				}
+			} else if logBuffer.HasData() {
+				// HasData() and ReadFromBuffer lock separately, so a racing write can make
+				// HasData() see data the empty-buffer read missed. Re-read; only bail if the
+				// position is genuinely behind the in-memory window (flushed to disk).
+				reBuf, _, rePooled, reErr := logBuffer.ReadFromBuffer(lastReadPosition)
+				if reErr == ResumeFromDiskError {
+					return lastReadPosition, isDone, ResumeFromDiskError
+				}
+				if reBuf != nil && rePooled {
+					logBuffer.ReleaseMemory(reBuf)
+				}
+				continue
+			}
+
+			// CRITICAL: Check if client is still connected after disk read
+			if !waitForDataFn() {
+				// Client disconnected - exit cleanly
+				glog.V(4).Infof("%s: Client disconnected after disk read", readerName)
+				return lastReadPosition, true, nil
+			}
+
+			// Wait for notification or timeout. While caught up to disk head,
+			// use the longer poll interval so idle readers don't spin every
+			// 250ms re-probing an empty buffer and disk.
+			waitTimeout := notificationHealthCheckInterval
+			if caughtUpToDiskHead {
+				waitTimeout = caughtUpDiskPollInterval
+			}
+			if logBuffer.awaitNotificationOrTimeoutFor(notifyChan, waitTimeout) {
+				glog.V(3).Infof("%s: Woke up from notification after disk read", readerName)
+				caughtUpToDiskHead = false
+			}
+			// Silent on timeout while caught up: the re-probe cadence alone is
+			// the signal; no new log line per tick.
+
+			// Exit cleanly on shutdown so we don't loop on ResumeFromDiskError.
+			if logBuffer.IsStopping() {
+				return lastReadPosition, true, nil
+			}
+
+			// Continue to next iteration (don't return ResumeFromDiskError)
+			continue
+		}
+		readSize := 0
+		if bytesBuf != nil {
+			readSize = bytesBuf.Len()
+		}
+		glog.V(4).Infof("%s ReadFromBuffer at %v posOffset %d. Read bytes %v bufferOffset %d", readerName, lastReadPosition, lastReadPosition.Offset, readSize, offset)
+		if bytesBuf == nil {
+			// CRITICAL: Check if subscription is still active BEFORE waiting
+			// This prevents infinite loops when client has disconnected
+			if !waitForDataFn() {
+				glog.V(4).Infof("%s: waitForDataFn returned false, subscription ending", readerName)
+				return lastReadPosition, true, nil
+			}
+
+			if offset >= 0 {
+				lastReadPosition = NewMessagePosition(lastReadPosition.Time.UnixNano(), offset)
+			}
+			if stopTsNs != 0 {
+				isDone = true
+				return
+			}
+
+			// If we're reading offset-based and there's no data in LogBuffer,
+			// return ResumeFromDiskError to let Subscribe try reading from disk again.
+			// This prevents infinite blocking when all data is on disk (e.g., after restart).
+			if startPosition.IsOffsetBased {
+				glog.V(4).Infof("%s: No data in LogBuffer for offset-based read at %v, checking if client still connected", readerName, lastReadPosition)
+				// Check if client is still connected before busy-looping
+				if !waitForDataFn() {
+					glog.V(4).Infof("%s: Client disconnected, stopping offset-based read", readerName)
+					return lastReadPosition, true, nil
+				}
+				// Wait for notification or timeout (instant wake-up when data arrives)
+				if logBuffer.awaitNotificationOrTimeout(notifyChan) {
+					glog.V(3).Infof("%s: Woke up from notification for offset-based read", readerName)
+				} else {
+					glog.V(4).Infof("%s: Notification timeout for offset-based, rechecking state", readerName)
+				}
+				// On shutdown, exit cleanly instead of returning ResumeFromDiskError.
+				if logBuffer.IsStopping() {
+					return lastReadPosition, true, nil
+				}
+				return lastReadPosition, isDone, ResumeFromDiskError
+			}
+
+			lastTsNs := logBuffer.LastTsNs.Load()
+
+			for lastTsNs == logBuffer.LastTsNs.Load() {
+				if !waitForDataFn() {
+					glog.V(4).Infof("%s: Client disconnected during timestamp wait", readerName)
+					return lastReadPosition, true, nil
+				}
+				// Wait for notification or timeout (instant wake-up when data arrives)
+				if logBuffer.awaitNotificationOrTimeout(notifyChan) {
+					glog.V(3).Infof("%s: Woke up from notification (main loop)", readerName)
+				} else if lastTsNs != logBuffer.LastTsNs.Load() {
+					break
+				} else {
+					glog.V(4).Infof("%s: Notification timeout (main loop), rechecking state", readerName)
+				}
+				// Exit the wait loop on shutdown so we don't spin against a
+				// closed shutdownCh.
+				if logBuffer.IsStopping() {
+					glog.V(4).Infof("%s: LogBuffer is stopping", readerName)
+					return lastReadPosition, true, nil
+				}
+			}
+			if logBuffer.IsStopping() {
+				glog.V(4).Infof("%s: LogBuffer is stopping", readerName)
+				return lastReadPosition, true, nil
+			}
+			continue
+		}
+
+		buf := bytesBuf.Bytes()
+		// fmt.Printf("ReadFromBuffer %s by %v size %d\n", readerName, lastReadPosition, len(buf))
+		glog.V(4).Infof("Processing buffer with %d bytes for %s", len(buf), readerName)
+
+		// If buffer is empty, check if client is still connected before looping
+		if len(buf) == 0 {
+			glog.V(4).Infof("Empty buffer for %s, checking if client still connected", readerName)
+			if !waitForDataFn() {
+				glog.V(4).Infof("%s: Client disconnected on empty buffer", readerName)
+				return lastReadPosition, true, nil
+			}
+			if logBuffer.awaitNotificationOrTimeout(notifyChan) {
+				glog.V(3).Infof("%s: Woke up from notification on empty buffer", readerName)
+			} else {
+				glog.V(4).Infof("%s: Empty buffer timeout, rechecking state", readerName)
+			}
+			// Exit cleanly on shutdown to avoid an idle spin on the empty buffer.
+			if logBuffer.IsStopping() {
+				return lastReadPosition, true, nil
+			}
+			continue
+		}
+
+		batchSize := 0
+
+		for pos := 0; pos+4 < len(buf); {
+
+			size := util.BytesToUint32(buf[pos : pos+4])
+			if pos+4+int(size) > len(buf) {
+				err = ResumeError
+				glog.Errorf("LoopProcessLogDataWithOffset: %s read buffer %v read %d entries [%d,%d) from [0,%d)", readerName, lastReadPosition, batchSize, pos, pos+int(size)+4, len(buf))
+				return
+			}
+			entryData := buf[pos+4 : pos+4+int(size)]
+
+			logEntry := &filer_pb.LogEntry{}
+			// data/key alias entryData; valid only within eachLogDataFn below.
+			if err = unmarshalLogEntryAliased(entryData, logEntry); err != nil {
+				glog.Errorf("unexpected unmarshal mq_pb.Message: %v", err)
+				pos += 4 + int(size)
+				continue
+			}
+
+			glog.V(4).Infof("Unmarshaled log entry %d: TsNs=%d, Offset=%d, Key=%s", batchSize+1, logEntry.TsNs, logEntry.Offset, string(logEntry.Key))
+
+			// Handle offset-based filtering for offset-based start positions
+			if startPosition.IsOffsetBased {
+				startOffset := startPosition.GetOffset()
+				glog.V(4).Infof("Offset-based filtering: logEntry.Offset=%d, startOffset=%d", logEntry.Offset, startOffset)
+				if logEntry.Offset < startOffset {
+					// Skip entries before the starting offset
+					glog.V(4).Infof("Skipping entry due to offset filter")
+					pos += 4 + int(size)
+					batchSize++
+					continue
+				}
+			}
+
+			if stopTsNs != 0 && logEntry.TsNs > stopTsNs {
+				glog.V(4).Infof("Stopping due to stopTsNs")
+				isDone = true
+				// println("stopTsNs", stopTsNs, "logEntry.TsNs", logEntry.TsNs)
+				return
+			}
+			// Use logEntry.Offset + 1 to move PAST the current entry
+			// This prevents infinite loops where we keep requesting the same offset
+			lastReadPosition = NewMessagePosition(logEntry.TsNs, logEntry.Offset+1)
+
+			glog.V(4).Infof("Calling eachLogDataFn for entry at offset %d, next position will be %d", logEntry.Offset, logEntry.Offset+1)
+			if isDone, err = eachLogDataFn(logEntry, logEntry.Offset); err != nil {
+				if !errors.Is(err, StopReadingError) {
+					glog.Errorf("LoopProcessLogDataWithOffset: %s process log entry %d key:%q ts_ns:%d offset:%d size:%d: %v", readerName, batchSize+1, logEntry.Key, logEntry.TsNs, logEntry.Offset, len(logEntry.Data), err)
+				}
+				return
+			}
+			if isDone {
+				glog.V(0).Infof("LoopProcessLogDataWithOffset: %s process log entry %d", readerName, batchSize+1)
+				return
+			}
+
+			pos += 4 + int(size)
+			batchSize++
+			entryCounter++
+
+		}
+
+	}
+
+}

weed/util/log_buffer/log_read_integration_test.go

@@ -0,0 +1,363 @@
+package log_buffer
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+// TestConcurrentProducerConsumer simulates the integration test scenario:
+// - One producer writing messages continuously
+// - Multiple consumers reading from different offsets
+// - Consumers reading sequentially (like Kafka consumers)
+func TestConcurrentProducerConsumer(t *testing.T) {
+	lb := NewLogBuffer("integration-test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	const numMessages = 1000
+	const numConsumers = 2
+	const messagesPerConsumer = numMessages / numConsumers
+
+	// Start producer
+	producerDone := make(chan bool)
+	go func() {
+		for i := 0; i < numMessages; i++ {
+			entry := &filer_pb.LogEntry{
+				TsNs:   time.Now().UnixNano(),
+				Key:    []byte("key"),
+				Data:   []byte("value"),
+				Offset: int64(i),
+			}
+			if err := lb.AddLogEntryToBuffer(entry); err != nil {
+				t.Errorf("Failed to add log entry: %v", err)
+				return
+			}
+			time.Sleep(1 * time.Millisecond) // Simulate production rate
+		}
+		producerDone <- true
+	}()
+
+	// Start consumers
+	consumerWg := sync.WaitGroup{}
+	consumerErrors := make(chan error, numConsumers)
+	consumedCounts := make([]int64, numConsumers)
+
+	for consumerID := 0; consumerID < numConsumers; consumerID++ {
+		consumerWg.Add(1)
+		go func(id int, startOffset int64, endOffset int64) {
+			defer consumerWg.Done()
+
+			currentOffset := startOffset
+			for currentOffset < endOffset {
+				// Read 10 messages at a time (like integration test)
+				messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(currentOffset, 10, 10240)
+				if err != nil {
+					consumerErrors <- err
+					return
+				}
+
+				if len(messages) == 0 {
+					// No data yet, wait a bit
+					time.Sleep(5 * time.Millisecond)
+					continue
+				}
+
+				// Count only messages in this consumer's assigned range
+				messagesInRange := 0
+				for i, msg := range messages {
+					if msg.Offset >= startOffset && msg.Offset < endOffset {
+						messagesInRange++
+						expectedOffset := currentOffset + int64(i)
+						if msg.Offset != expectedOffset {
+							t.Errorf("Consumer %d: Expected offset %d, got %d", id, expectedOffset, msg.Offset)
+						}
+					}
+				}
+
+				atomic.AddInt64(&consumedCounts[id], int64(messagesInRange))
+				currentOffset = nextOffset
+			}
+		}(consumerID, int64(consumerID*messagesPerConsumer), int64((consumerID+1)*messagesPerConsumer))
+	}
+
+	// Wait for producer to finish
+	<-producerDone
+
+	// Wait for consumers (with timeout)
+	done := make(chan bool)
+	go func() {
+		consumerWg.Wait()
+		done <- true
+	}()
+
+	select {
+	case <-done:
+		// Success
+	case err := <-consumerErrors:
+		t.Fatalf("Consumer error: %v", err)
+	case <-time.After(10 * time.Second):
+		t.Fatal("Timeout waiting for consumers to finish")
+	}
+
+	// Verify all messages were consumed
+	totalConsumed := int64(0)
+	for i, count := range consumedCounts {
+		t.Logf("Consumer %d consumed %d messages", i, count)
+		totalConsumed += count
+	}
+
+	if totalConsumed != numMessages {
+		t.Errorf("Expected to consume %d messages, but consumed %d", numMessages, totalConsumed)
+	}
+}
+
+// TestBackwardSeeksWhileProducing simulates consumer rebalancing where
+// consumers seek backward to earlier offsets while producer is still writing
+func TestBackwardSeeksWhileProducing(t *testing.T) {
+	lb := NewLogBuffer("backward-seek-test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	const numMessages = 500
+	const numSeeks = 10
+
+	// Start producer
+	producerDone := make(chan bool)
+	go func() {
+		for i := 0; i < numMessages; i++ {
+			entry := &filer_pb.LogEntry{
+				TsNs:   time.Now().UnixNano(),
+				Key:    []byte("key"),
+				Data:   []byte("value"),
+				Offset: int64(i),
+			}
+			if err := lb.AddLogEntryToBuffer(entry); err != nil {
+				t.Errorf("Failed to add log entry: %v", err)
+				return
+			}
+			time.Sleep(1 * time.Millisecond)
+		}
+		producerDone <- true
+	}()
+
+	// Consumer that seeks backward periodically
+	consumerDone := make(chan bool)
+	readOffsets := make(map[int64]int) // Track how many times each offset was read
+
+	go func() {
+		currentOffset := int64(0)
+		seeksRemaining := numSeeks
+
+		for currentOffset < numMessages {
+			// Read some messages
+			messages, nextOffset, _, endOfPartition, err := lb.ReadMessagesAtOffset(currentOffset, 10, 10240)
+			if err != nil {
+				// For stateless reads, "offset out of range" means data not in memory yet
+				// This is expected when reading historical data or before production starts
+				time.Sleep(5 * time.Millisecond)
+				continue
+			}
+
+			if len(messages) == 0 {
+				// No data available yet or caught up to producer
+				if !endOfPartition {
+					// Data might be coming, wait
+					time.Sleep(5 * time.Millisecond)
+				} else {
+					// At end of partition, wait for more production
+					time.Sleep(5 * time.Millisecond)
+				}
+				continue
+			}
+
+			// Track read offsets
+			for _, msg := range messages {
+				readOffsets[msg.Offset]++
+			}
+
+			// Periodically seek backward (simulating rebalancing)
+			if seeksRemaining > 0 && nextOffset > 50 && nextOffset%100 == 0 {
+				seekOffset := nextOffset - 20
+				t.Logf("Seeking backward from %d to %d", nextOffset, seekOffset)
+				currentOffset = seekOffset
+				seeksRemaining--
+			} else {
+				currentOffset = nextOffset
+			}
+		}
+
+		consumerDone <- true
+	}()
+
+	// Wait for both
+	<-producerDone
+	<-consumerDone
+
+	// Verify each offset was read at least once
+	for i := int64(0); i < numMessages; i++ {
+		if readOffsets[i] == 0 {
+			t.Errorf("Offset %d was never read", i)
+		}
+	}
+
+	t.Logf("Total unique offsets read: %d out of %d", len(readOffsets), numMessages)
+}
+
+// TestHighConcurrencyReads simulates multiple consumers reading from
+// different offsets simultaneously (stress test)
+func TestHighConcurrencyReads(t *testing.T) {
+	lb := NewLogBuffer("high-concurrency-test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	const numMessages = 1000
+	const numReaders = 10
+
+	// Pre-populate buffer
+	for i := 0; i < numMessages; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		if err := lb.AddLogEntryToBuffer(entry); err != nil {
+			t.Fatalf("Failed to add log entry: %v", err)
+		}
+	}
+
+	// Start many concurrent readers at different offsets
+	wg := sync.WaitGroup{}
+	errors := make(chan error, numReaders)
+
+	for reader := 0; reader < numReaders; reader++ {
+		wg.Add(1)
+		go func(startOffset int64) {
+			defer wg.Done()
+
+			// Read 100 messages from this offset
+			currentOffset := startOffset
+			readCount := 0
+
+			for readCount < 100 && currentOffset < numMessages {
+				messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(currentOffset, 10, 10240)
+				if err != nil {
+					errors <- err
+					return
+				}
+
+				// Verify offsets are sequential
+				for i, msg := range messages {
+					expected := currentOffset + int64(i)
+					if msg.Offset != expected {
+						t.Errorf("Reader at %d: expected offset %d, got %d", startOffset, expected, msg.Offset)
+					}
+				}
+
+				readCount += len(messages)
+				currentOffset = nextOffset
+			}
+		}(int64(reader * 10))
+	}
+
+	// Wait with timeout
+	done := make(chan bool)
+	go func() {
+		wg.Wait()
+		done <- true
+	}()
+
+	select {
+	case <-done:
+		// Success
+	case err := <-errors:
+		t.Fatalf("Reader error: %v", err)
+	case <-time.After(10 * time.Second):
+		t.Fatal("Timeout waiting for readers")
+	}
+}
+
+// TestRepeatedReadsAtSameOffset simulates what happens when Kafka
+// consumer re-fetches the same offset multiple times (due to timeouts or retries)
+func TestRepeatedReadsAtSameOffset(t *testing.T) {
+	lb := NewLogBuffer("repeated-reads-test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	const numMessages = 100
+
+	// Pre-populate buffer
+	for i := 0; i < numMessages; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		if err := lb.AddLogEntryToBuffer(entry); err != nil {
+			t.Fatalf("Failed to add log entry: %v", err)
+		}
+	}
+
+	// Read the same offset multiple times concurrently
+	const numReads = 10
+	const testOffset = int64(50)
+
+	wg := sync.WaitGroup{}
+	results := make([][]*filer_pb.LogEntry, numReads)
+
+	for i := 0; i < numReads; i++ {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			messages, _, _, _, err := lb.ReadMessagesAtOffset(testOffset, 10, 10240)
+			if err != nil {
+				t.Errorf("Read %d error: %v", idx, err)
+				return
+			}
+			results[idx] = messages
+		}(i)
+	}
+
+	wg.Wait()
+
+	// Verify all reads returned the same data
+	firstRead := results[0]
+	for i := 1; i < numReads; i++ {
+		if len(results[i]) != len(firstRead) {
+			t.Errorf("Read %d returned %d messages, expected %d", i, len(results[i]), len(firstRead))
+		}
+
+		for j := range results[i] {
+			if results[i][j].Offset != firstRead[j].Offset {
+				t.Errorf("Read %d message %d has offset %d, expected %d",
+					i, j, results[i][j].Offset, firstRead[j].Offset)
+			}
+		}
+	}
+}
+
+// TestEmptyPartitionPolling simulates consumers polling empty partitions
+// waiting for data (common in Kafka)
+func TestEmptyPartitionPolling(t *testing.T) {
+	lb := NewLogBuffer("empty-partition-test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+	lb.bufferStartOffset = 0
+	lb.offset = 0
+
+	// Try to read from empty partition
+	messages, nextOffset, _, endOfPartition, err := lb.ReadMessagesAtOffset(0, 10, 10240)
+
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	if len(messages) != 0 {
+		t.Errorf("Expected 0 messages, got %d", len(messages))
+	}
+	if nextOffset != 0 {
+		t.Errorf("Expected nextOffset=0, got %d", nextOffset)
+	}
+	if !endOfPartition {
+		t.Error("Expected endOfPartition=true for future offset")
+	}
+}

weed/util/log_buffer/log_read_stateless.go

@@ -0,0 +1,524 @@
+package log_buffer
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/proto"
+)
+
+// ReadMessagesAtOffset provides Kafka-style stateless reads from LogBuffer
+// Each call is completely independent - no state maintained between calls
+// Thread-safe for concurrent reads at different offsets
+//
+// This is the recommended API for stateless clients like Kafka gateway
+// Unlike Subscribe loops, this:
+// 1. Returns immediately with available data (or empty if none)
+// 2. Does not maintain any session state
+// 3. Safe for concurrent calls
+// 4. No cancellation/restart complexity
+//
+// Returns:
+// - messages: Array of messages starting at startOffset
+// - nextOffset: Offset to use for next fetch
+// - highWaterMark: Highest offset available in partition
+// - endOfPartition: True if no more data available
+// - err: Any error encountered
+func (logBuffer *LogBuffer) ReadMessagesAtOffset(startOffset int64, maxMessages int, maxBytes int) (
+	messages []*filer_pb.LogEntry,
+	nextOffset int64,
+	highWaterMark int64,
+	endOfPartition bool,
+	err error,
+) {
+	// Quick validation
+	if maxMessages <= 0 {
+		maxMessages = 100 // Default reasonable batch size
+	}
+	if maxBytes <= 0 {
+		maxBytes = 4 * 1024 * 1024 // 4MB default
+	}
+
+	messages = make([]*filer_pb.LogEntry, 0, maxMessages)
+	nextOffset = startOffset
+
+	// Try to read from in-memory buffers first (hot path)
+	logBuffer.RLock()
+	currentBufferEnd := logBuffer.offset
+	bufferStartOffset := logBuffer.bufferStartOffset
+	highWaterMark = currentBufferEnd
+
+	// Special case: empty buffer (no data written yet)
+	if currentBufferEnd == 0 && bufferStartOffset == 0 && logBuffer.pos == 0 {
+		logBuffer.RUnlock()
+		// Return empty result - partition exists but has no data yet
+		// Preserve the requested offset in nextOffset
+		return messages, startOffset, 0, true, nil
+	}
+
+	// Check if requested offset is in current buffer
+	if startOffset >= bufferStartOffset && startOffset < currentBufferEnd {
+		// Read from current buffer
+		glog.V(4).Infof("[StatelessRead] Reading from current buffer: start=%d, end=%d",
+			bufferStartOffset, currentBufferEnd)
+
+		if logBuffer.pos > 0 {
+			// Make a copy of the buffer to avoid concurrent modification
+			bufCopy := make([]byte, logBuffer.pos)
+			copy(bufCopy, logBuffer.buf[:logBuffer.pos])
+			logBuffer.RUnlock() // Release lock early
+
+			// Parse messages from buffer copy
+			messages, nextOffset, _, err = parseMessagesFromBuffer(
+				bufCopy, startOffset, maxMessages, maxBytes)
+
+			if err != nil {
+				return nil, startOffset, highWaterMark, false, err
+			}
+
+			glog.V(4).Infof("[StatelessRead] Read %d messages from current buffer, nextOffset=%d",
+				len(messages), nextOffset)
+
+			// Check if we reached the end
+			endOfPartition = (nextOffset >= currentBufferEnd) && (len(messages) == 0 || len(messages) < maxMessages)
+			return messages, nextOffset, highWaterMark, endOfPartition, nil
+		}
+
+		// Buffer is empty but offset is in range - check previous buffers
+		logBuffer.RUnlock()
+
+		// Try previous buffers
+		logBuffer.RLock()
+		for _, prevBuf := range logBuffer.prevBuffers.buffers {
+			if startOffset >= prevBuf.startOffset && startOffset <= prevBuf.offset {
+				if prevBuf.size > 0 {
+					// Found in previous buffer
+					bufCopy := make([]byte, prevBuf.size)
+					copy(bufCopy, prevBuf.buf[:prevBuf.size])
+					logBuffer.RUnlock()
+
+					messages, nextOffset, _, err = parseMessagesFromBuffer(
+						bufCopy, startOffset, maxMessages, maxBytes)
+
+					if err != nil {
+						return nil, startOffset, highWaterMark, false, err
+					}
+
+					glog.V(4).Infof("[StatelessRead] Read %d messages from previous buffer, nextOffset=%d",
+						len(messages), nextOffset)
+
+					endOfPartition = false // More data might be in current buffer
+					return messages, nextOffset, highWaterMark, endOfPartition, nil
+				}
+				// Empty previous buffer means data was flushed to disk - fall through to disk read
+				glog.V(2).Infof("[StatelessRead] Data at offset %d was flushed, attempting disk read", startOffset)
+				break
+			}
+		}
+		logBuffer.RUnlock()
+
+		// Data not in memory - attempt disk read if configured
+		// Don't return error here - data may be on disk!
+		// Fall through to disk read logic below
+		glog.V(2).Infof("[StatelessRead] Data at offset %d not in memory (buffer: %d-%d), attempting disk read",
+			startOffset, bufferStartOffset, currentBufferEnd)
+		// Don't return error - continue to disk read check below
+	} else {
+		// Offset is not in current buffer - check previous buffers FIRST before going to disk
+		// This handles the case where data was just flushed but is still in prevBuffers
+
+		for _, prevBuf := range logBuffer.prevBuffers.buffers {
+			if startOffset >= prevBuf.startOffset && startOffset <= prevBuf.offset {
+				if prevBuf.size > 0 {
+					// Found in previous buffer!
+					bufCopy := make([]byte, prevBuf.size)
+					copy(bufCopy, prevBuf.buf[:prevBuf.size])
+					logBuffer.RUnlock()
+
+					messages, nextOffset, _, err = parseMessagesFromBuffer(
+						bufCopy, startOffset, maxMessages, maxBytes)
+
+					if err != nil {
+						return nil, startOffset, highWaterMark, false, err
+					}
+
+					endOfPartition = false // More data might exist
+					return messages, nextOffset, highWaterMark, endOfPartition, nil
+				}
+				// Empty previous buffer - data was flushed to disk
+				glog.V(2).Infof("[StatelessRead] Found empty previous buffer for offset %d, will try disk", startOffset)
+				break
+			}
+		}
+		logBuffer.RUnlock()
+	}
+
+	// If we get here, unlock if not already unlocked
+	// (Note: logBuffer.RUnlock() was called above in all paths)
+
+	// Data not in memory - try disk read
+	// This handles two cases:
+	// 1. startOffset < bufferStartOffset: Historical data
+	// 2. startOffset in buffer range but not in memory: Data was flushed (from fall-through above)
+	if startOffset < currentBufferEnd {
+		// Historical data or flushed data - try to read from disk if ReadFromDiskFn is configured
+		if startOffset < bufferStartOffset {
+			glog.Errorf("[StatelessRead] CASE 1: Historical data - offset %d < bufferStart %d",
+				startOffset, bufferStartOffset)
+		} else {
+			glog.Errorf("[StatelessRead] CASE 2: Flushed data - offset %d in range [%d, %d) but not in memory",
+				startOffset, bufferStartOffset, currentBufferEnd)
+		}
+
+		// Check if disk read function is configured
+		if logBuffer.ReadFromDiskFn == nil {
+			glog.Errorf("[StatelessRead] CRITICAL: ReadFromDiskFn is NIL! Cannot read from disk.")
+			if startOffset < bufferStartOffset {
+				return messages, startOffset, highWaterMark, false, fmt.Errorf("offset %d too old (earliest in-memory: %d), and ReadFromDiskFn is nil",
+					startOffset, bufferStartOffset)
+			}
+			return messages, startOffset, highWaterMark, false, fmt.Errorf("offset %d not in memory (buffer: %d-%d), and ReadFromDiskFn is nil",
+				startOffset, bufferStartOffset, currentBufferEnd)
+		}
+
+		// Read from disk (this is async/non-blocking if the ReadFromDiskFn is properly implemented)
+		// The ReadFromDiskFn should handle its own timeouts and not block indefinitely
+		diskMessages, diskNextOffset, diskErr := readHistoricalDataFromDisk(
+			logBuffer, startOffset, maxMessages, maxBytes, highWaterMark)
+
+		if diskErr != nil {
+			glog.Errorf("[StatelessRead] CRITICAL: Disk read FAILED for offset %d: %v", startOffset, diskErr)
+			// IMPORTANT: Return retryable error instead of silently returning empty!
+			return messages, startOffset, highWaterMark, false, fmt.Errorf("disk read failed for offset %d: %v", startOffset, diskErr)
+		}
+
+		if len(diskMessages) == 0 {
+			glog.Errorf("[StatelessRead] WARNING: Disk read returned 0 messages for offset %d (HWM=%d, bufferStart=%d)",
+				startOffset, highWaterMark, bufferStartOffset)
+		}
+
+		// Return disk data
+		endOfPartition = diskNextOffset >= bufferStartOffset && len(diskMessages) < maxMessages
+		return diskMessages, diskNextOffset, highWaterMark, endOfPartition, nil
+	}
+
+	// startOffset >= currentBufferEnd - future offset, no data available yet
+	glog.V(4).Infof("[StatelessRead] Future offset %d >= buffer end %d, no data available",
+		startOffset, currentBufferEnd)
+	return messages, startOffset, highWaterMark, true, nil
+}
+
+// readHistoricalDataFromDisk reads messages from disk for historical offsets
+// This is called when the requested offset is older than what's in memory
+// Uses an in-memory cache to avoid repeated disk I/O for the same chunks
+func readHistoricalDataFromDisk(
+	logBuffer *LogBuffer,
+	startOffset int64,
+	maxMessages int,
+	maxBytes int,
+	highWaterMark int64,
+) (messages []*filer_pb.LogEntry, nextOffset int64, err error) {
+	const chunkSize = 1000 // Size of each cached chunk
+
+	// Calculate chunk start offset (aligned to chunkSize boundary)
+	chunkStartOffset := (startOffset / chunkSize) * chunkSize
+
+	// Try to get from cache first
+	cachedMessages, cacheHit := getCachedDiskChunk(logBuffer, chunkStartOffset)
+
+	if cacheHit {
+		// Found in cache - extract requested messages
+		result, nextOff, err := extractMessagesFromCache(cachedMessages, startOffset, maxMessages, maxBytes)
+
+		if err != nil {
+			// CRITICAL: Cache extraction failed because requested offset is BEYOND cached chunk
+			// This means disk files only contain partial data (e.g., 1000-1763) and the
+			// requested offset (e.g., 1764) is in a gap between disk and memory.
+			//
+			// SOLUTION: Return empty result with NO ERROR to let ReadMessagesAtOffset
+			// continue to check memory buffers. The data might be in memory even though
+			// it's not on disk.
+			glog.Errorf("[DiskCache] Offset %d is beyond cached chunk (start=%d, size=%d)",
+				startOffset, chunkStartOffset, len(cachedMessages))
+
+			// Return empty but NO ERROR - this signals "not on disk, try memory"
+			return nil, startOffset, nil
+		}
+
+		// Success - return cached data
+		return result, nextOff, nil
+	}
+
+	// Not in cache - read entire chunk from disk for caching
+	chunkMessages := make([]*filer_pb.LogEntry, 0, chunkSize)
+	chunkNextOffset := chunkStartOffset
+
+	// Create a position for the chunk start
+	chunkPosition := MessagePosition{
+		IsOffsetBased: true,
+		Offset:        chunkStartOffset,
+	}
+
+	// Define callback to collect the entire chunk
+	eachMessageFn := func(logEntry *filer_pb.LogEntry) (isDone bool, err error) {
+		// Read up to chunkSize messages for caching
+		if len(chunkMessages) >= chunkSize {
+			return true, nil
+		}
+
+		chunkMessages = append(chunkMessages, logEntry)
+		chunkNextOffset++
+
+		// Continue reading the chunk
+		return false, nil
+	}
+
+	// Read chunk from disk
+	_, _, readErr := logBuffer.ReadFromDiskFn(chunkPosition, 0, eachMessageFn)
+
+	if readErr != nil {
+		glog.Errorf("[DiskRead] CRITICAL: ReadFromDiskFn returned ERROR: %v", readErr)
+		return nil, startOffset, fmt.Errorf("failed to read from disk: %w", readErr)
+	}
+
+	// Cache the chunk for future reads
+	if len(chunkMessages) > 0 {
+		cacheDiskChunk(logBuffer, chunkStartOffset, chunkNextOffset-1, chunkMessages)
+	} else {
+		glog.Errorf("[DiskRead] WARNING: ReadFromDiskFn returned 0 messages for chunkStart=%d", chunkStartOffset)
+	}
+
+	// Extract requested messages from the chunk
+	result, resNextOffset, resErr := extractMessagesFromCache(chunkMessages, startOffset, maxMessages, maxBytes)
+	return result, resNextOffset, resErr
+}
+
+// getCachedDiskChunk retrieves a cached disk chunk if available
+func getCachedDiskChunk(logBuffer *LogBuffer, chunkStartOffset int64) ([]*filer_pb.LogEntry, bool) {
+	logBuffer.diskChunkCache.mu.RLock()
+	defer logBuffer.diskChunkCache.mu.RUnlock()
+
+	if chunk, exists := logBuffer.diskChunkCache.chunks[chunkStartOffset]; exists {
+		// Update last access time
+		chunk.lastAccess = time.Now()
+		return chunk.messages, true
+	}
+
+	return nil, false
+}
+
+// cacheDiskChunk stores a disk chunk in the cache with LRU eviction
+func cacheDiskChunk(logBuffer *LogBuffer, startOffset, endOffset int64, messages []*filer_pb.LogEntry) {
+	logBuffer.diskChunkCache.mu.Lock()
+	defer logBuffer.diskChunkCache.mu.Unlock()
+
+	// Check if we need to evict old chunks (LRU policy)
+	if len(logBuffer.diskChunkCache.chunks) >= logBuffer.diskChunkCache.maxChunks {
+		// Find least recently used chunk
+		var oldestOffset int64
+		var oldestTime time.Time
+		first := true
+
+		for offset, chunk := range logBuffer.diskChunkCache.chunks {
+			if first || chunk.lastAccess.Before(oldestTime) {
+				oldestOffset = offset
+				oldestTime = chunk.lastAccess
+				first = false
+			}
+		}
+
+		// Evict oldest chunk
+		delete(logBuffer.diskChunkCache.chunks, oldestOffset)
+		glog.V(4).Infof("[DiskCache] Evicted chunk at offset %d (LRU)", oldestOffset)
+	}
+
+	// Store new chunk
+	logBuffer.diskChunkCache.chunks[startOffset] = &CachedDiskChunk{
+		startOffset: startOffset,
+		endOffset:   endOffset,
+		messages:    messages,
+		lastAccess:  time.Now(),
+	}
+}
+
+// extractMessagesFromCache extracts requested messages from a cached chunk
+// chunkMessages contains messages starting from the chunk's aligned start offset
+// We need to skip to the requested startOffset within the chunk
+func extractMessagesFromCache(chunkMessages []*filer_pb.LogEntry, startOffset int64, maxMessages, maxBytes int) ([]*filer_pb.LogEntry, int64, error) {
+	const chunkSize = 1000
+	chunkStartOffset := (startOffset / chunkSize) * chunkSize
+
+	// Calculate position within chunk
+	positionInChunk := int(startOffset - chunkStartOffset)
+
+	// Check if requested offset is within the chunk
+	if positionInChunk < 0 {
+		glog.Errorf("[DiskCache] CRITICAL: Requested offset %d is BEFORE chunk start %d (positionInChunk=%d < 0)",
+			startOffset, chunkStartOffset, positionInChunk)
+		return nil, startOffset, fmt.Errorf("offset %d before chunk start %d", startOffset, chunkStartOffset)
+	}
+
+	if positionInChunk >= len(chunkMessages) {
+		// Requested offset is beyond the cached chunk
+		// This happens when disk files only contain partial data
+		// The requested offset might be in the gap between disk and memory
+
+		// Return empty (data not on disk) - caller will check memory buffers
+		return nil, startOffset, nil
+	}
+
+	// Extract messages starting from the requested position
+	messages := make([]*filer_pb.LogEntry, 0, maxMessages)
+	nextOffset := startOffset
+	totalBytes := 0
+
+	for i := positionInChunk; i < len(chunkMessages) && len(messages) < maxMessages; i++ {
+		entry := chunkMessages[i]
+		entrySize := proto.Size(entry)
+
+		// Check byte limit
+		if totalBytes > 0 && totalBytes+entrySize > maxBytes {
+			break
+		}
+
+		messages = append(messages, entry)
+		totalBytes += entrySize
+		nextOffset++
+	}
+
+	glog.V(4).Infof("[DiskCache] Extracted %d messages from cache (offset %d-%d, bytes=%d)",
+		len(messages), startOffset, nextOffset-1, totalBytes)
+
+	return messages, nextOffset, nil
+}
+
+// parseMessagesFromBuffer parses messages from a buffer byte slice
+// This is thread-safe as it operates on a copy of the buffer
+func parseMessagesFromBuffer(buf []byte, startOffset int64, maxMessages int, maxBytes int) (
+	messages []*filer_pb.LogEntry,
+	nextOffset int64,
+	totalBytes int,
+	err error,
+) {
+	messages = make([]*filer_pb.LogEntry, 0, maxMessages)
+	nextOffset = startOffset
+	totalBytes = 0
+	foundStart := false
+
+	messagesInBuffer := 0
+	for pos := 0; pos+4 < len(buf) && len(messages) < maxMessages && totalBytes < maxBytes; {
+		// Read message size
+		size := util.BytesToUint32(buf[pos : pos+4])
+		if pos+4+int(size) > len(buf) {
+			// Incomplete message at end of buffer
+			glog.V(4).Infof("[parseMessages] Incomplete message at pos %d, size %d, bufLen %d",
+				pos, size, len(buf))
+			break
+		}
+
+		// Parse message
+		entryData := buf[pos+4 : pos+4+int(size)]
+		logEntry := &filer_pb.LogEntry{}
+		if err = logEntry.UnmarshalVT(entryData); err != nil {
+			glog.Warningf("[parseMessages] Failed to unmarshal message: %v", err)
+			pos += 4 + int(size)
+			continue
+		}
+
+		messagesInBuffer++
+
+		// Initialize foundStart from first message
+		if !foundStart {
+			// Find the first message at or after startOffset
+			if logEntry.Offset >= startOffset {
+				foundStart = true
+				nextOffset = logEntry.Offset
+			} else {
+				// Skip messages before startOffset
+				glog.V(3).Infof("[parseMessages] Skipping message at offset %d (before startOffset %d)", logEntry.Offset, startOffset)
+				pos += 4 + int(size)
+				continue
+			}
+		}
+
+		// Check if this message matches expected offset
+		if foundStart && logEntry.Offset >= startOffset {
+			glog.V(3).Infof("[parseMessages] Adding message at offset %d (count=%d)", logEntry.Offset, len(messages)+1)
+			messages = append(messages, logEntry)
+			totalBytes += 4 + int(size)
+			nextOffset = logEntry.Offset + 1
+		}
+
+		pos += 4 + int(size)
+	}
+
+	glog.V(4).Infof("[parseMessages] Parsed %d messages, nextOffset=%d, totalBytes=%d",
+		len(messages), nextOffset, totalBytes)
+
+	return messages, nextOffset, totalBytes, nil
+}
+
+// GetHighWaterMark returns the highest offset available in this partition
+// This is a lightweight operation for clients to check partition state
+func (logBuffer *LogBuffer) GetHighWaterMark() int64 {
+	logBuffer.RLock()
+	defer logBuffer.RUnlock()
+	return logBuffer.offset
+}
+
+// GetLogStartOffset returns the earliest offset available (either in memory or on disk)
+// This is useful for clients to know the valid offset range
+func (logBuffer *LogBuffer) GetLogStartOffset() int64 {
+	logBuffer.RLock()
+	defer logBuffer.RUnlock()
+
+	// Check if we have offset information
+	if !logBuffer.hasOffsets {
+		return 0
+	}
+
+	// Return the current buffer start offset - this is the earliest offset in memory RIGHT NOW
+	// For stateless fetch, we only return what's currently available in memory
+	// We don't check prevBuffers because they may be stale or getting flushed
+	return logBuffer.bufferStartOffset
+}
+
+// WaitForDataWithTimeout waits up to maxWaitMs for data to be available at startOffset
+// Returns true if data became available, false if timeout
+// This allows "long poll" behavior for real-time consumers
+func (logBuffer *LogBuffer) WaitForDataWithTimeout(startOffset int64, maxWaitMs int) bool {
+	if maxWaitMs <= 0 {
+		return false
+	}
+
+	timeout := time.NewTimer(time.Duration(maxWaitMs) * time.Millisecond)
+	defer timeout.Stop()
+
+	// Register for notifications
+	notifyChan := logBuffer.RegisterSubscriber(fmt.Sprintf("fetch-%d", startOffset))
+	defer logBuffer.UnregisterSubscriber(fmt.Sprintf("fetch-%d", startOffset))
+
+	// Check if data is already available
+	logBuffer.RLock()
+	currentEnd := logBuffer.offset
+	logBuffer.RUnlock()
+
+	if currentEnd >= startOffset {
+		return true
+	}
+
+	// Wait for notification or timeout
+	select {
+	case <-notifyChan:
+		// Data might be available now
+		logBuffer.RLock()
+		currentEnd := logBuffer.offset
+		logBuffer.RUnlock()
+		return currentEnd >= startOffset
+	case <-timeout.C:
+		return false
+	}
+}

weed/util/log_buffer/log_read_stateless_test.go

@@ -0,0 +1,390 @@
+package log_buffer
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func TestReadMessagesAtOffset_EmptyBuffer(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+	lb.bufferStartOffset = 0
+	lb.offset = 0 // Empty buffer
+
+	messages, nextOffset, hwm, endOfPartition, err := lb.ReadMessagesAtOffset(100, 10, 1024)
+
+	// Reading from future offset (100) when buffer is at 0
+	// Should return empty, no error
+	if err != nil {
+		t.Errorf("Expected no error for future offset, got %v", err)
+	}
+	if len(messages) != 0 {
+		t.Errorf("Expected 0 messages, got %d", len(messages))
+	}
+	if nextOffset != 100 {
+		t.Errorf("Expected nextOffset=100, got %d", nextOffset)
+	}
+	if !endOfPartition {
+		t.Error("Expected endOfPartition=true for future offset")
+	}
+	if hwm != 0 {
+		t.Errorf("Expected highWaterMark=0, got %d", hwm)
+	}
+}
+
+func TestReadMessagesAtOffset_SingleMessage(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add a message
+	entry := &filer_pb.LogEntry{
+		TsNs:   time.Now().UnixNano(),
+		Key:    []byte("key1"),
+		Data:   []byte("value1"),
+		Offset: 0,
+	}
+	if err := lb.AddLogEntryToBuffer(entry); err != nil {
+		t.Fatalf("Failed to add log entry to buffer: %v", err)
+	}
+
+	// Read from offset 0
+	messages, nextOffset, _, endOfPartition, err := lb.ReadMessagesAtOffset(0, 10, 1024)
+
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+	if len(messages) != 1 {
+		t.Errorf("Expected 1 message, got %d", len(messages))
+	}
+	if nextOffset != 1 {
+		t.Errorf("Expected nextOffset=1, got %d", nextOffset)
+	}
+	if !endOfPartition {
+		t.Error("Expected endOfPartition=true after reading all messages")
+	}
+	if messages[0].Offset != 0 {
+		t.Errorf("Expected message offset=0, got %d", messages[0].Offset)
+	}
+	if string(messages[0].Key) != "key1" {
+		t.Errorf("Expected key='key1', got '%s'", string(messages[0].Key))
+	}
+}
+
+func TestReadMessagesAtOffset_MultipleMessages(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add 5 messages
+	for i := 0; i < 5; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		if err := lb.AddLogEntryToBuffer(entry); err != nil {
+			t.Fatalf("Failed to add log entry to buffer: %v", err)
+		}
+	}
+
+	// Read from offset 0, max 3 messages
+	messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(0, 3, 10240)
+
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+	if len(messages) != 3 {
+		t.Errorf("Expected 3 messages, got %d", len(messages))
+	}
+	if nextOffset != 3 {
+		t.Errorf("Expected nextOffset=3, got %d", nextOffset)
+	}
+
+	// Verify offsets are sequential
+	for i, msg := range messages {
+		if msg.Offset != int64(i) {
+			t.Errorf("Message %d: expected offset=%d, got %d", i, i, msg.Offset)
+		}
+	}
+}
+
+func TestReadMessagesAtOffset_StartFromMiddle(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add 10 messages (0-9)
+	for i := 0; i < 10; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		if err := lb.AddLogEntryToBuffer(entry); err != nil {
+			t.Fatalf("Failed to add log entry to buffer: %v", err)
+		}
+	}
+
+	// Read from offset 5
+	messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(5, 3, 10240)
+
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+	if len(messages) != 3 {
+		t.Errorf("Expected 3 messages, got %d", len(messages))
+	}
+	if nextOffset != 8 {
+		t.Errorf("Expected nextOffset=8, got %d", nextOffset)
+	}
+
+	// Verify we got messages 5, 6, 7
+	expectedOffsets := []int64{5, 6, 7}
+	for i, msg := range messages {
+		if msg.Offset != expectedOffsets[i] {
+			t.Errorf("Message %d: expected offset=%d, got %d", i, expectedOffsets[i], msg.Offset)
+		}
+	}
+}
+
+func TestReadMessagesAtOffset_MaxBytesLimit(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add messages with 100 bytes each
+	for i := 0; i < 10; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   make([]byte, 100), // 100 bytes
+			Offset: int64(i),
+		}
+		if err := lb.AddLogEntryToBuffer(entry); err != nil {
+			t.Fatalf("Failed to add log entry to buffer: %v", err)
+		}
+	}
+
+	// Request with max 250 bytes (should get ~2 messages)
+	messages, _, _, _, err := lb.ReadMessagesAtOffset(0, 100, 250)
+
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+
+	// Should get at least 1 message, but likely 2
+	if len(messages) == 0 {
+		t.Error("Expected at least 1 message")
+	}
+	if len(messages) > 3 {
+		t.Errorf("Expected max 3 messages with 250 byte limit, got %d", len(messages))
+	}
+}
+
+func TestReadMessagesAtOffset_ConcurrentReads(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add 100 messages
+	for i := 0; i < 100; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		if err := lb.AddLogEntryToBuffer(entry); err != nil {
+			t.Fatalf("Failed to add log entry to buffer: %v", err)
+		}
+	}
+
+	// Start 10 concurrent readers at different offsets
+	done := make(chan bool, 10)
+
+	for reader := 0; reader < 10; reader++ {
+		startOffset := int64(reader * 10)
+		go func(offset int64) {
+			messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(offset, 5, 10240)
+
+			if err != nil {
+				t.Errorf("Reader at offset %d: unexpected error: %v", offset, err)
+			}
+			if len(messages) != 5 {
+				t.Errorf("Reader at offset %d: expected 5 messages, got %d", offset, len(messages))
+			}
+			if nextOffset != offset+5 {
+				t.Errorf("Reader at offset %d: expected nextOffset=%d, got %d", offset, offset+5, nextOffset)
+			}
+
+			// Verify sequential offsets
+			for i, msg := range messages {
+				expectedOffset := offset + int64(i)
+				if msg.Offset != expectedOffset {
+					t.Errorf("Reader at offset %d: message %d has offset %d, expected %d",
+						offset, i, msg.Offset, expectedOffset)
+				}
+			}
+
+			done <- true
+		}(startOffset)
+	}
+
+	// Wait for all readers
+	for i := 0; i < 10; i++ {
+		<-done
+	}
+}
+
+func TestReadMessagesAtOffset_FutureOffset(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add 5 messages (0-4)
+	for i := 0; i < 5; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		if err := lb.AddLogEntryToBuffer(entry); err != nil {
+			t.Fatalf("Failed to add log entry to buffer: %v", err)
+		}
+	}
+
+	// Try to read from offset 10 (future)
+	messages, nextOffset, _, endOfPartition, err := lb.ReadMessagesAtOffset(10, 10, 10240)
+
+	if err != nil {
+		t.Errorf("Expected no error for future offset, got %v", err)
+	}
+	if len(messages) != 0 {
+		t.Errorf("Expected 0 messages for future offset, got %d", len(messages))
+	}
+	if nextOffset != 10 {
+		t.Errorf("Expected nextOffset=10, got %d", nextOffset)
+	}
+	if !endOfPartition {
+		t.Error("Expected endOfPartition=true for future offset")
+	}
+}
+
+func TestWaitForDataWithTimeout_DataAvailable(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add message at offset 0
+	entry := &filer_pb.LogEntry{
+		TsNs:   time.Now().UnixNano(),
+		Key:    []byte("key"),
+		Data:   []byte("value"),
+		Offset: 0,
+	}
+	if err := lb.AddLogEntryToBuffer(entry); err != nil {
+		t.Fatalf("Failed to add log entry to buffer: %v", err)
+	}
+
+	// Wait for data at offset 0 (should return immediately)
+	dataAvailable := lb.WaitForDataWithTimeout(0, 100)
+
+	if !dataAvailable {
+		t.Error("Expected data to be available at offset 0")
+	}
+}
+
+func TestWaitForDataWithTimeout_NoData(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+	lb.bufferStartOffset = 0
+	lb.offset = 0
+
+	// Don't add any messages, wait for offset 10
+
+	// Wait for data at offset 10 with short timeout
+	start := time.Now()
+	dataAvailable := lb.WaitForDataWithTimeout(10, 50)
+	elapsed := time.Since(start)
+
+	if dataAvailable {
+		t.Error("Expected no data to be available")
+	}
+	// Note: Actual wait time may be shorter if subscriber mechanism
+	// returns immediately. Just verify no data was returned.
+	t.Logf("Waited %v for timeout", elapsed)
+}
+
+func TestWaitForDataWithTimeout_DataArrives(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Start waiting in background
+	done := make(chan bool)
+	var dataAvailable bool
+
+	go func() {
+		dataAvailable = lb.WaitForDataWithTimeout(0, 500)
+		done <- true
+	}()
+
+	// Add data after 50ms
+	time.Sleep(50 * time.Millisecond)
+	entry := &filer_pb.LogEntry{
+		TsNs:   time.Now().UnixNano(),
+		Key:    []byte("key"),
+		Data:   []byte("value"),
+		Offset: 0,
+	}
+	if err := lb.AddLogEntryToBuffer(entry); err != nil {
+		t.Fatalf("Failed to add log entry to buffer: %v", err)
+	}
+
+	// Wait for result
+	<-done
+
+	if !dataAvailable {
+		t.Error("Expected data to become available after being added")
+	}
+}
+
+func TestGetHighWaterMark(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Initially should be 0
+	hwm := lb.GetHighWaterMark()
+	if hwm != 0 {
+		t.Errorf("Expected initial HWM=0, got %d", hwm)
+	}
+
+	// Add messages (offsets 0-4)
+	for i := 0; i < 5; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		if err := lb.AddLogEntryToBuffer(entry); err != nil {
+			t.Fatalf("Failed to add log entry to buffer: %v", err)
+		}
+	}
+
+	// HWM should be 5 (next offset to write, not last written offset)
+	// This matches Kafka semantics where HWM = last offset + 1
+	hwm = lb.GetHighWaterMark()
+	if hwm != 5 {
+		t.Errorf("Expected HWM=5 after adding 5 messages (0-4), got %d", hwm)
+	}
+}
+
+func TestGetLogStartOffset(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+	lb.bufferStartOffset = 10
+
+	lso := lb.GetLogStartOffset()
+	if lso != 10 {
+		t.Errorf("Expected LSO=10, got %d", lso)
+	}
+}

weed/util/log_buffer/log_read_test.go

@@ -0,0 +1,664 @@
+package log_buffer
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+)
+
+// TestLoopProcessLogDataWithOffset_ClientDisconnect tests that the loop exits
+// when the client disconnects (waitForDataFn returns false)
+func TestLoopProcessLogDataWithOffset_ClientDisconnect(t *testing.T) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Simulate client disconnect after 100ms
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	waitForDataFn := func() bool {
+		select {
+		case <-ctx.Done():
+			return false // Client disconnected
+		default:
+			return true
+		}
+	}
+
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+		return true, nil
+	}
+
+	startPosition := NewMessagePositionFromOffset(0)
+	startTime := time.Now()
+
+	// This should exit within 200ms (100ms timeout + some buffer)
+	_, isDone, _ := logBuffer.LoopProcessLogDataWithOffset("test-client", startPosition, 0, waitForDataFn, eachLogEntryFn)
+
+	elapsed := time.Since(startTime)
+
+	if !isDone {
+		t.Errorf("Expected isDone=true when client disconnects, got false")
+	}
+
+	if elapsed > 500*time.Millisecond {
+		t.Errorf("Loop took too long to exit: %v (expected < 500ms)", elapsed)
+	}
+
+	t.Logf("Loop exited cleanly in %v after client disconnect", elapsed)
+}
+
+// TestLoopProcessLogDataWithOffset_EmptyBuffer tests that the loop doesn't
+// busy-wait when the buffer is empty
+func TestLoopProcessLogDataWithOffset_EmptyBuffer(t *testing.T) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	callCount := 0
+	maxCalls := 4
+	mu := sync.Mutex{}
+
+	waitForDataFn := func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		callCount++
+		// Disconnect after maxCalls to prevent infinite loop
+		return callCount < maxCalls
+	}
+
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+		return true, nil
+	}
+
+	startPosition := NewMessagePositionFromOffset(0)
+	startTime := time.Now()
+
+	_, isDone, _ := logBuffer.LoopProcessLogDataWithOffset("test-client", startPosition, 0, waitForDataFn, eachLogEntryFn)
+
+	elapsed := time.Since(startTime)
+
+	if !isDone {
+		t.Errorf("Expected isDone=true when waitForDataFn returns false, got false")
+	}
+
+	minExpectedTime := time.Duration(maxCalls-1) * notificationHealthCheckInterval
+	if elapsed < minExpectedTime {
+		t.Errorf("Loop exited too quickly (%v), expected at least %v (suggests busy-waiting)", elapsed, minExpectedTime)
+	}
+
+	maxExpectedTime := time.Duration(maxCalls+1) * notificationHealthCheckInterval
+	if elapsed > maxExpectedTime {
+		t.Errorf("Loop took too long: %v (expected < %v)", elapsed, maxExpectedTime)
+	}
+
+	mu.Lock()
+	finalCallCount := callCount
+	mu.Unlock()
+
+	if finalCallCount != maxCalls {
+		t.Errorf("Expected exactly %d calls to waitForDataFn, got %d", maxCalls, finalCallCount)
+	}
+
+	t.Logf("Loop exited cleanly in %v after %d iterations (no busy-waiting detected)", elapsed, finalCallCount)
+}
+
+// TestLoopProcessLogDataWithOffset_NoDataResumeFromDisk tests that the loop
+// properly handles ResumeFromDiskError without busy-waiting
+func TestLoopProcessLogDataWithOffset_NoDataResumeFromDisk(t *testing.T) {
+	readFromDiskFn := func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (lastReadPosition MessagePosition, isDone bool, err error) {
+		// No data on disk
+		return startPosition, false, nil
+	}
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, readFromDiskFn, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	callCount := 0
+	maxCalls := 3
+	mu := sync.Mutex{}
+
+	waitForDataFn := func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		callCount++
+		// Disconnect after maxCalls
+		return callCount < maxCalls
+	}
+
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+		return true, nil
+	}
+
+	startPosition := NewMessagePositionFromOffset(0)
+	startTime := time.Now()
+
+	_, isDone, _ := logBuffer.LoopProcessLogDataWithOffset("test-client", startPosition, 0, waitForDataFn, eachLogEntryFn)
+
+	elapsed := time.Since(startTime)
+
+	if !isDone {
+		t.Errorf("Expected isDone=true when waitForDataFn returns false, got false")
+	}
+
+	minExpectedTime := time.Duration(maxCalls-1) * notificationHealthCheckInterval
+	if elapsed < minExpectedTime {
+		t.Errorf("Loop exited too quickly (%v), expected at least %v (suggests missing wait)", elapsed, minExpectedTime)
+	}
+
+	t.Logf("Loop exited cleanly in %v after %d iterations (proper sleep detected)", elapsed, callCount)
+}
+
+// TestLoopFlush_NotifiesSubscribersAfterFlush is a regression test for the
+// issue #9007 fix: loopFlush must call notifySubscribers() after processing a
+// flush so that readers parked on notifyChan wake up when a flush lands. The
+// classic bug scenario is a reader that got ResumeFromDiskError, did a disk
+// read that raced the flush and found nothing, and is now blocked on
+// notifyChan waiting for the data that just hit disk.
+//
+// We drain the AddToBuffer notification first, then ForceFlush, and assert a
+// new notification is delivered on notifyChan well before the fallback
+// timeout. If the loopFlush notification is removed, this test fails by
+// hitting the fallback.
+func TestLoopFlush_NotifiesSubscribersAfterFlush(t *testing.T) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+	}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	notifyChan := logBuffer.RegisterSubscriber("flush-notify-test")
+	defer logBuffer.UnregisterSubscriber("flush-notify-test")
+
+	if err := logBuffer.AddToBuffer(&mq_pb.DataMessage{
+		Key:   []byte("k"),
+		Value: []byte("v"),
+		TsNs:  time.Now().UnixNano(),
+	}); err != nil {
+		t.Fatalf("AddToBuffer: %v", err)
+	}
+
+	// Consume the AddToBuffer notification so the channel starts empty.
+	select {
+	case <-notifyChan:
+	case <-time.After(100 * time.Millisecond):
+		t.Fatal("expected a notification from AddToBuffer")
+	}
+
+	// ForceFlush waits for loopFlush to process the flush. After it returns,
+	// loopFlush must have called notifySubscribers() again.
+	start := time.Now()
+	logBuffer.ForceFlush()
+
+	select {
+	case <-notifyChan:
+		elapsed := time.Since(start)
+		// The fallback timeout is notificationHealthCheckInterval; the flush
+		// notification should arrive well before that.
+		if elapsed >= notificationHealthCheckInterval {
+			t.Errorf("flush notification too slow: %v (>= fallback %v)", elapsed, notificationHealthCheckInterval)
+		}
+		t.Logf("flush notification delivered in %v", elapsed)
+	case <-time.After(notificationHealthCheckInterval):
+		t.Fatalf("loopFlush did not notify subscribers within %v", notificationHealthCheckInterval)
+	}
+}
+
+// TestLoopProcessLogDataWithOffset_WakesOnDataArrival drives a real
+// LoopProcessLogDataWithOffset reader from an empty buffer (readFromDiskFn
+// returns nothing, forcing the reader to park on notifyChan after the
+// ResumeFromDiskError branch), then adds data from another goroutine and
+// asserts the reader completes well before the fallback timeout would fire.
+// This protects the end-to-end wake-up path; the loopFlush-specific
+// notification is covered by TestLoopFlush_NotifiesSubscribersAfterFlush.
+func TestLoopProcessLogDataWithOffset_WakesOnDataArrival(t *testing.T) {
+	readFromDiskFn := func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+		// No data on disk; return unchanged so the reader parks on notifyChan.
+		return startPosition, false, nil
+	}
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+	}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, readFromDiskFn, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	received := make(chan struct{})
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+		close(received)
+		return true, nil // isDone
+	}
+
+	waitForDataFn := func() bool { return true }
+
+	startPosition := NewMessagePositionFromOffset(0)
+
+	readerDone := make(chan struct{})
+	go func() {
+		_, _, _ = logBuffer.LoopProcessLogDataWithOffset(
+			"wake-test", startPosition, 0, waitForDataFn, eachLogEntryFn)
+		close(readerDone)
+	}()
+
+	// Give the reader time to reach awaitNotificationOrTimeout. Both wake
+	// paths under test (notifyChan via AddToBuffer and shutdownCh via
+	// ShutdownLogBuffer) are race-free even if the reader hasn't parked yet
+	// — the notification stays buffered / shutdownCh stays closed — but a
+	// generous head start makes it likelier we exercise the actual park-then-
+	// wake path rather than the already-pending fast path. 50ms is well below
+	// notificationHealthCheckInterval (250ms) and tolerates slow CI.
+	time.Sleep(50 * time.Millisecond)
+
+	start := time.Now()
+	if err := logBuffer.AddToBuffer(&mq_pb.DataMessage{
+		Key:   []byte("k"),
+		Value: []byte("v"),
+		TsNs:  time.Now().UnixNano(),
+	}); err != nil {
+		t.Fatalf("AddToBuffer: %v", err)
+	}
+
+	select {
+	case <-received:
+	case <-time.After(notificationHealthCheckInterval):
+		t.Fatalf("reader did not process the entry within %v (fallback timeout)", notificationHealthCheckInterval)
+	}
+	<-readerDone
+	elapsed := time.Since(start)
+
+	if elapsed >= notificationHealthCheckInterval {
+		t.Errorf("reader wake too slow: %v (>= fallback %v)", elapsed, notificationHealthCheckInterval)
+	}
+	t.Logf("reader processed the entry in %v after AddToBuffer", elapsed)
+}
+
+// TestLoopProcessLogDataWithOffset_WakesOnShutdown verifies that a reader
+// parked inside awaitNotificationOrTimeout via the ResumeFromDiskError branch
+// exits promptly when ShutdownLogBuffer is called, without waiting for the
+// 250ms health-check fallback. Regression guard for the IsStopping() shutdown
+// path: if awaitNotificationOrTimeout returns true via shutdownCh and the
+// caller does not check IsStopping(), the reader either spins against the
+// closed shutdownCh or returns ResumeFromDiskError instead of exiting.
+func TestLoopProcessLogDataWithOffset_WakesOnShutdown(t *testing.T) {
+	readFromDiskFn := func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+		// No data on disk; return unchanged so the reader parks on notifyChan.
+		return startPosition, false, nil
+	}
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+	}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, readFromDiskFn, nil)
+	// Note: not deferring ShutdownLogBuffer; we trigger it explicitly below.
+
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+		return false, nil
+	}
+	waitForDataFn := func() bool { return true }
+	startPosition := NewMessagePositionFromOffset(0)
+
+	type result struct {
+		isDone bool
+		err    error
+	}
+	resultCh := make(chan result, 1)
+	go func() {
+		_, isDone, err := logBuffer.LoopProcessLogDataWithOffset(
+			"shutdown-test", startPosition, 0, waitForDataFn, eachLogEntryFn)
+		resultCh <- result{isDone: isDone, err: err}
+	}()
+
+	// Give the reader time to reach awaitNotificationOrTimeout. Both wake
+	// paths under test (notifyChan via AddToBuffer and shutdownCh via
+	// ShutdownLogBuffer) are race-free even if the reader hasn't parked yet
+	// — the notification stays buffered / shutdownCh stays closed — but a
+	// generous head start makes it likelier we exercise the actual park-then-
+	// wake path rather than the already-pending fast path. 50ms is well below
+	// notificationHealthCheckInterval (250ms) and tolerates slow CI.
+	time.Sleep(50 * time.Millisecond)
+
+	start := time.Now()
+	logBuffer.ShutdownLogBuffer()
+
+	select {
+	case r := <-resultCh:
+		elapsed := time.Since(start)
+		if elapsed >= notificationHealthCheckInterval {
+			t.Errorf("reader did not wake on shutdown: %v (>= fallback %v)", elapsed, notificationHealthCheckInterval)
+		}
+		if !r.isDone {
+			t.Errorf("expected isDone=true on shutdown, got false")
+		}
+		if r.err != nil {
+			t.Errorf("expected err=nil on shutdown, got %v", r.err)
+		}
+		t.Logf("reader exited in %v after ShutdownLogBuffer", elapsed)
+	case <-time.After(2 * notificationHealthCheckInterval):
+		t.Fatalf("reader did not exit within %v after ShutdownLogBuffer", 2*notificationHealthCheckInterval)
+	}
+}
+
+// TestLoopProcessLogDataWithOffset_WithData tests normal operation with data
+func TestLoopProcessLogDataWithOffset_WithData(t *testing.T) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Add some test data to the buffer
+	testMessages := []*mq_pb.DataMessage{
+		{Key: []byte("key1"), Value: []byte("message1"), TsNs: 1},
+		{Key: []byte("key2"), Value: []byte("message2"), TsNs: 2},
+		{Key: []byte("key3"), Value: []byte("message3"), TsNs: 3},
+	}
+
+	for _, msg := range testMessages {
+		if err := logBuffer.AddToBuffer(msg); err != nil {
+			t.Fatalf("Failed to add message to buffer: %v", err)
+		}
+	}
+
+	receivedCount := 0
+	mu := sync.Mutex{}
+
+	// Disconnect after receiving at least 1 message to test that data processing works
+	waitForDataFn := func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		return receivedCount == 0 // Disconnect after first message
+	}
+
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+		mu.Lock()
+		receivedCount++
+		mu.Unlock()
+		return true, nil // Continue processing
+	}
+
+	startPosition := NewMessagePositionFromOffset(0)
+	startTime := time.Now()
+
+	_, isDone, _ := logBuffer.LoopProcessLogDataWithOffset("test-client", startPosition, 0, waitForDataFn, eachLogEntryFn)
+
+	elapsed := time.Since(startTime)
+
+	if !isDone {
+		t.Errorf("Expected isDone=true after client disconnect, got false")
+	}
+
+	mu.Lock()
+	finalCount := receivedCount
+	mu.Unlock()
+
+	if finalCount < 1 {
+		t.Errorf("Expected to receive at least 1 message, got %d", finalCount)
+	}
+
+	// Should complete quickly since data is available
+	if elapsed > 1*time.Second {
+		t.Errorf("Processing took too long: %v (expected < 1s)", elapsed)
+	}
+
+	t.Logf("Successfully processed %d message(s) in %v", finalCount, elapsed)
+}
+
+// TestLoopProcessLogDataWithOffset_ConcurrentDisconnect tests that the loop
+// handles concurrent client disconnects without panicking
+func TestLoopProcessLogDataWithOffset_ConcurrentDisconnect(t *testing.T) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	numClients := 10
+	var wg sync.WaitGroup
+
+	for i := 0; i < numClients; i++ {
+		wg.Add(1)
+		go func(clientID int) {
+			defer wg.Done()
+
+			ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+			defer cancel()
+
+			waitForDataFn := func() bool {
+				select {
+				case <-ctx.Done():
+					return false
+				default:
+					return true
+				}
+			}
+
+			eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+				return true, nil
+			}
+
+			startPosition := NewMessagePositionFromOffset(0)
+			_, _, _ = logBuffer.LoopProcessLogDataWithOffset("test-client", startPosition, 0, waitForDataFn, eachLogEntryFn)
+		}(i)
+	}
+
+	// Wait for all clients to finish with a timeout
+	done := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		t.Logf("All %d concurrent clients exited cleanly", numClients)
+	case <-time.After(5 * time.Second):
+		t.Errorf("Timeout waiting for concurrent clients to exit (possible deadlock or stuck loop)")
+	}
+}
+
+// TestLoopProcessLogDataWithOffset_StopTime tests that the loop respects stopTsNs
+func TestLoopProcessLogDataWithOffset_StopTime(t *testing.T) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	callCount := 0
+	waitForDataFn := func() bool {
+		callCount++
+		// Prevent infinite loop in case of test failure
+		return callCount < 10
+	}
+
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+		t.Errorf("Should not process any entries when stopTsNs is in the past")
+		return false, nil
+	}
+
+	startPosition := NewMessagePositionFromOffset(0)
+	stopTsNs := time.Now().Add(-1 * time.Hour).UnixNano() // Stop time in the past
+
+	startTime := time.Now()
+	_, isDone, _ := logBuffer.LoopProcessLogDataWithOffset("test-client", startPosition, stopTsNs, waitForDataFn, eachLogEntryFn)
+	elapsed := time.Since(startTime)
+
+	if !isDone {
+		t.Errorf("Expected isDone=true when stopTsNs is in the past, got false")
+	}
+
+	if elapsed > 1*time.Second {
+		t.Errorf("Loop should exit quickly when stopTsNs is in the past, took %v", elapsed)
+	}
+
+	t.Logf("Loop correctly exited for past stopTsNs in %v (waitForDataFn called %d times)", elapsed, callCount)
+}
+
+// TestLoopProcessLogData_BoundedEmptyBufferTerminates: a bounded subscription
+// (stopTsNs != 0) against a buffer that has never received a write - so
+// ReadFromBuffer returns ResumeFromDiskError - with no ReadFromDiskFn must
+// terminate like the caught-up path does, not park on the notification loop
+// forever. This is the filer SubscribeMetadata configuration: both meta log
+// buffers are built with ReadFromDiskFn == nil, and a freshly restarted filer
+// that has serviced zero metadata writes wedges every UntilNs-bounded
+// subscriber while heartbeats keep the stream looking alive.
+func TestLoopProcessLogData_BoundedEmptyBufferTerminates(t *testing.T) {
+	logBuffer := NewLogBuffer("test", time.Minute, nil, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Client stays connected, like the filer's heartbeat-sending closure.
+	waitForDataFn := func() bool { return true }
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry) (bool, error) {
+		t.Error("no entries should be delivered from an empty buffer")
+		return false, nil
+	}
+
+	startPosition := NewMessagePosition(time.Now().Add(-time.Hour).UnixNano(), EvictionGatedOffset)
+	stopTsNs := time.Now().UnixNano()
+
+	done := make(chan struct{})
+	var isDone bool
+	var err error
+	go func() {
+		_, isDone, err = logBuffer.LoopProcessLogData("bounded-empty", startPosition, stopTsNs, waitForDataFn, eachLogEntryFn)
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		if !isDone {
+			t.Errorf("expected isDone=true for a bounded read of an empty buffer, got false")
+		}
+		// A leaked ResumeFromDiskError would send the filer's outer loop into
+		// its gap machinery, which parks the bounded subscriber all over again.
+		if err != nil {
+			t.Errorf("expected err=nil for a bounded read of an empty buffer, got %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("bounded LoopProcessLogData wedged on an empty buffer instead of terminating")
+	}
+}
+
+func TestLoopProcessLogData_SlowConsumerFallsBehind(t *testing.T) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	baseTime := time.Now()
+	for i := 0; i < 1000; i++ {
+		ts := baseTime.Add(time.Duration(i) * time.Millisecond)
+		if err := logBuffer.AddDataToBuffer([]byte("key"), []byte("value"), ts.UnixNano()); err != nil {
+			t.Fatalf("AddDataToBuffer(%d): %v", i, err)
+		}
+	}
+
+	oldPosition := NewMessagePosition(baseTime.Add(-10*time.Second).UnixNano(), 1)
+
+	waitForDataFn := func() bool {
+		t.Errorf("waitForDataFn should not be called for a slow consumer that has fallen behind")
+		return false
+	}
+
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry) (bool, error) {
+		return false, nil
+	}
+
+	done := make(chan struct{})
+	var err error
+	go func() {
+		_, _, err = logBuffer.LoopProcessLogData("slow-consumer", oldPosition, 0, waitForDataFn, eachLogEntryFn)
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		if err != ResumeFromDiskError {
+			t.Fatalf("expected ResumeFromDiskError, got %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("LoopProcessLogData blocked instead of returning ResumeFromDiskError")
+	}
+}
+
+// TestLoopProcessLogData_DuplicateReaderLeaving is a regression test for issue
+// #10810: an idle metadata subscriber pinning a full core inside
+// awaitNotificationOrTimeoutFor. Two readers registered under the same name
+// share one notification channel, which happens whenever a client reconnects
+// before its previous stream noticed the disconnect. When the departing one
+// closed that channel, the surviving reader's select won on it instantly on
+// every pass, so the loop ran flat out - allocating a timer per iteration -
+// until the client went away.
+func TestLoopProcessLogData_DuplicateReaderLeaving(t *testing.T) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	const readerName = "localMeta:s3@"
+	startPosition := NewMessagePosition(time.Now().UnixNano(), -2)
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry) (bool, error) { return false, nil }
+
+	// startReader runs a reader and reports how many times it went round the
+	// loop; the loop calls waitForDataFn exactly once per pass.
+	startReader := func(iterations *atomic.Int64, stop *atomic.Bool) chan struct{} {
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			logBuffer.LoopProcessLogData(readerName, startPosition, 0, func() bool {
+				iterations.Add(1)
+				return !stop.Load()
+			}, eachLogEntryFn)
+		}()
+		for iterations.Load() == 0 {
+			time.Sleep(time.Millisecond)
+		}
+		return done
+	}
+
+	var leavingIterations, survivorIterations atomic.Int64
+	var stopLeaving, stopSurvivor atomic.Bool
+	leavingDone := startReader(&leavingIterations, &stopLeaving)
+	survivorDone := startReader(&survivorIterations, &stopSurvivor)
+
+	stopLeaving.Store(true)
+	<-leavingDone
+
+	const observeFor = 500 * time.Millisecond
+	before := survivorIterations.Load()
+	time.Sleep(observeFor)
+	spun := survivorIterations.Load() - before
+
+	// A low iteration count only means "not spinning" if the reader was still
+	// there to spin. Had LoopProcessLogData returned when its duplicate left,
+	// spun would be 0 and the count below would pass while proving nothing.
+	select {
+	case <-survivorDone:
+		t.Fatal("surviving reader returned while its duplicate unregistered; it must keep reading")
+	default:
+	}
+
+	stopSurvivor.Store(true)
+	<-survivorDone
+
+	maxIterations := int64(observeFor/notificationHealthCheckInterval) + 1
+	if spun > maxIterations {
+		t.Errorf("surviving reader looped %d times in %v, expected at most %d (suggests busy-waiting)", spun, observeFor, maxIterations)
+	}
+}
+
+// BenchmarkLoopProcessLogDataWithOffset_EmptyBuffer benchmarks the performance
+// of the loop with an empty buffer to ensure no busy-waiting
+func BenchmarkLoopProcessLogDataWithOffset_EmptyBuffer(b *testing.B) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	for i := 0; i < b.N; i++ {
+		callCount := 0
+		waitForDataFn := func() bool {
+			callCount++
+			return callCount < 3 // Exit after 3 calls
+		}
+
+		eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+			return true, nil
+		}
+
+		startPosition := NewMessagePositionFromOffset(0)
+		logBuffer.LoopProcessLogDataWithOffset("test-client", startPosition, 0, waitForDataFn, eachLogEntryFn)
+	}
+}

weed/util/log_buffer/read_ts_test.go

@@ -0,0 +1,230 @@
+package log_buffer
+
+import (
+	"bytes"
+	"math"
+	"math/rand"
+	"testing"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// readTsNs must return exactly what a full proto.Unmarshal would put in TsNs,
+// for every field combination, without decoding data/key.
+func TestReadTsNsMatchesUnmarshal(t *testing.T) {
+	cases := []*filer_pb.LogEntry{
+		{},                              // all defaults: ts_ns absent on the wire => 0
+		{TsNs: 1},                       // minimal
+		{TsNs: 1783396413597780505},     // realistic nanosecond timestamp
+		{TsNs: math.MaxInt64},           // 9-byte varint
+		{TsNs: -1},                      // negative int64 => 10-byte varint
+		{TsNs: math.MinInt64},           // extreme negative
+		{TsNs: 42, PartitionKeyHash: 7}, // ts followed by another varint field
+		{TsNs: 12345, Data: []byte("some-metadata-event-payload"), Key: []byte("/buckets/pvc-x/1.log")},
+		{TsNs: 99, Data: make([]byte, 64*1024), Key: []byte("k"), Offset: 555}, // large data, all fields
+		{Data: []byte("no-ts"), Key: []byte("k"), Offset: 9},                   // ts_ns == 0 with other fields present
+	}
+	for i, e := range cases {
+		data, err := proto.Marshal(e)
+		if err != nil {
+			t.Fatalf("case %d marshal: %v", i, err)
+		}
+		got, err := readTsNs(data)
+		if err != nil {
+			t.Fatalf("case %d readTsNs: %v", i, err)
+		}
+		if got != e.TsNs {
+			t.Errorf("case %d: readTsNs=%d want %d", i, got, e.TsNs)
+		}
+	}
+}
+
+// Fuzz against proto.Unmarshal with randomized entries.
+func TestReadTsNsFuzz(t *testing.T) {
+	r := rand.New(rand.NewSource(1))
+	for i := 0; i < 5000; i++ {
+		e := &filer_pb.LogEntry{
+			TsNs:             r.Int63() - r.Int63(),
+			PartitionKeyHash: r.Int31(),
+			Offset:           r.Int63(),
+		}
+		if r.Intn(2) == 0 {
+			e.Data = make([]byte, r.Intn(300))
+			r.Read(e.Data)
+		}
+		if r.Intn(2) == 0 {
+			e.Key = make([]byte, r.Intn(64))
+			r.Read(e.Key)
+		}
+		data, err := proto.Marshal(e)
+		if err != nil {
+			t.Fatalf("marshal: %v", err)
+		}
+		var want filer_pb.LogEntry
+		if err := proto.Unmarshal(data, &want); err != nil {
+			t.Fatalf("unmarshal: %v", err)
+		}
+		got, err := readTsNs(data)
+		if err != nil {
+			t.Fatalf("readTsNs iter %d: %v", i, err)
+		}
+		if got != want.TsNs {
+			t.Fatalf("iter %d: readTsNs=%d want %d", i, got, want.TsNs)
+		}
+	}
+}
+
+// readTsNs must not panic and must report an error on truncated/garbage input.
+func TestReadTsNsCorrupted(t *testing.T) {
+	inputs := [][]byte{
+		{0x08},                   // ts_ns tag with no value
+		{0x1a, 0xff, 0xff, 0xff}, // data field, length overruns buffer
+		{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f}, // overlong tag
+	}
+	for i, in := range inputs {
+		if _, err := readTsNs(in); err == nil {
+			t.Errorf("input %d: expected error, got nil", i)
+		}
+	}
+}
+
+// unmarshalLogEntryAliased must produce the same fields as proto.Unmarshal.
+func TestUnmarshalLogEntryAliasedMatchesUnmarshal(t *testing.T) {
+	r := rand.New(rand.NewSource(2))
+	for i := 0; i < 5000; i++ {
+		e := &filer_pb.LogEntry{
+			TsNs:             r.Int63() - r.Int63(),
+			PartitionKeyHash: r.Int31() - r.Int31(),
+			Offset:           r.Int63() - r.Int63(),
+		}
+		if r.Intn(2) == 0 {
+			e.Data = make([]byte, r.Intn(4096))
+			r.Read(e.Data)
+		}
+		if r.Intn(2) == 0 {
+			e.Key = make([]byte, r.Intn(128))
+			r.Read(e.Key)
+		}
+		data, err := proto.Marshal(e)
+		if err != nil {
+			t.Fatalf("marshal: %v", err)
+		}
+		var want, got filer_pb.LogEntry
+		if err := proto.Unmarshal(data, &want); err != nil {
+			t.Fatalf("unmarshal: %v", err)
+		}
+		if err := unmarshalLogEntryAliased(data, &got); err != nil {
+			t.Fatalf("aliased iter %d: %v", i, err)
+		}
+		if got.TsNs != want.TsNs || got.PartitionKeyHash != want.PartitionKeyHash ||
+			got.Offset != want.Offset || !bytes.Equal(got.Data, want.Data) || !bytes.Equal(got.Key, want.Key) {
+			t.Fatalf("iter %d mismatch:\n got=%+v\nwant=%+v", i, &got, &want)
+		}
+	}
+}
+
+// The decoded data/key must point into the source buffer, not a copy.
+func TestUnmarshalLogEntryAliasedAliases(t *testing.T) {
+	data, err := proto.Marshal(&filer_pb.LogEntry{TsNs: 5, Data: []byte("payload"), Key: []byte("path")})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var out filer_pb.LogEntry
+	if err := unmarshalLogEntryAliased(data, &out); err != nil {
+		t.Fatal(err)
+	}
+	if string(out.Data) != "payload" || string(out.Key) != "path" {
+		t.Fatalf("decoded data=%q key=%q", out.Data, out.Key)
+	}
+	for i := range data { // mutate source; an alias must observe it, a copy must not
+		data[i] ^= 0xFF
+	}
+	if string(out.Data) == "payload" || string(out.Key) == "path" {
+		t.Fatal("data/key were copied, expected an alias into the source buffer")
+	}
+}
+
+// sampleEntryData returns a marshaled LogEntry whose Data is a realistic
+// marshaled metadata event, as delivered on the subscribe path.
+func sampleEntryData(tb testing.TB) []byte {
+	event := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/buckets/pvc-bf458866-0413-4cca-99c6-3b7386a80855",
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name:       "object-000123",
+				Attributes: &filer_pb.FuseAttributes{FileSize: 4096, Mtime: 1783396413},
+				Chunks: []*filer_pb.FileChunk{
+					{FileId: "3,01637037d6", Size: 4096, Fid: &filer_pb.FileId{VolumeId: 3, FileKey: 23283, Cookie: 2015}},
+				},
+			},
+		},
+	}
+	payload, err := proto.Marshal(event)
+	if err != nil {
+		tb.Fatal(err)
+	}
+	data, err := proto.Marshal(&filer_pb.LogEntry{
+		TsNs: 1783396413597780505, PartitionKeyHash: 12345, Offset: 2013258,
+		Key: []byte("/buckets/pvc-bf458866/object-000123"), Data: payload,
+	})
+	if err != nil {
+		tb.Fatal(err)
+	}
+	return data
+}
+
+// Per-entry decode cost on the delivery path: the old full unmarshal copies the
+// data/key byte fields; the aliased decode does not. The delta is what the fix
+// saves for every entry a subscriber reads (and, for non-matching entries a
+// subscriber filters out, the whole saving with nothing decoded downstream).
+func BenchmarkLogEntryDecodeCopy(b *testing.B) {
+	data := sampleEntryData(b)
+	var out filer_pb.LogEntry
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := proto.Unmarshal(data, &out); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkLogEntryDecodeAlias(b *testing.B) {
+	data := sampleEntryData(b)
+	var out filer_pb.LogEntry
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := unmarshalLogEntryAliased(data, &out); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkReadTs(b *testing.B) {
+	e := &filer_pb.LogEntry{
+		TsNs:             1783396413597780505,
+		PartitionKeyHash: 12345,
+		Data:             make([]byte, 4*1024), // typical metadata event payload
+		Key:              []byte("/buckets/pvc-bf458866/1.log"),
+		Offset:           2013258,
+	}
+	data, err := proto.Marshal(e)
+	if err != nil {
+		b.Fatal(err)
+	}
+	buf := make([]byte, 4+len(data))
+	util.Uint32toBytes(buf, uint32(len(data)))
+	copy(buf[4:], data)
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, _, err := readTs(buf, 0); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

weed/util/log_buffer/sealed_buffer.go

@@ -0,0 +1,98 @@
+package log_buffer
+
+import (
+	"fmt"
+	"sync"
+	"time"
+)
+
+type MemBuffer struct {
+	buf         []byte
+	size        int
+	startTime   time.Time
+	stopTime    time.Time
+	startOffset int64 // First offset in this buffer
+	offset      int64 // Last offset in this buffer (endOffset)
+	// max pre-bump entry timestamp; stamped by the sealer after SealBuffer
+	maxOriginalTsNs int64
+
+	// snapshot is a GC-owned copy of buf[:size] shared by all readers of this
+	// sealed window, so N subscribers reading the same window cost one copy
+	// instead of N pooled copies. Created lazily by the first reader; travels
+	// with the window when SealBuffer shifts slots. Immutable once created.
+	snapMu   sync.Mutex
+	snapshot []byte
+}
+
+// sharedSnapshot returns the shared read-only copy of this sealed window,
+// creating it on first use. Callers must hold the LogBuffer read lock, which
+// keeps buf stable (SealBuffer mutates slots only under the write lock).
+func (mb *MemBuffer) sharedSnapshot() []byte {
+	mb.snapMu.Lock()
+	defer mb.snapMu.Unlock()
+	if mb.snapshot == nil {
+		mb.snapshot = append([]byte(nil), mb.buf[:mb.size]...)
+	}
+	return mb.snapshot
+}
+
+type SealedBuffers struct {
+	buffers []*MemBuffer
+}
+
+func newSealedBuffers(size int) *SealedBuffers {
+	sbs := &SealedBuffers{}
+
+	sbs.buffers = make([]*MemBuffer, size)
+	for i := 0; i < size; i++ {
+		sbs.buffers[i] = &MemBuffer{
+			buf: make([]byte, BufferSize),
+		}
+	}
+
+	return sbs
+}
+
+func (sbs *SealedBuffers) SealBuffer(startTime, stopTime time.Time, buf []byte, pos int, startOffset int64, endOffset int64) (newBuf []byte) {
+	oldBuf := sbs.buffers[0].buf
+	size := len(sbs.buffers)
+	for i := 0; i < size-1; i++ {
+		sbs.buffers[i].buf = sbs.buffers[i+1].buf
+		sbs.buffers[i].size = sbs.buffers[i+1].size
+		sbs.buffers[i].startTime = sbs.buffers[i+1].startTime
+		sbs.buffers[i].stopTime = sbs.buffers[i+1].stopTime
+		sbs.buffers[i].startOffset = sbs.buffers[i+1].startOffset
+		sbs.buffers[i].offset = sbs.buffers[i+1].offset
+		sbs.buffers[i].snapshot = sbs.buffers[i+1].snapshot // snapshot follows its window
+		sbs.buffers[i].maxOriginalTsNs = sbs.buffers[i+1].maxOriginalTsNs
+	}
+	sbs.buffers[size-1].buf = buf
+	sbs.buffers[size-1].size = pos
+	sbs.buffers[size-1].startTime = startTime
+	sbs.buffers[size-1].stopTime = stopTime
+	sbs.buffers[size-1].startOffset = startOffset
+	sbs.buffers[size-1].offset = endOffset
+	sbs.buffers[size-1].snapshot = nil
+	sbs.buffers[size-1].maxOriginalTsNs = 0
+	return oldBuf
+}
+
+func (mb *MemBuffer) locateByTs(lastReadTime time.Time) (pos int, err error) {
+	lastReadTs := lastReadTime.UnixNano()
+	for pos < mb.size {
+		size, t, readErr := readTs(mb.buf, pos)
+		if readErr != nil {
+			// Return error if buffer is corrupted
+			return 0, fmt.Errorf("locateByTs: buffer corruption at pos %d: %w", pos, readErr)
+		}
+		if t > lastReadTs {
+			return pos, nil
+		}
+		pos += size + 4
+	}
+	return mb.size, nil
+}
+
+func (mb *MemBuffer) String() string {
+	return fmt.Sprintf("[%v,%v] bytes:%d", mb.startTime, mb.stopTime, mb.size)
+}

weed/util/log_buffer/shared_snapshot_test.go

@@ -0,0 +1,275 @@
+package log_buffer
+
+import (
+	"fmt"
+	"runtime/debug"
+	"sync"
+	"testing"
+	"time"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func addEntries(tb testing.TB, lb *LogBuffer, n int, payloadSize int, tag string) {
+	payload := make([]byte, payloadSize)
+	for i := 0; i < n; i++ {
+		key := []byte(fmt.Sprintf("/%s/%d", tag, i))
+		if err := lb.AddDataToBuffer(key, payload, time.Now().UnixNano()); err != nil {
+			tb.Fatal(err)
+		}
+	}
+}
+
+// Sealed-window reads must hand every reader the same shared snapshot rather
+// than a fresh copy each.
+func TestSealedReadsShareOneSnapshot(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	addEntries(t, lb, 10, 1024, "a")
+	lb.ForceFlush() // seal the window
+	addEntries(t, lb, 1, 1024, "b")
+
+	pos := NewMessagePosition(1, -2) // far in the past => sealed window hit
+	buf1, _, pooled1, err1 := lb.ReadFromBuffer(pos)
+	buf2, _, pooled2, err2 := lb.ReadFromBuffer(pos)
+	if err1 != nil || err2 != nil {
+		t.Fatalf("read errors: %v %v", err1, err2)
+	}
+	if buf1 == nil || buf2 == nil {
+		t.Fatal("expected sealed data in memory")
+	}
+	if pooled1 || pooled2 {
+		t.Fatalf("sealed reads should not be pooled copies: %v %v", pooled1, pooled2)
+	}
+	b1, b2 := buf1.Bytes(), buf2.Bytes()
+	if len(b1) == 0 || len(b1) != len(b2) {
+		t.Fatalf("unexpected lengths %d %d", len(b1), len(b2))
+	}
+	if &b1[0] != &b2[0] {
+		t.Fatal("sealed reads made independent copies; expected one shared snapshot")
+	}
+}
+
+// A shared snapshot must stay intact after its window rotates out and the
+// backing array is recycled and overwritten by new writes.
+func TestSharedSnapshotSurvivesRecycle(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	addEntries(t, lb, 5, 2048, "keep")
+	lb.ForceFlush()
+
+	pos := NewMessagePosition(1, -2)
+	buf, _, pooled, err := lb.ReadFromBuffer(pos)
+	if err != nil || buf == nil || pooled {
+		t.Fatalf("expected shared sealed read, got buf=%v pooled=%v err=%v", buf != nil, pooled, err)
+	}
+	before := append([]byte(nil), buf.Bytes()...)
+
+	// Rotate the sealed window all the way out so its array is recycled and
+	// overwritten with different content.
+	for i := 0; i < PreviousBufferCount+1; i++ {
+		addEntries(t, lb, 5, 2048, "overwrite")
+		lb.ForceFlush()
+	}
+	addEntries(t, lb, 5, 2048, "overwrite")
+
+	if string(before) != string(buf.Bytes()) {
+		t.Fatal("shared snapshot content changed after its window was recycled")
+	}
+}
+
+// Race test: concurrent lagging readers against a writer that keeps sealing
+// and recycling windows. Every delivered entry must unmarshal cleanly with the
+// expected payload — corruption here means a reader saw recycled bytes.
+func TestSharedSnapshotConcurrentIntegrity(t *testing.T) {
+	// The writer below is deliberately unthrottled, so transient marshal garbage
+	// scales with host speed; on linux/386 a fast runner can hit the 4GB
+	// address-space ceiling. Cap the heap so the GC paces the writer instead.
+	prevLimit := debug.SetMemoryLimit(1 << 30)
+	defer debug.SetMemoryLimit(prevLimit)
+
+	lb := NewLogBuffer("test", time.Hour, func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	const payloadByte = 0x5A
+	payload := make([]byte, 8*1024)
+	for i := range payload {
+		payload[i] = payloadByte
+	}
+
+	var stop sync.WaitGroup
+	done := make(chan struct{})
+	stop.Add(1)
+	go func() { // writer: fill and seal aggressively so arrays recycle under the readers
+		defer stop.Done()
+		for i := 0; ; i++ {
+			select {
+			case <-done:
+				return
+			default:
+			}
+			if err := lb.AddDataToBuffer([]byte(fmt.Sprintf("/k/%d", i)), payload, time.Now().UnixNano()); err != nil {
+				t.Error(err)
+				return
+			}
+			if i%200 == 0 {
+				lb.ForceFlush()
+			}
+		}
+	}()
+
+	var readers sync.WaitGroup
+	errCh := make(chan error, 16)
+	for r := 0; r < 8; r++ {
+		readers.Add(1)
+		go func(r int) {
+			defer readers.Done()
+			deadline := time.Now().Add(2 * time.Second)
+			for time.Now().Before(deadline) {
+				start := NewMessagePosition(time.Now().Add(-500*time.Millisecond).UnixNano(), -2)
+				lb.LoopProcessLogData(fmt.Sprintf("r%d", r), start, 0,
+					func() bool { return time.Now().Before(deadline) },
+					func(le *filer_pb.LogEntry) (bool, error) {
+						if len(le.Data) != len(payload) {
+							err := fmt.Errorf("payload length %d, want %d", len(le.Data), len(payload))
+							select {
+							case errCh <- err:
+							default:
+							}
+							return true, err
+						}
+						for _, b := range le.Data {
+							if b != payloadByte {
+								err := fmt.Errorf("corrupted payload byte %x", b)
+								select {
+								case errCh <- err:
+								default:
+								}
+								return true, err
+							}
+						}
+						return false, nil
+					})
+			}
+		}(r)
+	}
+	readers.Wait()
+	close(done)
+	stop.Wait()
+
+	select {
+	case err := <-errCh:
+		t.Fatal(err)
+	default:
+	}
+}
+
+// Current-window reads must share the prefix snapshot: two behind-readers get
+// views of the same backing array, and the view content must stay intact while
+// the writer keeps appending and eventually seals and recycles the window.
+func TestCurrentWindowPrefixSharing(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	addEntries(t, lb, 8, 1024, "cur")
+
+	pos := NewMessagePosition(1, -2) // behind => full current-window read
+	buf1, _, pooled1, err1 := lb.ReadFromBuffer(pos)
+	buf2, _, pooled2, err2 := lb.ReadFromBuffer(pos)
+	if err1 != nil || err2 != nil || buf1 == nil || buf2 == nil {
+		t.Fatalf("reads: %v %v %v %v", buf1 != nil, err1, buf2 != nil, err2)
+	}
+	if pooled1 || pooled2 {
+		t.Fatalf("current-window behind-reads should be shared, got pooled %v %v", pooled1, pooled2)
+	}
+	b1, b2 := buf1.Bytes(), buf2.Bytes()
+	if len(b1) == 0 || len(b1) != len(b2) || &b1[0] != &b2[0] {
+		t.Fatalf("expected one shared prefix snapshot, lens %d %d", len(b1), len(b2))
+	}
+	before := append([]byte(nil), b1...)
+
+	// Keep appending: the prefix must be extended, not reallocated or mutated.
+	addEntries(t, lb, 8, 1024, "more")
+	buf3, _, _, err3 := lb.ReadFromBuffer(pos)
+	if err3 != nil || buf3 == nil {
+		t.Fatalf("read3: %v %v", buf3 != nil, err3)
+	}
+	if len(buf3.Bytes()) <= len(before) {
+		t.Fatalf("extended read %d should exceed first read %d", len(buf3.Bytes()), len(before))
+	}
+	if string(buf3.Bytes()[:len(before)]) != string(before) {
+		t.Fatal("prefix changed when the snapshot was extended")
+	}
+
+	// Seal + rotate the window fully out; earlier views must stay intact.
+	for i := 0; i < PreviousBufferCount+1; i++ {
+		lb.ForceFlush()
+		addEntries(t, lb, 4, 1024, "rotate")
+	}
+	if string(b1) != string(before) {
+		t.Fatal("shared current-window view changed after seal and recycle")
+	}
+}
+
+// A sealed window whose prefix snapshot was fully extended must reuse it
+// rather than re-copying on the first sealed read.
+func TestSealHandsOffCompleteSnapshot(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	addEntries(t, lb, 6, 512, "h")
+	pos := NewMessagePosition(1, -2)
+	buf1, _, _, err := lb.ReadFromBuffer(pos) // extends prefix snapshot to full window
+	if err != nil || buf1 == nil {
+		t.Fatalf("read1: %v %v", buf1 != nil, err)
+	}
+	lb.ForceFlush()
+	addEntries(t, lb, 1, 512, "next")
+
+	buf2, _, pooled, err := lb.ReadFromBuffer(pos) // sealed read of the same window
+	if err != nil || buf2 == nil || pooled {
+		t.Fatalf("read2: buf=%v pooled=%v err=%v", buf2 != nil, pooled, err)
+	}
+	b1, b2 := buf1.Bytes(), buf2.Bytes()
+	if len(b1) == 0 || len(b1) > len(b2) || &b1[0] != &b2[0] {
+		t.Fatalf("sealed read should reuse the handed-off snapshot (lens %d %d)", len(b1), len(b2))
+	}
+}
+
+// Sanity: sealed-read content must byte-match what a proto round-trip expects.
+func TestSharedSnapshotContentMatches(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	addEntries(t, lb, 3, 512, "x")
+	lb.ForceFlush()
+
+	buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(1, -2))
+	if err != nil || buf == nil {
+		t.Fatalf("read: buf=%v err=%v", buf != nil, err)
+	}
+	data := buf.Bytes()
+	count := 0
+	for pos := 0; pos+4 < len(data); {
+		size, _, err := readTs(data, pos)
+		if err != nil {
+			t.Fatalf("entry %d: %v", count, err)
+		}
+		var le filer_pb.LogEntry
+		if err := proto.Unmarshal(data[pos+4:pos+4+size], &le); err != nil {
+			t.Fatalf("entry %d unmarshal: %v", count, err)
+		}
+		if want := fmt.Sprintf("/x/%d", count); string(le.Key) != want {
+			t.Fatalf("entry %d key %q, want %q", count, le.Key, want)
+		}
+		pos += 4 + size
+		count++
+	}
+	if count != 3 {
+		t.Fatalf("read %d entries, want 3", count)
+	}
+}

weed/util/mem/slot_pool.go

@@ -0,0 +1,61 @@
+package mem
+
+import (
+	"sync"
+)
+
+var pools []*sync.Pool
+
+const (
+	min_size = 1024
+)
+
+func bitCount(size int) (count int) {
+	for ; size > min_size; count++ {
+		size = (size + 1) >> 1
+	}
+	return
+}
+
+func init() {
+	// 1KB ~ 256MB
+	pools = make([]*sync.Pool, bitCount(1024*1024*256))
+	for i := 0; i < len(pools); i++ {
+		slotSize := 1024 << i
+		pools[i] = &sync.Pool{
+			New: func() interface{} {
+				buffer := make([]byte, slotSize)
+				return &buffer
+			},
+		}
+	}
+}
+
+func getSlotPool(size int) (*sync.Pool, bool) {
+	index := bitCount(size)
+	if index >= len(pools) {
+		return nil, false
+	}
+	return pools[index], true
+}
+
+func AllocationSize(size int) int {
+	if _, found := getSlotPool(size); found {
+		return min_size << bitCount(size)
+	}
+	return size
+}
+
+func Allocate(size int) []byte {
+	if pool, found := getSlotPool(size); found {
+		slab := *pool.Get().(*[]byte)
+		return slab[:size]
+	}
+	return make([]byte, size)
+}
+
+func Free(buf []byte) {
+	if pool, found := getSlotPool(cap(buf)); found {
+		pool.Put(&buf)
+	}
+}

weed/util/mem/slot_pool_test.go

@@ -0,0 +1,58 @@
+package mem
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAllocateFree(t *testing.T) {
+	buf := Allocate(12)
+	Free(buf)
+	if cap(buf) != min_size {
+		t.Errorf("min size error allocated capacity=%d", cap(buf))
+	}
+	if len(buf) != 12 {
+		t.Errorf("size error")
+	}
+
+	buf = Allocate(4883)
+	Free(buf)
+	if cap(buf) != 1024<<bitCount(4883) {
+		t.Errorf("min size error allocated capacity=%d", cap(buf))
+	}
+	if len(buf) != 4883 {
+		t.Errorf("size error")
+	}
+
+}
+
+func TestAllocateFreeEdgeCases(t *testing.T) {
+	assert.Equal(t, 1, bitCount(2048))
+	assert.Equal(t, 2, bitCount(2049))
+
+	buf := Allocate(2048)
+	Free(buf)
+	buf = Allocate(2049)
+	Free(buf)
+}
+
+func TestBitCount(t *testing.T) {
+	count := bitCount(12)
+	if count != 0 {
+		t.Errorf("bitCount error count=%d", count)
+	}
+	if count != bitCount(min_size) {
+		t.Errorf("bitCount error count=%d", count)
+	}
+
+}
+
+func TestAllocationSize(t *testing.T) {
+	for _, size := range []int{0, 1, 1024, 1025, 4883, 2 << 20} {
+		buf := Allocate(size)
+		assert.Equal(t, cap(buf), AllocationSize(size))
+		Free(buf)
+	}
+	assert.Equal(t, 257<<20, AllocationSize(257<<20))
+}

weed/util/minfreespace.go

@@ -0,0 +1,106 @@
+package util
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+// MinFreeSpaceType is the type of MinFreeSpace.
+type MinFreeSpaceType int
+
+const (
+	// AsPercent set the MinFreeSpaceType to a percentage value from 0 to 100.
+	AsPercent MinFreeSpaceType = iota
+	// AsBytes set the MinFreeSpaceType to a absolute value bytes.
+	AsBytes
+)
+
+// MinFreeSpace is type that defines the limit for the minimum free space.
+type MinFreeSpace struct {
+	Type    MinFreeSpaceType
+	Bytes   uint64
+	Percent float32
+	Raw     string
+}
+
+// IsLow tells whether the free space is low or not.
+func (s MinFreeSpace) IsLow(freeBytes uint64, freePercent float32) (yes bool, desc string) {
+	switch s.Type {
+	case AsPercent:
+		yes = freePercent < s.Percent
+		op := IfElse(yes, "<", ">=")
+		return yes, fmt.Sprintf("disk free %.2f%% %s required %.2f%%", freePercent, op, s.Percent)
+	case AsBytes:
+		yes = freeBytes < s.Bytes
+		op := IfElse(yes, "<", ">=")
+		return yes, fmt.Sprintf("disk free %s %s required %s",
+			BytesToHumanReadable(freeBytes), op, BytesToHumanReadable(s.Bytes))
+	}
+
+	return false, ""
+}
+
+func (s MinFreeSpace) AvailableSpace(freeBytes uint64, totalBytes uint64) uint64 {
+	var minFreeSpaceBytes uint64
+	switch s.Type {
+	case AsPercent:
+		minFreeSpaceBytes = uint64((float32(totalBytes) * s.Percent) / 100)
+	case AsBytes:
+		minFreeSpaceBytes = s.Bytes
+	}
+	if minFreeSpaceBytes > freeBytes {
+		return 0
+	}
+	return freeBytes - minFreeSpaceBytes
+}
+
+// String returns a string representation of MinFreeSpace.
+func (s MinFreeSpace) String() string {
+	switch s.Type {
+	case AsPercent:
+		return fmt.Sprintf("%.2f%%", s.Percent)
+	default:
+		return s.Raw
+	}
+}
+
+// MustParseMinFreeSpace parses comma-separated argument for min free space setting.
+// minFreeSpace has the high priority than minFreeSpacePercent if it is set.
+func MustParseMinFreeSpace(minFreeSpace string, minFreeSpacePercent string) (spaces []MinFreeSpace) {
+	ss := strings.Split(EmptyTo(minFreeSpace, minFreeSpacePercent), ",")
+	for _, freeString := range ss {
+		if vv, e := ParseMinFreeSpace(freeString); e == nil {
+			spaces = append(spaces, *vv)
+		} else {
+			glog.Fatalf("The value specified in -minFreeSpace not a valid value %s", freeString)
+		}
+	}
+
+	return spaces
+}
+
+var ErrMinFreeSpaceBadValue = errors.New("minFreeSpace is invalid")
+
+// ParseMinFreeSpace parses min free space expression s as percentage like 1,10 or human readable size like 10G
+func ParseMinFreeSpace(s string) (*MinFreeSpace, error) {
+	s = strings.TrimSpace(s)
+	if percent, e := strconv.ParseFloat(s, 32); e == nil {
+		if percent < 0 || percent > 100 {
+			return nil, ErrMinFreeSpaceBadValue
+		}
+		return &MinFreeSpace{Type: AsPercent, Percent: float32(percent), Raw: s}, nil
+	}
+
+	if directSize, e := ParseBytes(s); e == nil {
+		if directSize <= 100 {
+			return nil, ErrMinFreeSpaceBadValue
+		}
+		return &MinFreeSpace{Type: AsBytes, Bytes: directSize, Raw: s}, nil
+	}
+
+	return nil, ErrMinFreeSpaceBadValue
+}

weed/util/minfreespace_test.go

@@ -0,0 +1,31 @@
+package util
+
+import "testing"
+
+func TestParseMinFreeSpace(t *testing.T) {
+	tests := []struct {
+		in    string
+		ok    bool
+		value *MinFreeSpace
+	}{
+		{in: "42", ok: true, value: &MinFreeSpace{Type: AsPercent, Percent: 42, Raw: "42"}},
+		{in: "-1", ok: false, value: nil},
+		{in: "101", ok: false, value: nil},
+		{in: "100B", ok: false, value: nil},
+		{in: "100Ki", ok: true, value: &MinFreeSpace{Type: AsBytes, Bytes: 100 * 1024, Raw: "100Ki"}},
+		{in: "100GiB", ok: true, value: &MinFreeSpace{Type: AsBytes, Bytes: 100 * 1024 * 1024 * 1024, Raw: "100GiB"}},
+		{in: "42M", ok: true, value: &MinFreeSpace{Type: AsBytes, Bytes: 42 * 1000 * 1000, Raw: "42M"}},
+		{in: " 42 ", ok: true, value: &MinFreeSpace{Type: AsPercent, Percent: 42, Raw: "42"}},
+		{in: " 42M ", ok: true, value: &MinFreeSpace{Type: AsBytes, Bytes: 42 * 1000 * 1000, Raw: "42M"}},
+	}
+
+	for _, p := range tests {
+		got, err := ParseMinFreeSpace(p.in)
+		if p.ok != (err == nil) {
+			t.Errorf("failed to test %v", p.in)
+		}
+		if p.ok && err == nil && *got != *p.value {
+			t.Errorf("failed to test %v", p.in)
+		}
+	}
+}

weed/util/net_timeout.go

@@ -0,0 +1,182 @@
+package util
+
+import (
+	"net"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+)
+
+// Listener wraps a net.Listener, and gives a place to store the timeout
+// parameters. On Accept, it will wrap the net.Conn with our own Conn for us.
+type Listener struct {
+	net.Listener
+	Timeout time.Duration
+}
+
+func (l *Listener) Accept() (net.Conn, error) {
+	c, err := l.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	stats.ConnectionOpen()
+	tc := &Conn{
+		Conn:    c,
+		Timeout: l.Timeout,
+	}
+	return tc, nil
+}
+
+// Conn wraps a net.Conn and implements a "no activity timeout".
+// Any activity (read or write) resets the deadline, so the connection
+// only times out when there's no activity in either direction.
+//
+// Callers may also set deadlines directly — net/http's server does this
+// to interrupt a blocked read (abortPendingRead sets a deadline in the
+// past) and to enforce its own Read/Write timeouts. The activity
+// extension must never overwrite such an externally-set deadline, or the
+// interrupt is lost and the read stays blocked until the activity
+// timeout fires (wedging connection drain on shutdown for up to the full
+// idle timeout). mu serializes deadline updates, and while an external
+// deadline is in force on a direction the activity extension for that
+// direction is suspended; it resumes once the caller clears the deadline
+// (sets the zero time).
+type Conn struct {
+	net.Conn
+	Timeout  time.Duration
+	isClosed bool
+
+	mu                    sync.Mutex
+	externalReadDeadline  bool
+	externalWriteDeadline bool
+}
+
+func (c *Conn) SetDeadline(t time.Time) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.externalReadDeadline = !t.IsZero()
+	c.externalWriteDeadline = !t.IsZero()
+	return c.Conn.SetDeadline(t)
+}
+
+func (c *Conn) SetReadDeadline(t time.Time) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.externalReadDeadline = !t.IsZero()
+	return c.Conn.SetReadDeadline(t)
+}
+
+func (c *Conn) SetWriteDeadline(t time.Time) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.externalWriteDeadline = !t.IsZero()
+	return c.Conn.SetWriteDeadline(t)
+}
+
+// extendDeadline extends the connection deadline from now.
+// This implements "no activity timeout" - any activity keeps the
+// connection alive in both directions (a long write-only response must
+// keep the read deadline alive too, or net/http's background read would
+// time out and cancel the in-flight request). Each direction is extended
+// independently: an externally-managed deadline on one side (e.g. a
+// server WriteTimeout) must not leave the other side's deadline stale.
+func (c *Conn) extendDeadline() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.Timeout <= 0 {
+		return nil
+	}
+	if !c.externalReadDeadline && !c.externalWriteDeadline {
+		return c.Conn.SetDeadline(time.Now().Add(c.Timeout))
+	}
+	if !c.externalReadDeadline {
+		if err := c.Conn.SetReadDeadline(time.Now().Add(c.Timeout)); err != nil {
+			return err
+		}
+	}
+	if !c.externalWriteDeadline {
+		if err := c.Conn.SetWriteDeadline(time.Now().Add(c.Timeout)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *Conn) Read(b []byte) (count int, e error) {
+	// Extend deadline before reading - any activity keeps connection alive
+	if err := c.extendDeadline(); err != nil {
+		return 0, err
+	}
+	count, e = c.Conn.Read(b)
+	if e == nil {
+		stats.BytesIn(int64(count))
+	}
+	return
+}
+
+func (c *Conn) Write(b []byte) (count int, e error) {
+	// Extend deadline before writing - any activity keeps connection alive
+	if err := c.extendDeadline(); err != nil {
+		return 0, err
+	}
+	count, e = c.Conn.Write(b)
+	if e == nil {
+		stats.BytesOut(int64(count))
+	}
+	return
+}
+
+func (c *Conn) Close() error {
+	err := c.Conn.Close()
+	if err == nil {
+		if !c.isClosed {
+			stats.ConnectionClose()
+			c.isClosed = true
+		}
+	}
+	return err
+}
+
+func NewListener(addr string, timeout time.Duration) (ipListener net.Listener, err error) {
+	listener, err := net.Listen("tcp", addr)
+	if err != nil {
+		return
+	}
+
+	ipListener = &Listener{
+		Listener: listener,
+		Timeout:  timeout,
+	}
+
+	return
+}
+
+func NewIpAndLocalListeners(host string, port int, timeout time.Duration) (ipListener net.Listener, localListener net.Listener, err error) {
+	listener, err := net.Listen("tcp", JoinHostPort(host, port))
+	if err != nil {
+		return
+	}
+
+	ipListener = &Listener{
+		Listener: listener,
+		Timeout:  timeout,
+	}
+
+	if host != "localhost" && host != "" && host != "0.0.0.0" && host != "127.0.0.1" && host != "[::]" && host != "[::1]" {
+		listener, err = net.Listen("tcp", JoinHostPort("localhost", port))
+		if err != nil {
+			glog.V(0).Infof("skip starting on %s:%d: %v", host, port, err)
+			return ipListener, nil, nil
+		}
+
+		localListener = &Listener{
+			Listener: listener,
+			Timeout:  timeout,
+		}
+	}
+
+	return
+}

weed/util/network.go

@@ -0,0 +1,80 @@
+package util
+
+import (
+	"net"
+	"strconv"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+func DetectedHostAddress() string {
+	netInterfaces, err := net.Interfaces()
+	if err != nil {
+		glog.V(0).Infof("failed to detect net interfaces: %v", err)
+		return ""
+	}
+
+	if v4Address := selectIpV4(netInterfaces, true); v4Address != "" {
+		return v4Address
+	}
+
+	if v6Address := selectIpV4(netInterfaces, false); v6Address != "" {
+		return v6Address
+	}
+
+	return "localhost"
+}
+
+func selectIpV4(netInterfaces []net.Interface, isIpV4 bool) string {
+	for _, netInterface := range netInterfaces {
+		if (netInterface.Flags & net.FlagUp) == 0 {
+			continue
+		}
+		addrs, err := netInterface.Addrs()
+		if err != nil {
+			glog.V(0).Infof("get interface addresses: %v", err)
+		}
+
+		for _, a := range addrs {
+			if ipNet, ok := a.(*net.IPNet); ok && !ipNet.IP.IsLoopback() {
+				if isIpV4 {
+					if ipNet.IP.To4() != nil {
+						return ipNet.IP.String()
+					}
+				} else {
+					if ipNet.IP.To4() == nil && ipNet.IP.To16() != nil {
+						// Filter out link-local IPv6 addresses (fe80::/10)
+						// They require zone identifiers and are not suitable for server binding
+						if !ipNet.IP.IsLinkLocalUnicast() {
+							return ipNet.IP.String()
+						}
+					}
+				}
+			}
+		}
+	}
+	return ""
+}
+
+func JoinHostPort(host string, port int) string {
+	return JoinHostPortStr(host, strconv.Itoa(port))
+}
+
+func JoinHostPortStr(host string, port string) string {
+	if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") {
+		return host + ":" + port
+	}
+	return net.JoinHostPort(host, port)
+}
+
+// GetVolumeServerId returns the volume server ID.
+// If id is provided (non-empty after trimming), use it as the identifier.
+// Otherwise, fall back to ip:port for backward compatibility.
+func GetVolumeServerId(id, ip string, port int) string {
+	volumeServerId := strings.TrimSpace(id)
+	if volumeServerId == "" {
+		volumeServerId = JoinHostPort(ip, port)
+	}
+	return volumeServerId
+}

weed/util/outbound_dial.go

@@ -0,0 +1,100 @@
+package util
+
+import (
+	"context"
+	"net"
+	"strings"
+	"sync/atomic"
+	"time"
+)
+
+// outboundLocalAddr is the optional source address that outbound TCP
+// connections initiated by this process bind to. It mirrors the -ip.bind
+// setting so connections this process opens leave from the configured
+// address instead of one the OS picks arbitrarily, which may not even be able
+// to reach the target. A nil value keeps the historical behavior of letting
+// the OS choose the source address.
+var (
+	outboundLocalAddr    atomic.Pointer[net.TCPAddr]
+	outboundLocalAddrSet atomic.Bool
+)
+
+// SetOutboundLocalIP records ip as the source address for outbound TCP
+// connections. An empty, wildcard (0.0.0.0 / ::), or unparseable value leaves
+// the source address to the OS.
+//
+// The first call wins. `weed server` applies the top-level -ip.bind before its
+// embedded master/volume/filer/s3/... start, so a component's own (possibly
+// different) bind setting can't later clobber the process-wide source address
+// the others already dial from.
+func SetOutboundLocalIP(ip string) {
+	if !outboundLocalAddrSet.CompareAndSwap(false, true) {
+		return
+	}
+	parsed := net.ParseIP(ip)
+	if parsed == nil || parsed.IsUnspecified() {
+		return
+	}
+	outboundLocalAddr.Store(&net.TCPAddr{IP: parsed})
+}
+
+// OutboundLocalAddr returns the configured source address for outbound TCP
+// connections, or nil if none is configured.
+func OutboundLocalAddr() *net.TCPAddr {
+	return outboundLocalAddr.Load()
+}
+
+// OutboundDialContext dials network/address with the standard library's
+// default timeouts, binding the local (source) address to the configured
+// -ip.bind value for non-loopback tcp targets. Loopback targets and non-tcp
+// networks (e.g. unix sockets) keep the OS-chosen source address: a routable
+// bind IP cannot originate a packet on the loopback interface.
+func OutboundDialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	d := net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 30 * time.Second,
+	}
+	if localAddr := outboundLocalAddrForDial(network, address); localAddr != nil {
+		d.LocalAddr = localAddr
+	}
+	return d.DialContext(ctx, network, address)
+}
+
+// outboundLocalAddrForDial returns the source address to bind for a dial to
+// network/address, or nil to leave the source address to the OS.
+func outboundLocalAddrForDial(network, address string) *net.TCPAddr {
+	localAddr := OutboundLocalAddr()
+	if localAddr == nil || !strings.HasPrefix(network, "tcp") || isLoopbackHost(address) {
+		return nil
+	}
+	// A source address can only originate a connection to a target of the same
+	// IP family. When the target is a literal IP of the other family, leave the
+	// source to the OS rather than forcing a dial that is bound to fail.
+	if host, _, err := net.SplitHostPort(address); err == nil {
+		if targetIP := net.ParseIP(host); targetIP != nil && isIPv4(targetIP) != isIPv4(localAddr.IP) {
+			return nil
+		}
+	}
+	return localAddr
+}
+
+// isIPv4 reports whether ip is an IPv4 (or IPv4-mapped) address.
+func isIPv4(ip net.IP) bool {
+	return ip.To4() != nil
+}
+
+// isLoopbackHost reports whether the host part of a "host:port" (or bare host)
+// address refers to the loopback interface.
+func isLoopbackHost(address string) bool {
+	host, _, err := net.SplitHostPort(address)
+	if err != nil {
+		host = address
+	}
+	if strings.EqualFold(host, "localhost") {
+		return true
+	}
+	if ip := net.ParseIP(host); ip != nil {
+		return ip.IsLoopback()
+	}
+	return false
+}

weed/util/outbound_dial_test.go

@@ -0,0 +1,129 @@
+package util
+
+import (
+	"testing"
+)
+
+// resetOutbound clears the process-global outbound source address so each test
+// starts from a clean slate (SetOutboundLocalIP is otherwise first-write-wins).
+func resetOutbound() {
+	outboundLocalAddrSet.Store(false)
+	outboundLocalAddr.Store(nil)
+}
+
+func TestSetOutboundLocalIP(t *testing.T) {
+	t.Cleanup(resetOutbound)
+
+	cases := []struct {
+		name   string
+		ip     string
+		wantIP string // empty means no binding
+	}{
+		{"ipv4", "10.0.0.5", "10.0.0.5"},
+		{"ipv6", "fe80::1", "fe80::1"},
+		{"empty", "", ""},
+		{"wildcard v4", "0.0.0.0", ""},
+		{"wildcard v6", "::", ""},
+		{"garbage", "not-an-ip", ""},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			resetOutbound()
+			SetOutboundLocalIP(tc.ip)
+			got := OutboundLocalAddr()
+			if tc.wantIP == "" {
+				if got != nil {
+					t.Fatalf("expected no bound address, got %v", got)
+				}
+				return
+			}
+			if got == nil {
+				t.Fatalf("expected bound address %s, got nil", tc.wantIP)
+			}
+			if got.IP.String() != tc.wantIP {
+				t.Fatalf("bound address = %s, want %s", got.IP.String(), tc.wantIP)
+			}
+			if got.Port != 0 {
+				t.Fatalf("bound port = %d, want 0 (ephemeral)", got.Port)
+			}
+		})
+	}
+}
+
+// TestSetOutboundLocalIPFirstWins guards the behavior that keeps `weed server`
+// from letting a component's own bind setting clobber the process-wide address.
+func TestSetOutboundLocalIPFirstWins(t *testing.T) {
+	t.Cleanup(resetOutbound)
+	resetOutbound()
+
+	SetOutboundLocalIP("10.0.0.5") // server-level bind, applied first
+	SetOutboundLocalIP("10.0.0.9") // a component's own bind: must be ignored
+	SetOutboundLocalIP("")         // a component clearing: must not unbind
+
+	got := OutboundLocalAddr()
+	if got == nil || got.IP.String() != "10.0.0.5" {
+		t.Fatalf("first call should win, got %v", got)
+	}
+}
+
+func TestOutboundLocalAddrForDial(t *testing.T) {
+	t.Cleanup(resetOutbound)
+
+	// No bind configured: never binds a source address.
+	resetOutbound()
+	if got := outboundLocalAddrForDial("tcp", "10.0.0.9:8080"); got != nil {
+		t.Fatalf("unconfigured dial should not bind, got %v", got)
+	}
+
+	resetOutbound()
+	SetOutboundLocalIP("10.0.0.5")
+	cases := []struct {
+		name     string
+		network  string
+		address  string
+		wantBind bool
+	}{
+		{"remote tcp binds", "tcp", "10.0.0.9:8080", true},
+		{"tcp4 binds", "tcp4", "example.com:443", true},
+		{"hostname binds", "tcp", "filer.internal:8888", true},
+		{"loopback ip skipped", "tcp", "127.0.0.1:9333", false},
+		{"loopback name skipped", "tcp", "localhost:9333", false},
+		{"ipv6 loopback skipped", "tcp", "[::1]:9333", false},
+		{"ipv6 literal target skipped", "tcp", "[2001:db8::1]:8080", false},
+		{"unix network skipped", "unix", "/tmp/x.sock", false},
+		{"udp network skipped", "udp", "10.0.0.9:53", false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := outboundLocalAddrForDial(tc.network, tc.address)
+			if tc.wantBind && got == nil {
+				t.Fatalf("%s/%s: expected source binding, got nil", tc.network, tc.address)
+			}
+			if !tc.wantBind && got != nil {
+				t.Fatalf("%s/%s: expected no source binding, got %v", tc.network, tc.address, got)
+			}
+		})
+	}
+}
+
+func TestOutboundLocalAddrFamilyMismatch(t *testing.T) {
+	t.Cleanup(resetOutbound)
+
+	resetOutbound()
+	SetOutboundLocalIP("10.0.0.5") // IPv4 source
+	if got := outboundLocalAddrForDial("tcp", "[2001:db8::1]:8080"); got != nil {
+		t.Fatalf("IPv4 source to IPv6 literal target should skip binding, got %v", got)
+	}
+	if got := outboundLocalAddrForDial("tcp", "10.0.0.9:8080"); got == nil {
+		t.Fatalf("IPv4 source to IPv4 literal target should bind")
+	}
+
+	resetOutbound()
+	SetOutboundLocalIP("2001:db8::5") // IPv6 source
+	if got := outboundLocalAddrForDial("tcp", "10.0.0.9:8080"); got != nil {
+		t.Fatalf("IPv6 source to IPv4 literal target should skip binding, got %v", got)
+	}
+	if got := outboundLocalAddrForDial("tcp", "[2001:db8::1]:8080"); got == nil {
+		t.Fatalf("IPv6 source to IPv6 literal target should bind")
+	}
+}

weed/util/parse.go

@@ -0,0 +1,81 @@
+package util
+
+import (
+	"fmt"
+	"net"
+	"net/url"
+	"strconv"
+	"strings"
+)
+
+func ParseInt(text string, defaultValue int) int {
+	count, parseError := strconv.ParseInt(text, 10, 64)
+	if parseError != nil {
+		if len(text) > 0 {
+			return 0
+		}
+		return defaultValue
+	}
+	return int(count)
+}
+func ParseUint64(text string, defaultValue uint64) uint64 {
+	count, parseError := strconv.ParseUint(text, 10, 64)
+	if parseError != nil {
+		if len(text) > 0 {
+			return 0
+		}
+		return defaultValue
+	}
+	return count
+}
+
+func ParseBool(s string, defaultValue bool) bool {
+	value, err := strconv.ParseBool(s)
+	if err != nil {
+		return defaultValue
+	}
+	return value
+}
+
+func BoolToString(b bool) string {
+	return strconv.FormatBool(b)
+}
+
+func ParseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error) {
+	if !strings.HasPrefix(entryPath, "http://") && !strings.HasPrefix(entryPath, "https://") {
+		entryPath = "http://" + entryPath
+	}
+
+	var u *url.URL
+	u, err = url.Parse(entryPath)
+	if err != nil {
+		return
+	}
+	filerServer = u.Hostname()
+	portString := u.Port()
+	if portString != "" {
+		filerPort, err = strconv.ParseInt(portString, 10, 32)
+	}
+	path = u.Path
+	return
+}
+
+func ParseHostPort(hostPort string) (filerServer string, filerPort int64, err error) {
+	host, port, err := net.SplitHostPort(hostPort)
+	if err != nil {
+		err = fmt.Errorf("failed to parse %s", hostPort)
+		return
+	}
+
+	filerPort, err = strconv.ParseInt(port, 10, 64)
+	if err == nil {
+		filerServer = host
+	}
+
+	return
+}
+
+func CanonicalizeETag(etag string) string {
+	canonicalETag := strings.TrimPrefix(etag, "\"")
+	return strings.TrimSuffix(canonicalETag, "\"")
+}

weed/util/parse_test.go

@@ -0,0 +1,57 @@
+package util
+
+import "testing"
+
+func TestParseHostPort(t *testing.T) {
+	tests := []struct {
+		name       string
+		hostPort   string
+		wantHost   string
+		wantPort   int64
+		wantErr    bool
+	}{
+		{
+			name:     "hostname",
+			hostPort: "localhost:8888",
+			wantHost: "localhost",
+			wantPort: 8888,
+		},
+		{
+			name:     "ipv4",
+			hostPort: "127.0.0.1:8888",
+			wantHost: "127.0.0.1",
+			wantPort: 8888,
+		},
+		{
+			name:     "bracketed ipv6",
+			hostPort: "[::1]:8888",
+			wantHost: "::1",
+			wantPort: 8888,
+		},
+		{
+			name:     "missing port",
+			hostPort: "localhost",
+			wantErr:  true,
+		},
+		{
+			name:     "invalid port",
+			hostPort: "localhost:bad",
+			wantErr:  true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotHost, gotPort, err := ParseHostPort(tt.hostPort)
+			if gotErr := err != nil; gotErr != tt.wantErr {
+				t.Fatalf("ParseHostPort(%q) error = %v, wantErr %v", tt.hostPort, err, tt.wantErr)
+			}
+			if tt.wantErr {
+				return
+			}
+			if gotHost != tt.wantHost || gotPort != tt.wantPort {
+				t.Fatalf("ParseHostPort(%q) = (%q, %d), want (%q, %d)", tt.hostPort, gotHost, gotPort, tt.wantHost, tt.wantPort)
+			}
+		})
+	}
+}

weed/util/path_filter.go

@@ -0,0 +1,128 @@
+package util
+
+import (
+	"strings"
+)
+
+// PathPrefixFilter provides filtering based on include and exclude path prefixes.
+// When both include and exclude prefixes match a path, the deepest matching prefix wins.
+// This enables fine-grained control like: exclude /buckets/legacy but include /buckets/legacy/important
+type PathPrefixFilter struct {
+	includePrefixes []string // normalized with trailing /
+	excludePrefixes []string // normalized with trailing /
+}
+
+// NewPathPrefixFilter creates a new PathPrefixFilter from comma-separated include and exclude prefix strings.
+// Each prefix is normalized to have a trailing slash for directory boundary matching.
+// Invalid prefixes (empty or not starting with /) are skipped with a warning via the provided warn function.
+func NewPathPrefixFilter(includePrefixes, excludePrefixes string, warn func(format string, args ...interface{})) *PathPrefixFilter {
+	pf := &PathPrefixFilter{}
+
+	pf.includePrefixes = parsePrefixes(includePrefixes, warn)
+	pf.excludePrefixes = parsePrefixes(excludePrefixes, warn)
+
+	return pf
+}
+
+// parsePrefixes parses a comma-separated list of prefixes and normalizes them.
+func parsePrefixes(prefixList string, warn func(format string, args ...interface{})) []string {
+	if prefixList == "" {
+		return nil
+	}
+
+	var result []string
+	for _, p := range strings.Split(prefixList, ",") {
+		p = strings.TrimSpace(p)
+		if p == "" {
+			continue
+		}
+		if !strings.HasPrefix(p, "/") {
+			if warn != nil {
+				warn("prefix %q does not start with '/', skipping", p)
+			}
+			continue
+		}
+		// Normalize: ensure trailing slash for directory boundary matching
+		if !strings.HasSuffix(p, "/") {
+			p = p + "/"
+		}
+		result = append(result, p)
+	}
+	return result
+}
+
+// HasFilters returns true if any include or exclude prefixes are configured.
+func (pf *PathPrefixFilter) HasFilters() bool {
+	return len(pf.includePrefixes) > 0 || len(pf.excludePrefixes) > 0
+}
+
+// ShouldInclude returns true if the path should be included based on the configured prefixes.
+//
+// Logic:
+//   - If no filters are configured, include everything.
+//   - Find the deepest matching prefix from either include or exclude list.
+//   - If the deepest match is in includePrefixes, include the path.
+//   - If the deepest match is in excludePrefixes, exclude the path.
+//   - If no match is found and includePrefixes is non-empty, exclude (explicit include required).
+//   - If no match is found and includePrefixes is empty, include (default allow with excludes).
+func (pf *PathPrefixFilter) ShouldInclude(fullpath string) bool {
+	if !pf.HasFilters() {
+		return true
+	}
+
+	// Normalize path for matching
+	checkPath := fullpath
+	if !strings.HasSuffix(checkPath, "/") {
+		checkPath = checkPath + "/"
+	}
+
+	// Find deepest matching prefix from each list
+	includeMatch := findDeepestMatch(checkPath, pf.includePrefixes)
+	excludeMatch := findDeepestMatch(checkPath, pf.excludePrefixes)
+
+	// Determine result based on which match is deeper
+	if includeMatch != "" && excludeMatch != "" {
+		// Both matched - deeper prefix wins
+		return len(includeMatch) >= len(excludeMatch)
+	}
+
+	if includeMatch != "" {
+		return true
+	}
+
+	if excludeMatch != "" {
+		return false
+	}
+
+	// No match found
+	if len(pf.includePrefixes) > 0 {
+		// If includes are specified, require explicit include
+		return false
+	}
+
+	// Default: include if only excludes are specified
+	return true
+}
+
+// findDeepestMatch finds the longest prefix that matches the path.
+func findDeepestMatch(path string, prefixes []string) string {
+	var deepest string
+	for _, prefix := range prefixes {
+		if strings.HasPrefix(path, prefix) {
+			if len(prefix) > len(deepest) {
+				deepest = prefix
+			}
+		}
+	}
+	return deepest
+}
+
+// GetIncludePrefixes returns the configured include prefixes.
+func (pf *PathPrefixFilter) GetIncludePrefixes() []string {
+	return pf.includePrefixes
+}
+
+// GetExcludePrefixes returns the configured exclude prefixes.
+func (pf *PathPrefixFilter) GetExcludePrefixes() []string {
+	return pf.excludePrefixes
+}

weed/util/path_filter_test.go

@@ -0,0 +1,201 @@
+package util
+
+import (
+	"testing"
+)
+
+func TestPathPrefixFilter_Empty(t *testing.T) {
+	pf := NewPathPrefixFilter("", "", nil)
+
+	if pf.HasFilters() {
+		t.Error("empty filter should have no filters")
+	}
+
+	// Should include everything when no filters
+	tests := []string{"/", "/foo", "/foo/bar", "/buckets/test"}
+	for _, path := range tests {
+		if !pf.ShouldInclude(path) {
+			t.Errorf("empty filter should include %q", path)
+		}
+	}
+}
+
+func TestPathPrefixFilter_ExcludeOnly(t *testing.T) {
+	pf := NewPathPrefixFilter("", "/buckets/legacy,/buckets/old", nil)
+
+	tests := []struct {
+		path    string
+		include bool
+	}{
+		{"/buckets/active", true},
+		{"/buckets/active/file.txt", true},
+		{"/buckets/legacy", false},
+		{"/buckets/legacy/file.txt", false},
+		{"/buckets/legacy_new", true}, // boundary check: not a prefix match
+		{"/buckets/old", false},
+		{"/buckets/old/data", false},
+		{"/other", true},
+	}
+
+	for _, tc := range tests {
+		got := pf.ShouldInclude(tc.path)
+		if got != tc.include {
+			t.Errorf("ShouldInclude(%q) = %v, want %v", tc.path, got, tc.include)
+		}
+	}
+}
+
+func TestPathPrefixFilter_IncludeOnly(t *testing.T) {
+	pf := NewPathPrefixFilter("/buckets/important,/data", "", nil)
+
+	tests := []struct {
+		path    string
+		include bool
+	}{
+		{"/buckets/important", true},
+		{"/buckets/important/file.txt", true},
+		{"/data", true},
+		{"/data/file.txt", true},
+		{"/buckets/other", false}, // not in include list
+		{"/other", false},
+	}
+
+	for _, tc := range tests {
+		got := pf.ShouldInclude(tc.path)
+		if got != tc.include {
+			t.Errorf("ShouldInclude(%q) = %v, want %v", tc.path, got, tc.include)
+		}
+	}
+}
+
+func TestPathPrefixFilter_DeeperPrefixWins(t *testing.T) {
+	// Exclude /buckets/keep but include /buckets/keep/important
+	pf := NewPathPrefixFilter("/buckets/keep/important", "/buckets/keep", nil)
+
+	tests := []struct {
+		path    string
+		include bool
+	}{
+		{"/buckets/keep", false},
+		{"/buckets/keep/other", false},
+		{"/buckets/keep/important", true},          // deeper include wins
+		{"/buckets/keep/important/file.txt", true}, // deeper include wins
+		{"/buckets/other", false},                  // not matched, include required
+	}
+
+	for _, tc := range tests {
+		got := pf.ShouldInclude(tc.path)
+		if got != tc.include {
+			t.Errorf("ShouldInclude(%q) = %v, want %v", tc.path, got, tc.include)
+		}
+	}
+}
+
+func TestPathPrefixFilter_DeeperExcludeWins(t *testing.T) {
+	// Include /buckets but exclude /buckets/legacy
+	pf := NewPathPrefixFilter("/buckets", "/buckets/legacy", nil)
+
+	tests := []struct {
+		path    string
+		include bool
+	}{
+		{"/buckets", true},
+		{"/buckets/active", true},
+		{"/buckets/legacy", false},          // deeper exclude wins
+		{"/buckets/legacy/file.txt", false}, // deeper exclude wins
+		{"/other", false},                   // not in include list
+	}
+
+	for _, tc := range tests {
+		got := pf.ShouldInclude(tc.path)
+		if got != tc.include {
+			t.Errorf("ShouldInclude(%q) = %v, want %v", tc.path, got, tc.include)
+		}
+	}
+}
+
+func TestPathPrefixFilter_MultipleOverlappingPrefixes(t *testing.T) {
+	// Complex scenario with multiple overlapping prefixes
+	pf := NewPathPrefixFilter(
+		"/a,/a/b/c/d",     // includes
+		"/a/b,/a/b/c/d/e", // excludes
+		nil,
+	)
+
+	tests := []struct {
+		path    string
+		include bool
+	}{
+		{"/a", true},            // direct include match
+		{"/a/x", true},          // under include /a
+		{"/a/b", false},         // deeper exclude /a/b beats /a
+		{"/a/b/x", false},       // under exclude /a/b
+		{"/a/b/c", false},       // under exclude /a/b
+		{"/a/b/c/d", true},      // deeper include /a/b/c/d beats /a/b
+		{"/a/b/c/d/x", true},    // under include /a/b/c/d
+		{"/a/b/c/d/e", false},   // deeper exclude /a/b/c/d/e beats /a/b/c/d
+		{"/a/b/c/d/e/f", false}, // under exclude /a/b/c/d/e
+	}
+
+	for _, tc := range tests {
+		got := pf.ShouldInclude(tc.path)
+		if got != tc.include {
+			t.Errorf("ShouldInclude(%q) = %v, want %v", tc.path, got, tc.include)
+		}
+	}
+}
+
+func TestPathPrefixFilter_InvalidPrefixes(t *testing.T) {
+	var warnings []string
+	warn := func(format string, args ...interface{}) {
+		warnings = append(warnings, format)
+	}
+
+	pf := NewPathPrefixFilter("invalid,/valid", "also_invalid", warn)
+
+	if len(warnings) != 2 {
+		t.Errorf("expected 2 warnings, got %d", len(warnings))
+	}
+
+	// Only valid prefix should be stored
+	if len(pf.includePrefixes) != 1 {
+		t.Errorf("expected 1 include prefix, got %d", len(pf.includePrefixes))
+	}
+	if len(pf.excludePrefixes) != 0 {
+		t.Errorf("expected 0 exclude prefixes, got %d", len(pf.excludePrefixes))
+	}
+}
+
+func TestPathPrefixFilter_TrailingSlashNormalization(t *testing.T) {
+	pf := NewPathPrefixFilter("/path/to/dir", "/exclude/this/", nil)
+
+	// Both should be normalized with trailing slash
+	if pf.includePrefixes[0] != "/path/to/dir/" {
+		t.Errorf("include prefix not normalized: %q", pf.includePrefixes[0])
+	}
+	if pf.excludePrefixes[0] != "/exclude/this/" {
+		t.Errorf("exclude prefix not normalized: %q", pf.excludePrefixes[0])
+	}
+}
+
+func TestPathPrefixFilter_BoundaryMatching(t *testing.T) {
+	pf := NewPathPrefixFilter("", "/buckets/legacy1", nil)
+
+	tests := []struct {
+		path    string
+		include bool
+	}{
+		{"/buckets/legacy1", false},
+		{"/buckets/legacy1/file", false},
+		{"/buckets/legacy1_backup", true}, // not a prefix match due to boundary
+		{"/buckets/legacy10", true},       // not a prefix match due to boundary
+		{"/buckets/legacy", true},
+	}
+
+	for _, tc := range tests {
+		got := pf.ShouldInclude(tc.path)
+		if got != tc.include {
+			t.Errorf("ShouldInclude(%q) = %v, want %v", tc.path, got, tc.include)
+		}
+	}
+}

weed/util/pgxutil/pgx_conn.go

@@ -0,0 +1,158 @@
+// Package pgxutil holds shared helpers for opening *sql.DB handles backed
+// by jackc/pgx, used by the postgres filer and credential stores so they
+// stay consistent on connection setup, mTLS handling and PgBouncer
+// compatibility.
+package pgxutil
+
+import (
+	"database/sql"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/stdlib"
+)
+
+// DSNOptions describes the parameters used to assemble a libpq-style
+// keyword=value connection string. Empty fields are omitted.
+type DSNOptions struct {
+	Hostname    string
+	Port        int
+	User        string
+	Password    string
+	Database    string
+	Schema      string
+	SSLMode     string
+	SSLCert     string
+	SSLKey      string
+	SSLRootCert string
+	SSLCRL      string
+	// PgBouncerCompatible toggles two things at the caller's site: the
+	// search_path is omitted from the DSN (PgBouncer rejects it under
+	// transaction pooling) and OpenDB is told to use the simple query
+	// protocol.
+	PgBouncerCompatible bool
+}
+
+// quoteDSNValue wraps v per the libpq keyword/value rules: empty values
+// and values containing whitespace, single quotes, or backslashes are
+// single-quoted with internal `'` and `\` escaped. Plain alphanumeric
+// values are returned unchanged. See PostgreSQL docs §"Connection Strings".
+func quoteDSNValue(v string) string {
+	if v == "" {
+		return "''"
+	}
+	needsQuote := false
+	for i := 0; i < len(v); i++ {
+		c := v[i]
+		if c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\'' || c == '\\' {
+			needsQuote = true
+			break
+		}
+	}
+	if !needsQuote {
+		return v
+	}
+	var b strings.Builder
+	b.Grow(len(v) + 2)
+	b.WriteByte('\'')
+	for i := 0; i < len(v); i++ {
+		c := v[i]
+		if c == '\'' || c == '\\' {
+			b.WriteByte('\\')
+		}
+		b.WriteByte(c)
+	}
+	b.WriteByte('\'')
+	return b.String()
+}
+
+// BuildDSN assembles two libpq-style connection strings from opts: the
+// real DSN passed to pgx, and an adapted copy with the password redacted
+// for use in error messages and logs. Values containing spaces, single
+// quotes, or backslashes are quoted per libpq rules so passwords and
+// cert paths sourced from secret managers can carry arbitrary characters
+// without breaking pgx.ParseConfig.
+func BuildDSN(opts DSNOptions) (dsn, adaptedDSN string) {
+	dsn = "connect_timeout=30"
+	if opts.Hostname != "" {
+		dsn += " host=" + quoteDSNValue(opts.Hostname)
+	}
+	if opts.Port != 0 {
+		dsn += " port=" + strconv.Itoa(opts.Port)
+	}
+	if opts.SSLMode != "" {
+		dsn += " sslmode=" + quoteDSNValue(opts.SSLMode)
+	}
+	if opts.SSLCert != "" {
+		dsn += " sslcert=" + quoteDSNValue(opts.SSLCert)
+	}
+	if opts.SSLKey != "" {
+		dsn += " sslkey=" + quoteDSNValue(opts.SSLKey)
+	}
+	if opts.SSLRootCert != "" {
+		dsn += " sslrootcert=" + quoteDSNValue(opts.SSLRootCert)
+	}
+	if opts.SSLCRL != "" {
+		dsn += " sslcrl=" + quoteDSNValue(opts.SSLCRL)
+	}
+	if opts.User != "" {
+		dsn += " user=" + quoteDSNValue(opts.User)
+	}
+	adaptedDSN = dsn
+	if opts.Password != "" {
+		dsn += " password=" + quoteDSNValue(opts.Password)
+		adaptedDSN += " password=ADAPTED"
+	}
+	if opts.Database != "" {
+		dsn += " dbname=" + quoteDSNValue(opts.Database)
+		adaptedDSN += " dbname=" + quoteDSNValue(opts.Database)
+	}
+	if opts.Schema != "" && !opts.PgBouncerCompatible {
+		dsn += " search_path=" + quoteDSNValue(opts.Schema)
+		adaptedDSN += " search_path=" + quoteDSNValue(opts.Schema)
+	}
+	return dsn, adaptedDSN
+}
+
+// OpenDB parses dsn into a pgx ConnConfig, applies PgBouncer compatibility
+// settings when requested, opens a *sql.DB via stdlib.OpenDB, and verifies
+// it with Ping.
+//
+// In pgx/v5 the prefer_simple_protocol DSN parameter was removed, so simple
+// protocol mode must be configured on the ConnConfig via DefaultQueryExecMode.
+// We use stdlib.OpenDB(config) rather than RegisterConnConfig + sql.Open so
+// we don't leak entries in stdlib's global connection config map on either
+// success or failure paths.
+//
+// adaptedDSN is used only for error messages (the caller is expected to
+// have redacted any password).
+func OpenDB(dsn, adaptedDSN string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (*sql.DB, error) {
+	connConfig, parseErr := pgx.ParseConfig(dsn)
+	if parseErr != nil {
+		return nil, fmt.Errorf("can not parse connection config for %s error:%v", adaptedDSN, parseErr)
+	}
+
+	// PgBouncer compatibility: use the simple query protocol and disable
+	// statement caching. This avoids prepared statement issues with
+	// PgBouncer's transaction pooling mode.
+	if pgbouncerCompatible {
+		connConfig.DefaultQueryExecMode = pgx.QueryExecModeSimpleProtocol
+		connConfig.StatementCacheCapacity = 0
+		connConfig.DescriptionCacheCapacity = 0
+	}
+
+	db := stdlib.OpenDB(*connConfig)
+	db.SetMaxIdleConns(maxIdle)
+	db.SetMaxOpenConns(maxOpen)
+	db.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)
+
+	if err := db.Ping(); err != nil {
+		db.Close()
+		return nil, fmt.Errorf("connect to %s error:%v", adaptedDSN, err)
+	}
+
+	return db, nil
+}

weed/util/pgxutil/pgx_conn_test.go

@@ -0,0 +1,59 @@
+package pgxutil
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/jackc/pgx/v5"
+)
+
+func TestQuoteDSNValue(t *testing.T) {
+	cases := []struct {
+		in, want string
+	}{
+		{"", "''"},
+		{"plain", "plain"},
+		{"with space", "'with space'"},
+		{"it's", `'it\'s'`},
+		{`back\slash`, `'back\\slash'`},
+		{`mix it's \and spaces`, `'mix it\'s \\and spaces'`},
+	}
+	for _, c := range cases {
+		if got := quoteDSNValue(c.in); got != c.want {
+			t.Errorf("quoteDSNValue(%q) = %q, want %q", c.in, got, c.want)
+		}
+	}
+}
+
+// TestBuildDSN_QuotesProblematicValues verifies that the assembled DSN
+// stays parseable by pgx when values contain spaces, single quotes or
+// backslashes — the kind of password/cert paths a secret manager can
+// hand us in production.
+func TestBuildDSN_QuotesProblematicValues(t *testing.T) {
+	opts := DSNOptions{
+		Hostname: "db.example.com",
+		Port:     5432,
+		User:     "iam user",
+		Password: `p@ss it's word\\`,
+		Database: "weed",
+		Schema:   "public",
+		SSLMode:  "disable",
+	}
+	dsn, adapted := BuildDSN(opts)
+	if strings.Contains(adapted, opts.Password) {
+		t.Errorf("adapted DSN must redact the password, got %q", adapted)
+	}
+	cfg, err := pgx.ParseConfig(dsn)
+	if err != nil {
+		t.Fatalf("pgx.ParseConfig rejected the assembled DSN: %v\nDSN: %s", err, dsn)
+	}
+	if cfg.User != opts.User {
+		t.Errorf("User: got %q want %q", cfg.User, opts.User)
+	}
+	if cfg.Password != opts.Password {
+		t.Errorf("Password: got %q want %q", cfg.Password, opts.Password)
+	}
+	if cfg.Database != opts.Database {
+		t.Errorf("Database: got %q want %q", cfg.Database, opts.Database)
+	}
+}

weed/util/queue.go

@@ -0,0 +1,74 @@
+package util
+
+import (
+	"sync"
+)
+
+type node[T any] struct {
+	data T
+	next *node[T]
+}
+
+type Queue[T any] struct {
+	head  *node[T]
+	tail  *node[T]
+	count int
+	sync.RWMutex
+}
+
+func NewQueue[T any]() *Queue[T] {
+	q := &Queue[T]{}
+	return q
+}
+
+func (q *Queue[T]) Len() int {
+	q.RLock()
+	defer q.RUnlock()
+	return q.count
+}
+
+func (q *Queue[T]) Enqueue(item T) {
+	q.Lock()
+	defer q.Unlock()
+
+	n := &node[T]{data: item}
+
+	if q.tail == nil {
+		q.tail = n
+		q.head = n
+	} else {
+		q.tail.next = n
+		q.tail = n
+	}
+	q.count++
+}
+
+func (q *Queue[T]) Dequeue() (result T) {
+	q.Lock()
+	defer q.Unlock()
+
+	if q.head == nil {
+		return
+	}
+
+	n := q.head
+	q.head = n.next
+
+	if q.head == nil {
+		q.tail = nil
+	}
+	q.count--
+
+	return n.data
+}
+
+func (q *Queue[T]) Peek() (result T) {
+	q.RLock()
+	defer q.RUnlock()
+
+	if q.head == nil {
+		return
+	}
+
+	return q.head.data
+}

weed/util/queue_test.go

@@ -0,0 +1,23 @@
+package util
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewQueue(t *testing.T) {
+
+	q := NewQueue[int]()
+
+	for i := 0; i < 10; i++ {
+		q.Enqueue(i)
+	}
+
+	assert.Equal(t, q.Len(), 10)
+
+	for i := 0; i < 10; i++ {
+		assert.Equal(t, q.Dequeue(), i)
+	}
+
+}

weed/util/queue_unbounded.go

@@ -0,0 +1,45 @@
+package util
+
+import "sync"
+
+type UnboundedQueue struct {
+	outbound     []string
+	outboundLock sync.RWMutex
+	inbound      []string
+	inboundLock  sync.RWMutex
+}
+
+func NewUnboundedQueue() *UnboundedQueue {
+	q := &UnboundedQueue{}
+	return q
+}
+
+func (q *UnboundedQueue) EnQueue(items ...string) {
+	q.inboundLock.Lock()
+	defer q.inboundLock.Unlock()
+
+	q.inbound = append(q.inbound, items...)
+
+}
+
+func (q *UnboundedQueue) Consume(fn func([]string)) {
+	q.outboundLock.Lock()
+	defer q.outboundLock.Unlock()
+
+	if len(q.outbound) == 0 {
+		q.inboundLock.Lock()
+		inboundLen := len(q.inbound)
+		if inboundLen > 0 {
+			t := q.outbound
+			q.outbound = q.inbound
+			q.inbound = t
+		}
+		q.inboundLock.Unlock()
+	}
+
+	if len(q.outbound) > 0 {
+		fn(q.outbound)
+		q.outbound = q.outbound[:0]
+	}
+
+}

weed/util/queue_unbounded_test.go

@@ -0,0 +1,25 @@
+package util
+
+import "testing"
+
+func TestEnqueueAndConsume(t *testing.T) {
+
+	q := NewUnboundedQueue()
+
+	q.EnQueue("1", "2", "3")
+
+	f := func(items []string) {
+		for _, t := range items {
+			println(t)
+		}
+		println("-----------------------")
+	}
+	q.Consume(f)
+
+	q.Consume(f)
+
+	q.EnQueue("4", "5")
+	q.EnQueue("6", "7")
+	q.Consume(f)
+
+}

weed/util/reflect.go

@@ -0,0 +1,40 @@
+package util
+
+import "reflect"
+
+// IsNotEmpty returns true if the given value is not zero or empty.
+func IsNotEmpty(given interface{}) bool {
+	return !IsEmpty(given)
+}
+
+// IsEmpty returns true if the given value has the zero value for its type.
+func IsEmpty(given interface{}) bool {
+	g := reflect.ValueOf(given)
+	if !g.IsValid() {
+		return true
+	}
+
+	if g.Kind() == reflect.Ptr {
+		g = g.Elem()
+	}
+
+	// Basically adapted from text/template.isTrue
+	switch g.Kind() {
+	case reflect.Array, reflect.Slice, reflect.Map, reflect.String:
+		return g.Len() == 0
+	case reflect.Bool:
+		return !g.Bool()
+	case reflect.Complex64, reflect.Complex128:
+		return g.Complex() == 0
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return g.Int() == 0
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		return g.Uint() == 0
+	case reflect.Float32, reflect.Float64:
+		return g.Float() == 0
+	case reflect.Struct:
+		return g.IsZero()
+	default:
+		return g.IsNil()
+	}
+}

weed/util/request_id/request_id.go

@@ -0,0 +1,70 @@
+package request_id
+
+import (
+	"context"
+	"crypto/rand"
+	"fmt"
+	"net/http"
+	"time"
+)
+
+const AmzRequestIDHeader = "x-amz-request-id"
+
+type contextKey struct{}
+
+func Set(ctx context.Context, id string) context.Context {
+	return context.WithValue(ctx, contextKey{}, id)
+}
+
+func Get(ctx context.Context) string {
+	if ctx == nil {
+		return ""
+	}
+	id, _ := ctx.Value(contextKey{}).(string)
+	return id
+}
+
+func InjectToRequest(ctx context.Context, req *http.Request) {
+	if req != nil {
+		req.Header.Set(AmzRequestIDHeader, Get(ctx))
+	}
+}
+
+func New() string {
+	var buf [4]byte
+	rand.Read(buf[:])
+	return fmt.Sprintf("%X%08X", time.Now().UTC().UnixNano(), buf)
+}
+
+// GetFromRequest returns the server-generated request ID from the context.
+func GetFromRequest(r *http.Request) string {
+	if r == nil {
+		return ""
+	}
+	return Get(r.Context())
+}
+
+// Ensure guarantees a server-generated request ID exists in the context.
+// It always generates a new ID if one is not already present in the context,
+// ignoring any client-sent x-amz-request-id header to prevent spoofing.
+func Ensure(r *http.Request) (*http.Request, string) {
+	if r == nil {
+		return nil, ""
+	}
+	if id := Get(r.Context()); id != "" {
+		return r, id
+	}
+	id := New()
+	r = r.WithContext(Set(r.Context(), id))
+	return r, id
+}
+
+func Middleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		r, reqID := Ensure(r)
+		if w.Header().Get(AmzRequestIDHeader) == "" {
+			w.Header().Set(AmzRequestIDHeader, reqID)
+		}
+		next.ServeHTTP(w, r)
+	})
+}