snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/util/log_buffer/disk_buffer_cache.go

@@ -0,0 +1,195 @@
+package log_buffer
+
+import (
+	"container/list"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+// DiskBufferCache is a small LRU cache for recently-read historical data buffers
+// This reduces Filer load when multiple consumers are catching up on historical messages
+type DiskBufferCache struct {
+	maxSize   int
+	ttl       time.Duration
+	cache     map[string]*cacheEntry
+	lruList   *list.List
+	mu        sync.RWMutex
+	hits      int64
+	misses    int64
+	evictions int64
+}
+
+type cacheEntry struct {
+	key        string
+	data       []byte
+	offset     int64
+	timestamp  time.Time
+	lruElement *list.Element
+	isNegative bool // true if this is a negative cache entry (data not found)
+}
+
+// NewDiskBufferCache creates a new cache with the specified size and TTL
+// Recommended size: 3-5 buffers (each ~8MB)
+// Recommended TTL: 30-60 seconds
+func NewDiskBufferCache(maxSize int, ttl time.Duration) *DiskBufferCache {
+	cache := &DiskBufferCache{
+		maxSize: maxSize,
+		ttl:     ttl,
+		cache:   make(map[string]*cacheEntry),
+		lruList: list.New(),
+	}
+
+	// Start background cleanup goroutine
+	go cache.cleanupLoop()
+
+	return cache
+}
+
+// Get retrieves a buffer from the cache
+// Returns (data, offset, found)
+// If found=true and data=nil, this is a negative cache entry (data doesn't exist)
+func (c *DiskBufferCache) Get(key string) ([]byte, int64, bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	entry, exists := c.cache[key]
+	if !exists {
+		c.misses++
+		return nil, 0, false
+	}
+
+	// Check if entry has expired
+	if time.Since(entry.timestamp) > c.ttl {
+		c.evict(entry)
+		c.misses++
+		return nil, 0, false
+	}
+
+	// Move to front of LRU list (most recently used)
+	c.lruList.MoveToFront(entry.lruElement)
+	c.hits++
+
+	if entry.isNegative {
+		glog.V(4).Infof("📦 CACHE HIT (NEGATIVE): key=%s - data not found (hits=%d misses=%d)",
+			key, c.hits, c.misses)
+	} else {
+		glog.V(4).Infof("📦 CACHE HIT: key=%s offset=%d size=%d (hits=%d misses=%d)",
+			key, entry.offset, len(entry.data), c.hits, c.misses)
+	}
+
+	return entry.data, entry.offset, true
+}
+
+// Put adds a buffer to the cache
+// If data is nil, this creates a negative cache entry (data doesn't exist)
+func (c *DiskBufferCache) Put(key string, data []byte, offset int64) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	isNegative := data == nil
+
+	// Check if entry already exists
+	if entry, exists := c.cache[key]; exists {
+		// Update existing entry
+		entry.data = data
+		entry.offset = offset
+		entry.timestamp = time.Now()
+		entry.isNegative = isNegative
+		c.lruList.MoveToFront(entry.lruElement)
+		if isNegative {
+			glog.V(4).Infof("📦 CACHE UPDATE (NEGATIVE): key=%s - data not found", key)
+		} else {
+			glog.V(4).Infof("📦 CACHE UPDATE: key=%s offset=%d size=%d", key, offset, len(data))
+		}
+		return
+	}
+
+	// Evict oldest entry if cache is full
+	if c.lruList.Len() >= c.maxSize {
+		oldest := c.lruList.Back()
+		if oldest != nil {
+			c.evict(oldest.Value.(*cacheEntry))
+		}
+	}
+
+	// Add new entry
+	entry := &cacheEntry{
+		key:        key,
+		data:       data,
+		offset:     offset,
+		timestamp:  time.Now(),
+		isNegative: isNegative,
+	}
+	entry.lruElement = c.lruList.PushFront(entry)
+	c.cache[key] = entry
+
+	if isNegative {
+		glog.V(4).Infof("📦 CACHE PUT (NEGATIVE): key=%s - data not found (cache_size=%d/%d)",
+			key, c.lruList.Len(), c.maxSize)
+	} else {
+		glog.V(4).Infof("📦 CACHE PUT: key=%s offset=%d size=%d (cache_size=%d/%d)",
+			key, offset, len(data), c.lruList.Len(), c.maxSize)
+	}
+}
+
+// evict removes an entry from the cache (must be called with lock held)
+func (c *DiskBufferCache) evict(entry *cacheEntry) {
+	delete(c.cache, entry.key)
+	c.lruList.Remove(entry.lruElement)
+	c.evictions++
+	glog.V(4).Infof("📦 CACHE EVICT: key=%s (evictions=%d)", entry.key, c.evictions)
+}
+
+// cleanupLoop periodically removes expired entries
+func (c *DiskBufferCache) cleanupLoop() {
+	ticker := time.NewTicker(c.ttl / 2)
+	defer ticker.Stop()
+
+	for range ticker.C {
+		c.cleanup()
+	}
+}
+
+// cleanup removes expired entries
+func (c *DiskBufferCache) cleanup() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	now := time.Now()
+	var toEvict []*cacheEntry
+
+	// Find expired entries
+	for _, entry := range c.cache {
+		if now.Sub(entry.timestamp) > c.ttl {
+			toEvict = append(toEvict, entry)
+		}
+	}
+
+	// Evict expired entries
+	for _, entry := range toEvict {
+		c.evict(entry)
+	}
+
+	if len(toEvict) > 0 {
+		glog.V(3).Infof("📦 CACHE CLEANUP: evicted %d expired entries", len(toEvict))
+	}
+}
+
+// Stats returns cache statistics
+func (c *DiskBufferCache) Stats() (hits, misses, evictions int64, size int) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.hits, c.misses, c.evictions, c.lruList.Len()
+}
+
+// Clear removes all entries from the cache
+func (c *DiskBufferCache) Clear() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.cache = make(map[string]*cacheEntry)
+	c.lruList = list.New()
+	glog.V(2).Infof("📦 CACHE CLEARED")
+}

weed/util/log_buffer/flush_budget_test.go

@@ -0,0 +1,273 @@
+package log_buffer
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util/mem"
+)
+
+func TestFlushBudgetBoundsQueuedBytes(t *testing.T) {
+	b := newFlushBudget(100)
+
+	if got := b.reserve(0, 60); got != 60 {
+		t.Fatalf("first reserve returned %d, want 60", got)
+	}
+
+	// 60 + 60 is over the limit, so the second producer has to wait.
+	reserved := make(chan int)
+	go func() { reserved <- b.reserve(1, 60) }()
+
+	select {
+	case <-reserved:
+		t.Fatal("second reserve went through while the budget was full")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	b.release(60)
+	select {
+	case got := <-reserved:
+		if got != 60 {
+			t.Errorf("second reserve returned %d, want 60", got)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("second reserve never woke up after the release")
+	}
+}
+
+// A window bigger than the whole budget still has to get through, or an
+// oversized entry would wedge the flush loop instead of merely spiking it.
+func TestFlushBudgetAdmitsOversizedWindow(t *testing.T) {
+	b := newFlushBudget(100)
+
+	if got := b.reserve(0, 50); got != 50 {
+		t.Fatalf("reserve returned %d, want 50", got)
+	}
+
+	reserved := make(chan int)
+	go func() { reserved <- b.reserve(1, 500) }()
+
+	select {
+	case <-reserved:
+		t.Fatal("oversized reserve went through before the queue drained")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	b.release(50)
+	select {
+	case got := <-reserved:
+		if got != 100 {
+			t.Errorf("oversized reserve booked %d, want the whole %d budget", got, 100)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("oversized reserve never got through on an empty queue")
+	}
+}
+
+// Windows have to reach the flush loop in the order they were sealed. A
+// producer can park here for seconds, so letting a later window overtake an
+// earlier one would persist them out of order and walk the flushed watermarks
+// backwards. Sized so only one window fits at a time, which is what makes the
+// admission order observable rather than a race between the woken goroutines.
+func TestFlushBudgetAdmitsInSealOrder(t *testing.T) {
+	const windows = 5
+	b := newFlushBudget(100)
+	b.reserve(0, 100) // fills the budget, so every window below has to wait
+
+	// Park them in reverse seal order, so arrival order cannot be what produces
+	// the right answer. Each window needs the whole budget, so exactly one is
+	// admitted per release and the order is observable.
+	admitted := make(chan uint64, windows)
+	for seq := uint64(windows); seq >= 1; seq-- {
+		go func(seq uint64) {
+			b.reserve(seq, 100)
+			admitted <- seq
+		}(seq)
+		time.Sleep(20 * time.Millisecond) // let it reach the wait
+	}
+
+	for want := uint64(1); want <= windows; want++ {
+		select {
+		case seq := <-admitted:
+			t.Fatalf("window %d was admitted while the budget was still held", seq)
+		case <-time.After(20 * time.Millisecond):
+		}
+
+		b.release(100)
+		select {
+		case got := <-admitted:
+			if got != want {
+				t.Fatalf("admitted window %d, want %d", got, want)
+			}
+		case <-time.After(2 * time.Second):
+			t.Fatalf("window %d never admitted", want)
+		}
+	}
+}
+
+// Shutdown must not be held up by a producer parked on the budget.
+func TestFlushBudgetCloseReleasesWaiters(t *testing.T) {
+	b := newFlushBudget(100)
+	b.reserve(0, 100)
+
+	done := make(chan struct{})
+	go func() {
+		b.reserve(1, 100)
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		t.Fatal("reserve went through while the budget was full")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	b.close()
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("close did not release the parked producer")
+	}
+}
+
+// What the queue actually holds is the pooled slab, which mem.Allocate rounds
+// up to a size class. Charging the window length instead would let the queue
+// retain roughly twice the ceiling.
+func TestQueueFlushChargesTheSlabNotTheWindow(t *testing.T) {
+	stall := make(chan struct{})
+	lb := NewLogBuffer("slab", time.Hour, func(_ *LogBuffer, _, _ time.Time, _ []byte, _, _ int64) {
+		<-stall
+	}, nil, func() {})
+	defer func() { close(stall); lb.ShutdownLogBuffer() }()
+
+	// 5 MiB rounds up to an 8 MiB slot.
+	const windowSize = 5 << 20
+	data := mem.Allocate(windowSize)
+	if cap(data) == len(data) {
+		t.Skipf("allocator returned an exact fit (%d bytes), nothing to distinguish", cap(data))
+	}
+
+	if !lb.queueFlush(&dataToFlush{data: data}) {
+		t.Fatal("queueFlush dropped the window")
+	}
+
+	lb.flushBudget.mu.Lock()
+	queued := lb.flushBudget.queued
+	lb.flushBudget.mu.Unlock()
+
+	if queued != cap(data) {
+		t.Errorf("charged %d bytes for a window holding a %d byte slab (len %d)", queued, cap(data), len(data))
+	}
+}
+
+// The window is already sealed by the time queueFlush runs, so dropping it
+// loses records the caller was told were accepted. A shutdown racing the
+// hand-off must not cost data while the queue still has room for it.
+func TestQueueFlushKeepsSealedWindowWhenQueueHasRoom(t *testing.T) {
+	stall := make(chan struct{})
+	var flushed atomic.Int64
+	lb := NewLogBuffer("shutdown-race", time.Hour, func(_ *LogBuffer, _, _ time.Time, _ []byte, _, _ int64) {
+		<-stall
+		flushed.Add(1)
+	}, nil, func() {})
+	defer func() { close(stall) }()
+
+	// Simulate a shutdown landing between the seal and the hand-off.
+	close(lb.shutdownCh)
+	lb.isStopping.Store(true)
+
+	// Stay inside the channel's capacity so room is guaranteed for every one of
+	// them; loopFlush is parked in the stalled flushFn and drains nothing.
+	for i := 0; i < flushQueueDepth; i++ {
+		if !lb.queueFlush(&dataToFlush{data: mem.Allocate(1024), seq: uint64(i)}) {
+			t.Fatalf("window %d was dropped even though the queue had room", i)
+		}
+	}
+}
+
+// The point of the budget: a stalled flush must stop producers rather than let
+// them keep handing over copies. Asserted on how many producers get through,
+// which is independent of the counter reserve maintains.
+func TestStalledFlushBlocksOversizedProducers(t *testing.T) {
+	stall := make(chan struct{})
+	var completed atomic.Int64
+
+	lb := NewLogBuffer("stalled", time.Hour, func(_ *LogBuffer, _, _ time.Time, _ []byte, _, _ int64) {
+		<-stall
+	}, nil, func() {})
+
+	// Just over BufferSize is enough to get a window per entry; the package
+	// leaves plenty of other LogBuffers alive, so keep the footprint small
+	// enough for a 32-bit runner.
+	entry := make([]byte, BufferSize+1)
+	const producers = flushQueueDepth
+
+	var wg sync.WaitGroup
+	for i := 0; i < producers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := lb.AddDataToBuffer(nil, entry, 0); err == nil {
+				completed.Add(1)
+			}
+		}()
+	}
+
+	time.Sleep(2 * time.Second)
+
+	got := completed.Load()
+	// Some have to get through -- a budget that admits nobody is a deadlock,
+	// not a bound -- and the rest have to be parked.
+	if got == 0 {
+		t.Error("no producer got through a stalled flush; the budget deadlocked")
+	}
+	if got >= producers {
+		t.Errorf("all %d producers got through a stalled flush; the budget did not bind", producers)
+	}
+
+	close(stall)
+	lb.ShutdownLogBuffer()
+	wg.Wait()
+}
+
+// ...and once the flush drains, everyone gets through: the bound must not
+// deadlock the producers it parks.
+func TestBlockedProducersDrainOnceFlushResumes(t *testing.T) {
+	stall := make(chan struct{})
+	var completed atomic.Int64
+
+	lb := NewLogBuffer("drain", time.Hour, func(_ *LogBuffer, _, _ time.Time, _ []byte, _, _ int64) {
+		<-stall
+	}, nil, func() {})
+
+	entry := make([]byte, BufferSize+1)
+	const producers = 6
+
+	var wg sync.WaitGroup
+	for i := 0; i < producers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := lb.AddDataToBuffer(nil, entry, 0); err == nil {
+				completed.Add(1)
+			}
+		}()
+	}
+
+	time.Sleep(500 * time.Millisecond)
+	close(stall)
+
+	done := make(chan struct{})
+	go func() { wg.Wait(); close(done) }()
+	select {
+	case <-done:
+	case <-time.After(30 * time.Second):
+		t.Fatal("producers never drained after the flush resumed")
+	}
+	if got := completed.Load(); got != producers {
+		t.Errorf("%d of %d producers completed", got, producers)
+	}
+	lb.ShutdownLogBuffer()
+}

weed/util/log_buffer/log_buffer_corruption_test.go

@@ -0,0 +1,224 @@
+package log_buffer
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// TestReadTsCorruptedBuffer tests that readTs properly returns an error for corrupted data
+func TestReadTsCorruptedBuffer(t *testing.T) {
+	// Create a corrupted buffer with invalid protobuf data
+	buf := make([]byte, 100)
+
+	// Set size field to 10 bytes (using proper encoding)
+	util.Uint32toBytes(buf[0:4], 10)
+
+	// Fill with garbage data that won't unmarshal as LogEntry
+	for i := 4; i < 14; i++ {
+		buf[i] = 0xFF
+	}
+
+	// Attempt to read timestamp
+	size, ts, err := readTs(buf, 0)
+
+	// Should return an error
+	if err == nil {
+		t.Error("Expected error for corrupted buffer, got nil")
+	}
+
+	// Size and ts should be zero on error
+	if size != 0 {
+		t.Errorf("Expected size=0 on error, got %d", size)
+	}
+
+	if ts != 0 {
+		t.Errorf("Expected ts=0 on error, got %d", ts)
+	}
+
+	// Error should indicate corruption
+	if !errors.Is(err, ErrBufferCorrupted) {
+		t.Logf("Error message: %v", err)
+		// Check if error message contains expected text
+		if err.Error() == "" || len(err.Error()) == 0 {
+			t.Error("Expected non-empty error message")
+		}
+	}
+
+	t.Logf("✓ readTs correctly returned error for corrupted buffer: %v", err)
+}
+
+// TestReadTsValidBuffer tests that readTs works correctly for valid data
+func TestReadTsValidBuffer(t *testing.T) {
+	// Create a valid LogEntry
+	logEntry := &filer_pb.LogEntry{
+		TsNs: 123456789,
+		Key:  []byte("test-key"),
+	}
+
+	// Marshal it
+	data, err := proto.Marshal(logEntry)
+	if err != nil {
+		t.Fatalf("Failed to marshal LogEntry: %v", err)
+	}
+
+	// Create buffer with size prefix using util function
+	buf := make([]byte, 4+len(data))
+	util.Uint32toBytes(buf[0:4], uint32(len(data)))
+	copy(buf[4:], data)
+
+	// Read timestamp
+	size, ts, err := readTs(buf, 0)
+
+	// Should succeed
+	if err != nil {
+		t.Fatalf("Expected no error for valid buffer, got: %v", err)
+	}
+
+	// Should return correct values
+	if size != len(data) {
+		t.Errorf("Expected size=%d, got %d", len(data), size)
+	}
+
+	if ts != logEntry.TsNs {
+		t.Errorf("Expected ts=%d, got %d", logEntry.TsNs, ts)
+	}
+
+	t.Logf("✓ readTs correctly parsed valid buffer: size=%d, ts=%d", size, ts)
+}
+
+// TestReadFromBufferCorruption tests that ReadFromBuffer propagates corruption errors
+func TestReadFromBufferCorruption(t *testing.T) {
+	lb := NewLogBuffer("test-corruption", time.Second, nil, nil, func() {})
+
+	// Add a valid entry first using AddDataToBuffer
+	validKey := []byte("valid")
+	validData, _ := proto.Marshal(&filer_pb.LogEntry{
+		TsNs: 1000,
+		Key:  validKey,
+	})
+	if err := lb.AddDataToBuffer(validKey, validData, 1000); err != nil {
+		t.Fatalf("Failed to add data to buffer: %v", err)
+	}
+
+	// Manually corrupt the buffer by writing garbage
+	// This simulates a corruption scenario
+	if len(lb.idx) > 0 {
+		pos := lb.idx[0]
+		// Overwrite the protobuf data with garbage
+		for i := pos + 4; i < pos+8 && i < len(lb.buf); i++ {
+			lb.buf[i] = 0xFF
+		}
+	}
+
+	// Try to read - should detect corruption
+	startPos := MessagePosition{Time: lb.startTime}
+	buf, offset, _, err := lb.ReadFromBuffer(startPos)
+
+	// Should return corruption error
+	if err == nil {
+		t.Error("Expected corruption error, got nil")
+		if buf != nil {
+			t.Logf("Unexpected success: got buffer with %d bytes", buf.Len())
+		}
+	} else {
+		// Verify it's a corruption error
+		if !errors.Is(err, ErrBufferCorrupted) {
+			t.Logf("Got error (not ErrBufferCorrupted sentinel, but still an error): %v", err)
+		}
+		t.Logf("✓ ReadFromBuffer correctly detected corruption: %v", err)
+	}
+
+	t.Logf("ReadFromBuffer result: buf=%v, offset=%d, err=%v", buf != nil, offset, err)
+}
+
+// TestLocateByTsCorruption tests that locateByTs propagates corruption errors
+func TestLocateByTsCorruption(t *testing.T) {
+	// Create a MemBuffer with corrupted data
+	mb := &MemBuffer{
+		buf:  make([]byte, 100),
+		size: 14,
+	}
+
+	// Set size field (using proper encoding)
+	util.Uint32toBytes(mb.buf[0:4], 10)
+
+	// Fill with garbage
+	for i := 4; i < 14; i++ {
+		mb.buf[i] = 0xFF
+	}
+
+	// Try to locate by timestamp
+	pos, err := mb.locateByTs(mb.startTime)
+
+	// Should return error
+	if err == nil {
+		t.Errorf("Expected corruption error, got nil (pos=%d)", pos)
+	} else {
+		t.Logf("✓ locateByTs correctly detected corruption: %v", err)
+	}
+}
+
+// TestErrorPropagationChain tests the complete error propagation from readTs -> locateByTs -> ReadFromBuffer
+func TestErrorPropagationChain(t *testing.T) {
+	t.Run("Corruption in readTs", func(t *testing.T) {
+		// Already covered by TestReadTsCorruptedBuffer
+		t.Log("✓ readTs error propagation tested")
+	})
+
+	t.Run("Corruption in locateByTs", func(t *testing.T) {
+		// Already covered by TestLocateByTsCorruption
+		t.Log("✓ locateByTs error propagation tested")
+	})
+
+	t.Run("Corruption in ReadFromBuffer binary search", func(t *testing.T) {
+		// Already covered by TestReadFromBufferCorruption
+		t.Log("✓ ReadFromBuffer error propagation tested")
+	})
+
+	t.Log("✓ Complete error propagation chain verified")
+}
+
+// TestNoSilentCorruption verifies that corruption never returns (0, 0) silently
+func TestNoSilentCorruption(t *testing.T) {
+	// Create various corrupted buffers
+	testCases := []struct {
+		name string
+		buf  []byte
+		pos  int
+	}{
+		{
+			name: "Invalid protobuf",
+			buf:  []byte{10, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
+			pos:  0,
+		},
+		{
+			name: "Truncated data",
+			buf:  []byte{100, 0, 0, 0, 1, 2, 3}, // Size says 100 but only 3 bytes available
+			pos:  0,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			size, ts, err := readTs(tc.buf, tc.pos)
+
+			// CRITICAL: Must return error, never silent (0, 0)
+			if err == nil {
+				t.Errorf("CRITICAL: readTs returned (%d, %d, nil) for corrupted buffer - this causes silent data corruption!", size, ts)
+			} else {
+				t.Logf("✓ Correctly returned error instead of silent (0, 0): %v", err)
+			}
+
+			// On error, size and ts should be 0
+			if size != 0 || ts != 0 {
+				t.Errorf("On error, expected (0, 0), got (%d, %d)", size, ts)
+			}
+		})
+	}
+}

weed/util/log_buffer/log_buffer_eviction_gate_test.go

@@ -0,0 +1,390 @@
+package log_buffer
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+// TestEvictionWatermarkTracksSealedRing drives the watermark through the real
+// eviction path instead of writing the field: only copyToFlushInternal sees the
+// window about to be dropped, and it reads it one statement before SealBuffer
+// shifts it out of slot 0. The buffer here never flushes, matching the
+// aggregated meta ring, so the watermark is its only emptiness proof.
+func TestEvictionWatermarkTracksSealedRing(t *testing.T) {
+	lb := NewLogBuffer("evict-watermark", time.Minute, nil, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	// Each timestamp is further past the previous window's start than the flush
+	// interval, so every append seals the window before it.
+	base := time.Now().Add(-30 * time.Minute).Truncate(time.Second)
+	step := 2 * time.Minute
+	at := func(i int) time.Time { return base.Add(time.Duration(i) * step) }
+
+	if got := lb.GetLastEvictedTsNs(); got != 0 {
+		t.Fatalf("fresh buffer evicted through %v, want 0", time.Unix(0, got))
+	}
+
+	// PreviousBufferCount seals only fill the ring; the next one drops window 0.
+	for i := 0; i <= PreviousBufferCount; i++ {
+		if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: at(i).UnixNano(), Data: []byte("x"), Key: []byte("k")}); err != nil {
+			t.Fatalf("add %d: %v", i, err)
+		}
+		if got := lb.GetLastEvictedTsNs(); got != 0 {
+			t.Fatalf("after %d appends evicted through %v, want nothing evicted yet", i+1, time.Unix(0, got))
+		}
+	}
+
+	if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: at(PreviousBufferCount + 1).UnixNano(), Data: []byte("x"), Key: []byte("k")}); err != nil {
+		t.Fatalf("add evicting entry: %v", err)
+	}
+	// Window 0 held only the first entry, so its stopTime is that entry's ts.
+	if got, want := lb.GetLastEvictedTsNs(), at(0).UnixNano(); got != want {
+		t.Fatalf("evicted through %v, want the dropped window's stop %v", time.Unix(0, got), time.Unix(0, want))
+	}
+
+	// One more eviction advances the watermark; it never regresses.
+	if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: at(PreviousBufferCount + 2).UnixNano(), Data: []byte("x"), Key: []byte("k")}); err != nil {
+		t.Fatalf("add second evicting entry: %v", err)
+	}
+	if got, want := lb.GetLastEvictedTsNs(), at(1).UnixNano(); got != want {
+		t.Fatalf("evicted through %v, want %v", time.Unix(0, got), time.Unix(0, want))
+	}
+}
+
+// TestGapResumeCursorReadsSingleEntrySealedWindow pins the resume cursor
+// against the shape that actually breaks: a sealed window holding one entry,
+// where startTime == stopTime. The sealed-buffer lookup only enters a window
+// whose stopTime is strictly after the cursor, so a cursor sitting exactly on
+// earliest walks straight past such a window and its sole event is never
+// delivered. Low-volume metadata windows are routinely one entry.
+func TestGapResumeCursorReadsSingleEntrySealedWindow(t *testing.T) {
+	lb := NewLogBuffer("gap-resume-sealed", time.Minute, nil, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	// Timestamps far enough apart that every append seals the window before it,
+	// so each sealed window holds exactly one entry.
+	base := time.Now().Add(-time.Hour).Truncate(time.Second)
+	for i := 0; i < 3; i++ {
+		if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{
+			TsNs: base.Add(time.Duration(i) * 2 * time.Minute).UnixNano(), Data: []byte("x"), Key: []byte("k"),
+		}); err != nil {
+			t.Fatalf("add %d: %v", i, err)
+		}
+	}
+	earliest := lb.GetEarliestTime()
+	if earliest.IsZero() {
+		t.Fatal("expected in-memory data")
+	}
+
+	// firstTsFrom reports the timestamp of the first entry a read hands back.
+	firstTsFrom := func(cursorTsNs int64) int64 {
+		buf, _, pooled, err := lb.ReadFromBuffer(NewMessagePosition(cursorTsNs, gapResumeTestOffset))
+		if err != nil {
+			t.Fatalf("read at %v: %v", time.Unix(0, cursorTsNs), err)
+		}
+		if buf == nil {
+			t.Fatalf("read at %v returned no data", time.Unix(0, cursorTsNs))
+		}
+		if pooled {
+			defer lb.ReleaseMemory(buf)
+		}
+		_, ts, readErr := readTs(buf.Bytes(), 0)
+		if readErr != nil {
+			t.Fatalf("decode first entry: %v", readErr)
+		}
+		return ts
+	}
+
+	// The resume the resolvers issue must deliver the earliest entry itself.
+	if got := firstTsFrom(earliest.UnixNano() - 1); got != earliest.UnixNano() {
+		t.Fatalf("resume just below earliest starts at %v, want the earliest entry %v",
+			time.Unix(0, got), earliest)
+	}
+	// Resuming exactly on earliest is the shape that loses it.
+	if got := firstTsFrom(earliest.UnixNano()); got == earliest.UnixNano() {
+		t.Fatal("expected a cursor exactly on earliest to skip the single-entry window (precondition)")
+	}
+}
+
+// gapResumeTestOffset mirrors the sentinel the filer's gap resume carries.
+const gapResumeTestOffset = -2
+
+// TestGapResumeCursorReadsFromMemory pins the contract the filer's gap resolver
+// depends on: the cursor it resumes with must be one this read actually serves.
+// ReadFromBuffer only falls through to memory for a cursor below the in-memory
+// window when the offset is a sentinel, and answers ResumeFromDiskError for
+// every positive one -- a resume carrying a positive offset would bounce
+// straight back to the resolver, which sees no progress and parks a subscriber
+// whose data is sitting in the ring.
+func TestGapResumeCursorReadsFromMemory(t *testing.T) {
+	lb := NewLogBuffer("gap-resume", time.Minute, nil, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	base := time.Now().Add(-time.Hour).Truncate(time.Second)
+	for i := 0; i < 3; i++ {
+		if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{
+			TsNs: base.Add(time.Duration(i) * time.Second).UnixNano(), Data: []byte("x"), Key: []byte("k"),
+		}); err != nil {
+			t.Fatalf("add %d: %v", i, err)
+		}
+	}
+	earliest := lb.GetEarliestTime()
+	if earliest.IsZero() {
+		t.Fatal("expected in-memory data")
+	}
+
+	// The resolver resumes at earliest itself, read inclusively.
+	if buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(earliest.UnixNano(), -2)); err != nil || buf == nil {
+		t.Fatalf("resume at earliest: buf=%v err=%v", buf != nil, err)
+	}
+	// A sentinel just below it is served too, so the exact boundary is not load-bearing.
+	if buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(earliest.UnixNano()-1, -2)); err != nil || buf == nil {
+		t.Fatalf("resume just below earliest: buf=%v err=%v", buf != nil, err)
+	}
+	// A positive offset below the window is refused: this is the shape a gap
+	// resume must never take.
+	if _, _, _, err := lb.ReadFromBuffer(NewMessagePosition(earliest.UnixNano()-1, 1)); err != ResumeFromDiskError {
+		t.Fatalf("positive-offset cursor below the window: want ResumeFromDiskError, got %v", err)
+	}
+}
+
+// TestFlushSubscriberContract pins what the filer's gap parks rest on: a flush
+// subscriber is woken when a flush lands and the flush watermark is already
+// visible at that moment - loopFlush stores lastFlushTsNs before notifying, so
+// a waiter that re-checks the watermark on wake-up cannot miss the flush that
+// woke it. Appends alone never signal this channel, and unregistering closes
+// it so an abandoned waiter is not stranded.
+func TestFlushSubscriberContract(t *testing.T) {
+	flushed := make(chan struct{}, 16)
+	// A flush interval far longer than the test, so appends never auto-seal a
+	// window and ForceFlush is the only source of flushes: each round then has
+	// exactly one flush, and the token received is known to belong to it.
+	lb := NewLogBuffer("flush-sub", time.Hour,
+		func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+			flushed <- struct{}{}
+		}, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	ch := lb.RegisterFlushSubscriber("s")
+
+	// An append is not a flush: nothing may arrive on the channel. The probe
+	// sits just below the first round timestamp: below, so no round entry gets
+	// collision-bumped and its stored stopTime stays comparable to the local
+	// value each round asserts against; just below, because a gap wider than
+	// the flush interval would make round 0's own append auto-seal the probe
+	// window - a second flush whose token throws every round off by one.
+	if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: time.Now().Add(-time.Hour - time.Millisecond).UnixNano(), Data: []byte("x"), Key: []byte("k")}); err != nil {
+		t.Fatalf("add: %v", err)
+	}
+	select {
+	case <-ch:
+		t.Fatal("an append must not wake a flush subscriber")
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	// A flush wakes it with the watermark already stored - the contract the gap
+	// parks re-check on. The store-before-notify ordering itself is pinned by a
+	// load-bearing comment at the loopFlush site (a reorder's window is
+	// nanoseconds on that goroutine, untestable from here); these round-trips
+	// verify the observable contract and catch a notify with no store at all.
+	base := time.Now().Add(-time.Hour)
+	for i := 0; i < 8; i++ {
+		ts := base.Add(time.Duration(i) * time.Millisecond).UnixNano()
+		// The round entry must be above the buffer head: a bumped timestamp
+		// flushes a stopTime far above the local ts compared below, making the
+		// round's assertion vacuously true.
+		if last := lb.LastTsNs.Load(); last >= ts {
+			t.Fatalf("round %d: ts not above buffer head; the assertion would be vacuous", i)
+		}
+		if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: ts, Data: []byte("x"), Key: []byte("k")}); err != nil {
+			t.Fatalf("add round %d: %v", i, err)
+		}
+		go lb.ForceFlush()
+		select {
+		case <-ch:
+			if got := lb.GetLastFlushTsNs(); got < ts {
+				t.Fatalf("round %d: woken with watermark %v short of the flushed entry %v; a waiter re-checking it on wake-up misses the flush that woke it",
+					i, time.Unix(0, got), time.Unix(0, ts))
+			}
+		case <-time.After(5 * time.Second):
+			t.Fatalf("round %d: a flush must wake the flush subscriber", i)
+		}
+		select {
+		case <-flushed:
+		case <-time.After(5 * time.Second):
+			t.Fatalf("round %d: flushFn did not run", i)
+		}
+	}
+
+	// Unregistering closes the channel so an abandoned waiter unblocks.
+	lb.UnregisterFlushSubscriber("s")
+	select {
+	case _, ok := <-ch:
+		if ok {
+			t.Fatal("unregister must close the channel, not send on it")
+		}
+	case <-time.After(time.Second):
+		t.Fatal("unregister must close the channel")
+	}
+
+	// Unknown ids and double unregisters are harmless.
+	lb.UnregisterFlushSubscriber("s")
+	lb.UnregisterFlushSubscriber("never-registered")
+}
+
+// TestEvictionGatedCursor pins the gated sentinel: below the eviction watermark
+// it is refused to disk under the read's own lock - the only place the check is
+// atomic with the serve decision - while the plain -2 sentinel keeps master's
+// serve-from-earliest behavior for the message queue's readers.
+func TestEvictionGatedCursor(t *testing.T) {
+	lb := NewLogBuffer("gated-cursor", time.Minute, nil, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	// Entries far enough apart that every append seals the window before it;
+	// wrapping the ring moves the watermark for real.
+	base := time.Now().Add(-time.Hour).Truncate(time.Second)
+	for i := 0; i < PreviousBufferCount+2; i++ {
+		if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{
+			TsNs: base.Add(time.Duration(i) * 2 * time.Minute).UnixNano(), Data: []byte("x"), Key: []byte("k"),
+		}); err != nil {
+			t.Fatalf("add %d: %v", i, err)
+		}
+	}
+	evicted := lb.GetLastEvictedTsNs()
+	if evicted == 0 {
+		t.Fatal("precondition: the ring evicted nothing")
+	}
+
+	// Below the watermark: gated goes to disk, -2 keeps serving (MQ contract).
+	if _, _, _, err := lb.ReadFromBuffer(NewMessagePosition(evicted-1, EvictionGatedOffset)); err != ResumeFromDiskError {
+		t.Fatalf("gated below watermark: want ResumeFromDiskError, got %v", err)
+	}
+	if buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(evicted-1, -2)); err != nil || buf == nil {
+		t.Fatalf("plain sentinel below watermark: buf=%v err=%v, master behavior must hold", buf != nil, err)
+	}
+	// At and above the watermark the gated cursor serves normally.
+	if buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(evicted, EvictionGatedOffset)); err != nil || buf == nil {
+		t.Fatalf("gated at watermark: buf=%v err=%v", buf != nil, err)
+	}
+	if buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(evicted+1, EvictionGatedOffset)); err != nil || buf == nil {
+		t.Fatalf("gated above watermark: buf=%v err=%v", buf != nil, err)
+	}
+}
+
+// TestMarkEvictedThroughGatesYoungRing pins the young-ring gate: with the mark
+// set, a below-window gated cursor goes to disk before the first real eviction
+// instead of being served the earliest retained entry.
+func TestMarkEvictedThroughGatesYoungRing(t *testing.T) {
+	lb := NewLogBuffer("young-ring", time.Minute, nil, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	seed := time.Now().Add(-time.Minute).UnixNano()
+	lb.MarkEvictedThrough(seed)
+	if got := lb.GetLastEvictedTsNs(); got != seed {
+		t.Fatalf("evicted through %v, want the mark %v", time.Unix(0, got), time.Unix(0, seed))
+	}
+	if got := lb.GetLastEvictedOriginalTsNs(); got != seed {
+		t.Fatalf("original watermark %v, want the mark %v", time.Unix(0, got), time.Unix(0, seed))
+	}
+	// A lower mark must not regress an established boundary.
+	lb.MarkEvictedThrough(seed - int64(time.Hour))
+	if got := lb.GetLastEvictedTsNs(); got != seed {
+		t.Fatalf("lower mark regressed the watermark to %v", time.Unix(0, got))
+	}
+
+	// One post-mark arrival: the ring's window starts well above the mark.
+	if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{
+		TsNs: time.Now().UnixNano(), Data: []byte("x"), Key: []byte("k"),
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Below the mark: gated goes to disk, -2 keeps serving (MQ contract).
+	if _, _, _, err := lb.ReadFromBuffer(NewMessagePosition(seed-1, EvictionGatedOffset)); err != ResumeFromDiskError {
+		t.Fatalf("gated below the mark: want ResumeFromDiskError, got %v", err)
+	}
+	if buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(seed-1, -2)); err != nil || buf == nil {
+		t.Fatalf("plain sentinel below the mark: buf=%v err=%v", buf != nil, err)
+	}
+	if buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(seed, EvictionGatedOffset)); err != nil || buf == nil {
+		t.Fatalf("gated at the mark: buf=%v err=%v", buf != nil, err)
+	}
+}
+
+// TestEvictionOriginalWatermark pins the second timestamp space. The ring bumps
+// an out-of-order arrival past its head, so a bump-heavy interval (peer history
+// replay) leaves stopTimes above anything on any peer's disk; a gap gate
+// comparing disk cursors against those would park a subscriber that drained
+// every peer's log. The original watermark tracks what was actually received.
+func TestEvictionOriginalWatermark(t *testing.T) {
+	lb := NewLogBuffer("orig-watermark", time.Minute, nil, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	// Newest original first: it sets the ring head, so every later arrival is
+	// bumped. Seals go by bumped time, which advances by nanoseconds, so each
+	// window is sealed explicitly; wrapping the ring evicts.
+	newest := time.Now().Add(-time.Hour).Truncate(time.Second).UnixNano()
+	for i := 0; i < PreviousBufferCount+2; i++ {
+		orig := newest - int64(i)*int64(time.Minute)
+		if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: orig, Data: []byte("x"), Key: []byte("k")}); err != nil {
+			t.Fatalf("add %d: %v", i, err)
+		}
+		lb.ForceFlush()
+	}
+
+	bumped := lb.GetLastEvictedTsNs()
+	original := lb.GetLastEvictedOriginalTsNs()
+	if bumped == 0 || original == 0 {
+		t.Fatalf("precondition: nothing evicted (bumped=%d original=%d)", bumped, original)
+	}
+	if original > newest {
+		t.Fatalf("original watermark %v exceeds the highest received timestamp %v", time.Unix(0, original), time.Unix(0, newest))
+	}
+	if bumped <= newest {
+		t.Fatalf("bumped watermark %v not past the ring head %v: the spaces did not diverge", time.Unix(0, bumped), time.Unix(0, newest))
+	}
+	// The punchline: a disk cursor that drained every peer's log sits at the
+	// highest received timestamp - clearing the original watermark while still
+	// below the bumped one, which no disk timestamp can ever reach.
+	diskHead := newest
+	if diskHead < original {
+		t.Fatalf("disk head %v short of the original watermark %v", time.Unix(0, diskHead), time.Unix(0, original))
+	}
+	if diskHead >= bumped {
+		t.Fatal("disk head reached the bumped watermark; the gate would not have parked and this test proves nothing")
+	}
+}
+
+// TestOriginalWatermarkWindowAttribution pins which window an entry's received
+// timestamp is credited to. An append that rolls the window over seals the
+// previous one first; crediting the incoming timestamp before that hands it to
+// the sealed window (inflating its watermark: spurious parks) and loses it
+// from the new one (deflating: gaps proven empty that are not).
+func TestOriginalWatermarkWindowAttribution(t *testing.T) {
+	lb := NewLogBuffer("orig-attribution", time.Minute, nil, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	base := time.Now().Add(-time.Hour).Truncate(time.Second)
+	t1 := base.UnixNano()
+	t2 := base.Add(2 * time.Minute).UnixNano() // past the flush interval: seals window A
+
+	if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: t1, Data: []byte("x"), Key: []byte("k")}); err != nil {
+		t.Fatalf("add t1: %v", err)
+	}
+	if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: t2, Data: []byte("x"), Key: []byte("k")}); err != nil {
+		t.Fatalf("add t2: %v", err)
+	}
+
+	sealed := lb.prevBuffers.buffers[len(lb.prevBuffers.buffers)-1]
+	if sealed.size == 0 {
+		t.Fatal("precondition: the second append did not seal the first window")
+	}
+	if got := sealed.maxOriginalTsNs; got != t1 {
+		t.Fatalf("sealed window credited with %v, want its own entry %v (t2 belongs to the open window)", time.Unix(0, got), time.Unix(0, t1))
+	}
+	if got := lb.curWindowMaxOriginalTsNs; got != t2 {
+		t.Fatalf("open window holds %v, want the entry that rolled it over %v", time.Unix(0, got), time.Unix(0, t2))
+	}
+}

weed/util/log_buffer/log_buffer_flush_gap_test.go

@@ -0,0 +1,697 @@
+package log_buffer
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"google.golang.org/protobuf/proto"
+)
+
+// TestFlushOffsetGap_ReproduceDataLoss reproduces the critical bug where messages
+// are lost in the gap between flushed disk data and in-memory buffer.
+//
+// OBSERVED BEHAVIOR FROM LOGS:
+//
+//	Request offset: 1764
+//	Disk contains: 1000-1763 (764 messages)
+//	Memory buffer starts at: 1800
+//	Gap: 1764-1799 (36 messages) ← MISSING!
+//
+// This test verifies:
+// 1. All messages sent to buffer are accounted for
+// 2. No gaps exist between disk and memory offsets
+// 3. Flushed data and in-memory data have continuous offset ranges
+func TestFlushOffsetGap_ReproduceDataLoss(t *testing.T) {
+	var flushedMessages []*filer_pb.LogEntry
+	var flushMu sync.Mutex
+
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+		t.Logf("FLUSH: minOffset=%d maxOffset=%d size=%d bytes", minOffset, maxOffset, len(buf))
+
+		// Parse and store flushed messages
+		flushMu.Lock()
+		defer flushMu.Unlock()
+
+		// Parse buffer to extract messages
+		parsedCount := 0
+		for pos := 0; pos+4 < len(buf); {
+			if pos+4 > len(buf) {
+				break
+			}
+
+			size := uint32(buf[pos])<<24 | uint32(buf[pos+1])<<16 | uint32(buf[pos+2])<<8 | uint32(buf[pos+3])
+			if pos+4+int(size) > len(buf) {
+				break
+			}
+
+			entryData := buf[pos+4 : pos+4+int(size)]
+			logEntry := &filer_pb.LogEntry{}
+			if err := proto.Unmarshal(entryData, logEntry); err == nil {
+				flushedMessages = append(flushedMessages, logEntry)
+				parsedCount++
+			}
+
+			pos += 4 + int(size)
+		}
+
+		t.Logf("  Parsed %d messages from flush buffer", parsedCount)
+	}
+
+	logBuffer := NewLogBuffer("test", 100*time.Millisecond, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Send 100 messages
+	messageCount := 100
+	t.Logf("Sending %d messages...", messageCount)
+
+	for i := 0; i < messageCount; i++ {
+		if err := logBuffer.AddToBuffer(&mq_pb.DataMessage{
+			Key:   []byte(fmt.Sprintf("key-%d", i)),
+			Value: []byte(fmt.Sprintf("message-%d", i)),
+			TsNs:  time.Now().UnixNano(),
+		}); err != nil {
+			t.Fatalf("Failed to add buffer: %v", err)
+		}
+	}
+
+	// Force flush multiple times to simulate real workload
+	t.Logf("Forcing flush...")
+	logBuffer.ForceFlush()
+
+	// Add more messages after flush
+	for i := messageCount; i < messageCount+50; i++ {
+		if err := logBuffer.AddToBuffer(&mq_pb.DataMessage{
+			Key:   []byte(fmt.Sprintf("key-%d", i)),
+			Value: []byte(fmt.Sprintf("message-%d", i)),
+			TsNs:  time.Now().UnixNano(),
+		}); err != nil {
+			t.Fatalf("Failed to add buffer: %v", err)
+		}
+	}
+
+	// Force another flush
+	logBuffer.ForceFlush()
+	time.Sleep(200 * time.Millisecond) // Wait for flush to complete
+
+	// Now check the buffer state
+	logBuffer.RLock()
+	bufferStartOffset := logBuffer.bufferStartOffset
+	currentOffset := logBuffer.offset
+	pos := logBuffer.pos
+	logBuffer.RUnlock()
+
+	flushMu.Lock()
+	flushedCount := len(flushedMessages)
+	var maxFlushedOffset int64 = -1
+	var minFlushedOffset int64 = -1
+	if flushedCount > 0 {
+		minFlushedOffset = flushedMessages[0].Offset
+		maxFlushedOffset = flushedMessages[flushedCount-1].Offset
+	}
+	flushMu.Unlock()
+
+	t.Logf("\nBUFFER STATE AFTER FLUSH:")
+	t.Logf("  bufferStartOffset: %d", bufferStartOffset)
+	t.Logf("  currentOffset (HWM): %d", currentOffset)
+	t.Logf("  pos (bytes in buffer): %d", pos)
+	t.Logf("  Messages sent: %d (offsets 0-%d)", messageCount+50, messageCount+49)
+	t.Logf("  Messages flushed to disk: %d (offsets %d-%d)", flushedCount, minFlushedOffset, maxFlushedOffset)
+
+	// CRITICAL CHECK: Is there a gap between flushed data and memory buffer?
+	if flushedCount > 0 && maxFlushedOffset >= 0 {
+		gap := bufferStartOffset - (maxFlushedOffset + 1)
+
+		t.Logf("\nOFFSET CONTINUITY CHECK:")
+		t.Logf("  Last flushed offset: %d", maxFlushedOffset)
+		t.Logf("  Buffer starts at: %d", bufferStartOffset)
+		t.Logf("  Gap: %d offsets", gap)
+
+		if gap > 0 {
+			t.Errorf("CRITICAL BUG REPRODUCED: OFFSET GAP DETECTED!")
+			t.Errorf("   Disk has offsets %d-%d", minFlushedOffset, maxFlushedOffset)
+			t.Errorf("   Memory buffer starts at: %d", bufferStartOffset)
+			t.Errorf("   MISSING OFFSETS: %d-%d (%d messages)", maxFlushedOffset+1, bufferStartOffset-1, gap)
+			t.Errorf("   These messages are LOST - neither on disk nor in memory!")
+		} else if gap < 0 {
+			t.Errorf("OFFSET OVERLAP: Memory buffer starts BEFORE last flushed offset!")
+			t.Errorf("   This indicates data corruption or race condition")
+		} else {
+			t.Logf("PASS: No gap detected - offsets are continuous")
+		}
+
+		// Check if we can read all expected offsets
+		t.Logf("\nREADABILITY CHECK:")
+		for testOffset := int64(0); testOffset < currentOffset; testOffset += 10 {
+			// Try to read from buffer
+			requestPosition := NewMessagePositionFromOffset(testOffset)
+			buf, _, _, err := logBuffer.ReadFromBuffer(requestPosition)
+
+			isReadable := (buf != nil && len(buf.Bytes()) > 0) || err == ResumeFromDiskError
+			status := "OK"
+			if !isReadable && err == nil {
+				status = "NOT READABLE"
+			}
+
+			t.Logf("  Offset %d: %s (buf=%v, err=%v)", testOffset, status, buf != nil, err)
+
+			// If offset is in the gap, it should fail to read
+			if flushedCount > 0 && testOffset > maxFlushedOffset && testOffset < bufferStartOffset {
+				if isReadable {
+					t.Errorf("   Unexpected: Offset %d in gap range should NOT be readable!", testOffset)
+				} else {
+					t.Logf("   Expected: Offset %d in gap is not readable (data lost)", testOffset)
+				}
+			}
+		}
+	}
+
+	// Check that all sent messages are accounted for
+	expectedMessageCount := messageCount + 50
+	messagesInMemory := int(currentOffset - bufferStartOffset)
+	totalAccountedFor := flushedCount + messagesInMemory
+
+	t.Logf("\nMESSAGE ACCOUNTING:")
+	t.Logf("  Expected: %d messages", expectedMessageCount)
+	t.Logf("  Flushed to disk: %d", flushedCount)
+	t.Logf("  In memory buffer: %d (offset range %d-%d)", messagesInMemory, bufferStartOffset, currentOffset-1)
+	t.Logf("  Total accounted for: %d", totalAccountedFor)
+	t.Logf("  Missing: %d messages", expectedMessageCount-totalAccountedFor)
+
+	if totalAccountedFor < expectedMessageCount {
+		t.Errorf("DATA LOSS CONFIRMED: %d messages are missing!", expectedMessageCount-totalAccountedFor)
+	} else {
+		t.Logf("All messages accounted for")
+	}
+}
+
+// TestFlushOffsetGap_CheckPrevBuffers tests if messages might be stuck in prevBuffers
+// instead of being properly flushed to disk.
+func TestFlushOffsetGap_CheckPrevBuffers(t *testing.T) {
+	var flushCount int
+	var flushMu sync.Mutex
+
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+		flushMu.Lock()
+		flushCount++
+		count := flushCount
+		flushMu.Unlock()
+
+		t.Logf("FLUSH #%d: minOffset=%d maxOffset=%d size=%d bytes", count, minOffset, maxOffset, len(buf))
+	}
+
+	logBuffer := NewLogBuffer("test", 100*time.Millisecond, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Send messages in batches with flushes in between
+	for batch := 0; batch < 5; batch++ {
+		t.Logf("\nBatch %d:", batch)
+
+		// Send 20 messages
+		for i := 0; i < 20; i++ {
+			offset := int64(batch*20 + i)
+			if err := logBuffer.AddToBuffer(&mq_pb.DataMessage{
+				Key:   []byte(fmt.Sprintf("key-%d", offset)),
+				Value: []byte(fmt.Sprintf("message-%d", offset)),
+				TsNs:  time.Now().UnixNano(),
+			}); err != nil {
+				t.Fatalf("Failed to add buffer: %v", err)
+			}
+		}
+
+		// Check state before flush
+		logBuffer.RLock()
+		beforeFlushOffset := logBuffer.offset
+		beforeFlushStart := logBuffer.bufferStartOffset
+		logBuffer.RUnlock()
+
+		// Force flush
+		logBuffer.ForceFlush()
+		time.Sleep(50 * time.Millisecond)
+
+		// Check state after flush
+		logBuffer.RLock()
+		afterFlushOffset := logBuffer.offset
+		afterFlushStart := logBuffer.bufferStartOffset
+		prevBufferCount := len(logBuffer.prevBuffers.buffers)
+
+		// Check prevBuffers state
+		t.Logf("  Before flush: offset=%d, bufferStartOffset=%d", beforeFlushOffset, beforeFlushStart)
+		t.Logf("  After flush: offset=%d, bufferStartOffset=%d, prevBuffers=%d",
+			afterFlushOffset, afterFlushStart, prevBufferCount)
+
+		// Check each prevBuffer
+		for i, prevBuf := range logBuffer.prevBuffers.buffers {
+			if prevBuf.size > 0 {
+				t.Logf("    prevBuffer[%d]: offsets %d-%d, size=%d bytes (NOT FLUSHED!)",
+					i, prevBuf.startOffset, prevBuf.offset, prevBuf.size)
+			}
+		}
+		logBuffer.RUnlock()
+
+		// CRITICAL: Check if bufferStartOffset advanced correctly
+		expectedNewStart := beforeFlushOffset
+		if afterFlushStart != expectedNewStart {
+			t.Errorf("  bufferStartOffset mismatch!")
+			t.Errorf("     Expected: %d (= offset before flush)", expectedNewStart)
+			t.Errorf("     Actual: %d", afterFlushStart)
+			t.Errorf("     Gap: %d offsets", expectedNewStart-afterFlushStart)
+		}
+	}
+}
+
+// TestFlushOffsetGap_ConcurrentWriteAndFlush tests for race conditions
+// between writing new messages and flushing old ones.
+func TestFlushOffsetGap_ConcurrentWriteAndFlush(t *testing.T) {
+	var allFlushedOffsets []int64
+	var flushMu sync.Mutex
+
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+		t.Logf("FLUSH: offsets %d-%d (%d bytes)", minOffset, maxOffset, len(buf))
+
+		flushMu.Lock()
+		// Record the offset range that was flushed
+		for offset := minOffset; offset <= maxOffset; offset++ {
+			allFlushedOffsets = append(allFlushedOffsets, offset)
+		}
+		flushMu.Unlock()
+	}
+
+	logBuffer := NewLogBuffer("test", 50*time.Millisecond, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Concurrently write messages and force flushes
+	var wg sync.WaitGroup
+
+	// Writer goroutine
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 200; i++ {
+			if err := logBuffer.AddToBuffer(&mq_pb.DataMessage{
+				Key:   []byte(fmt.Sprintf("key-%d", i)),
+				Value: []byte(fmt.Sprintf("message-%d", i)),
+				TsNs:  time.Now().UnixNano(),
+			}); err != nil {
+				t.Errorf("Failed to add buffer: %v", err)
+				return
+			}
+			if i%50 == 0 {
+				time.Sleep(10 * time.Millisecond)
+			}
+		}
+	}()
+
+	// Flusher goroutine
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 5; i++ {
+			time.Sleep(30 * time.Millisecond)
+			logBuffer.ForceFlush()
+		}
+	}()
+
+	wg.Wait()
+	time.Sleep(200 * time.Millisecond) // Wait for final flush
+
+	// Check final state
+	logBuffer.RLock()
+	finalOffset := logBuffer.offset
+	finalBufferStart := logBuffer.bufferStartOffset
+	logBuffer.RUnlock()
+
+	flushMu.Lock()
+	flushedCount := len(allFlushedOffsets)
+	flushMu.Unlock()
+
+	expectedCount := int(finalOffset)
+	inMemory := int(finalOffset - finalBufferStart)
+	totalAccountedFor := flushedCount + inMemory
+
+	t.Logf("\nFINAL STATE:")
+	t.Logf("  Total messages sent: %d (offsets 0-%d)", expectedCount, expectedCount-1)
+	t.Logf("  Flushed to disk: %d", flushedCount)
+	t.Logf("  In memory: %d (offsets %d-%d)", inMemory, finalBufferStart, finalOffset-1)
+	t.Logf("  Total accounted: %d", totalAccountedFor)
+	t.Logf("  Missing: %d", expectedCount-totalAccountedFor)
+
+	if totalAccountedFor < expectedCount {
+		t.Errorf("DATA LOSS in concurrent scenario: %d messages missing!", expectedCount-totalAccountedFor)
+	}
+}
+
+// TestFlushOffsetGap_ProductionScenario reproduces the actual production scenario
+// where the broker uses AddLogEntryToBuffer with explicit Kafka offsets.
+// This simulates leader publishing with offset assignment.
+func TestFlushOffsetGap_ProductionScenario(t *testing.T) {
+	var flushedData []struct {
+		minOffset int64
+		maxOffset int64
+		messages  []*filer_pb.LogEntry
+	}
+	var flushMu sync.Mutex
+
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+		// Parse messages from buffer
+		messages := []*filer_pb.LogEntry{}
+		for pos := 0; pos+4 < len(buf); {
+			size := uint32(buf[pos])<<24 | uint32(buf[pos+1])<<16 | uint32(buf[pos+2])<<8 | uint32(buf[pos+3])
+			if pos+4+int(size) > len(buf) {
+				break
+			}
+			entryData := buf[pos+4 : pos+4+int(size)]
+			logEntry := &filer_pb.LogEntry{}
+			if err := proto.Unmarshal(entryData, logEntry); err == nil {
+				messages = append(messages, logEntry)
+			}
+			pos += 4 + int(size)
+		}
+
+		flushMu.Lock()
+		flushedData = append(flushedData, struct {
+			minOffset int64
+			maxOffset int64
+			messages  []*filer_pb.LogEntry
+		}{minOffset, maxOffset, messages})
+		flushMu.Unlock()
+
+		t.Logf("FLUSH: minOffset=%d maxOffset=%d, parsed %d messages", minOffset, maxOffset, len(messages))
+	}
+
+	logBuffer := NewLogBuffer("test", time.Hour, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Simulate broker behavior: assign Kafka offsets and add to buffer
+	// This is what PublishWithOffset() does
+	nextKafkaOffset := int64(0)
+
+	// Round 1: Add 50 messages with Kafka offsets 0-49
+	t.Logf("\n=== ROUND 1: Adding messages 0-49 ===")
+	for i := 0; i < 50; i++ {
+		logEntry := &filer_pb.LogEntry{
+			Key:    []byte(fmt.Sprintf("key-%d", i)),
+			Data:   []byte(fmt.Sprintf("message-%d", i)),
+			TsNs:   time.Now().UnixNano(),
+			Offset: nextKafkaOffset, // Explicit Kafka offset
+		}
+		if err := logBuffer.AddLogEntryToBuffer(logEntry); err != nil {
+			t.Fatalf("Failed to add log entry: %v", err)
+		}
+		nextKafkaOffset++
+	}
+
+	// Check buffer state before flush
+	logBuffer.RLock()
+	beforeFlushOffset := logBuffer.offset
+	beforeFlushStart := logBuffer.bufferStartOffset
+	logBuffer.RUnlock()
+	t.Logf("Before flush: logBuffer.offset=%d, bufferStartOffset=%d, nextKafkaOffset=%d",
+		beforeFlushOffset, beforeFlushStart, nextKafkaOffset)
+
+	// Flush
+	logBuffer.ForceFlush()
+	time.Sleep(100 * time.Millisecond)
+
+	// Check buffer state after flush
+	logBuffer.RLock()
+	afterFlushOffset := logBuffer.offset
+	afterFlushStart := logBuffer.bufferStartOffset
+	logBuffer.RUnlock()
+	t.Logf("After flush: logBuffer.offset=%d, bufferStartOffset=%d",
+		afterFlushOffset, afterFlushStart)
+
+	// Round 2: Add another 50 messages with Kafka offsets 50-99
+	t.Logf("\n=== ROUND 2: Adding messages 50-99 ===")
+	for i := 0; i < 50; i++ {
+		logEntry := &filer_pb.LogEntry{
+			Key:    []byte(fmt.Sprintf("key-%d", 50+i)),
+			Data:   []byte(fmt.Sprintf("message-%d", 50+i)),
+			TsNs:   time.Now().UnixNano(),
+			Offset: nextKafkaOffset,
+		}
+		if err := logBuffer.AddLogEntryToBuffer(logEntry); err != nil {
+			t.Fatalf("Failed to add log entry: %v", err)
+		}
+		nextKafkaOffset++
+	}
+
+	logBuffer.ForceFlush()
+	time.Sleep(100 * time.Millisecond)
+
+	// Verification: Check if all Kafka offsets are accounted for
+	flushMu.Lock()
+	t.Logf("\n=== VERIFICATION ===")
+	t.Logf("Expected Kafka offsets: 0-%d", nextKafkaOffset-1)
+
+	allOffsets := make(map[int64]bool)
+	for flushIdx, flush := range flushedData {
+		t.Logf("Flush #%d: minOffset=%d, maxOffset=%d, messages=%d",
+			flushIdx, flush.minOffset, flush.maxOffset, len(flush.messages))
+
+		for _, msg := range flush.messages {
+			if allOffsets[msg.Offset] {
+				t.Errorf("  DUPLICATE: Offset %d appears multiple times!", msg.Offset)
+			}
+			allOffsets[msg.Offset] = true
+		}
+	}
+	flushMu.Unlock()
+
+	// Check for missing offsets
+	missingOffsets := []int64{}
+	for expectedOffset := int64(0); expectedOffset < nextKafkaOffset; expectedOffset++ {
+		if !allOffsets[expectedOffset] {
+			missingOffsets = append(missingOffsets, expectedOffset)
+		}
+	}
+
+	if len(missingOffsets) > 0 {
+		t.Errorf("\nMISSING OFFSETS DETECTED: %d offsets missing", len(missingOffsets))
+		if len(missingOffsets) <= 20 {
+			t.Errorf("Missing: %v", missingOffsets)
+		} else {
+			t.Errorf("Missing: %v ... and %d more", missingOffsets[:20], len(missingOffsets)-20)
+		}
+		t.Errorf("\nThis reproduces the production bug!")
+	} else {
+		t.Logf("\nSUCCESS: All %d Kafka offsets accounted for (0-%d)", nextKafkaOffset, nextKafkaOffset-1)
+	}
+
+	// Check buffer offset consistency
+	logBuffer.RLock()
+	finalOffset := logBuffer.offset
+	finalBufferStart := logBuffer.bufferStartOffset
+	logBuffer.RUnlock()
+
+	t.Logf("\nFinal buffer state:")
+	t.Logf("  logBuffer.offset: %d", finalOffset)
+	t.Logf("  bufferStartOffset: %d", finalBufferStart)
+	t.Logf("  Expected (nextKafkaOffset): %d", nextKafkaOffset)
+
+	if finalOffset != nextKafkaOffset {
+		t.Errorf("logBuffer.offset mismatch: expected %d, got %d", nextKafkaOffset, finalOffset)
+	}
+}
+
+// TestFlushOffsetGap_ConcurrentReadDuringFlush tests if concurrent reads
+// during flush can cause messages to be missed.
+func TestFlushOffsetGap_ConcurrentReadDuringFlush(t *testing.T) {
+	var flushedOffsets []int64
+	var flushMu sync.Mutex
+
+	readFromDiskFn := func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+		// Simulate reading from disk - return flushed offsets
+		flushMu.Lock()
+		defer flushMu.Unlock()
+
+		for _, offset := range flushedOffsets {
+			if offset >= startPosition.Offset {
+				logEntry := &filer_pb.LogEntry{
+					Key:    []byte(fmt.Sprintf("key-%d", offset)),
+					Data:   []byte(fmt.Sprintf("message-%d", offset)),
+					TsNs:   time.Now().UnixNano(),
+					Offset: offset,
+				}
+				isDone, err := eachLogEntryFn(logEntry)
+				if err != nil || isDone {
+					return NewMessagePositionFromOffset(offset + 1), isDone, err
+				}
+			}
+		}
+		return startPosition, false, nil
+	}
+
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+		// Parse and store flushed offsets
+		flushMu.Lock()
+		defer flushMu.Unlock()
+
+		for pos := 0; pos+4 < len(buf); {
+			size := uint32(buf[pos])<<24 | uint32(buf[pos+1])<<16 | uint32(buf[pos+2])<<8 | uint32(buf[pos+3])
+			if pos+4+int(size) > len(buf) {
+				break
+			}
+			entryData := buf[pos+4 : pos+4+int(size)]
+			logEntry := &filer_pb.LogEntry{}
+			if err := proto.Unmarshal(entryData, logEntry); err == nil {
+				flushedOffsets = append(flushedOffsets, logEntry.Offset)
+			}
+			pos += 4 + int(size)
+		}
+
+		t.Logf("FLUSH: Stored %d offsets to disk (minOffset=%d, maxOffset=%d)",
+			len(flushedOffsets), minOffset, maxOffset)
+	}
+
+	logBuffer := NewLogBuffer("test", time.Hour, flushFn, readFromDiskFn, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Add 100 messages
+	t.Logf("Adding 100 messages...")
+	for i := int64(0); i < 100; i++ {
+		logEntry := &filer_pb.LogEntry{
+			Key:    []byte(fmt.Sprintf("key-%d", i)),
+			Data:   []byte(fmt.Sprintf("message-%d", i)),
+			TsNs:   time.Now().UnixNano(),
+			Offset: i,
+		}
+		if err := logBuffer.AddLogEntryToBuffer(logEntry); err != nil {
+			t.Fatalf("Failed to add log entry: %v", err)
+		}
+	}
+
+	// Flush (moves data to disk)
+	t.Logf("Flushing...")
+	logBuffer.ForceFlush()
+	time.Sleep(100 * time.Millisecond)
+
+	// Now try to read all messages using ReadMessagesAtOffset
+	t.Logf("\nReading messages from offset 0...")
+	messages, nextOffset, hwm, endOfPartition, err := logBuffer.ReadMessagesAtOffset(0, 1000, 1024*1024)
+
+	t.Logf("Read result: messages=%d, nextOffset=%d, hwm=%d, endOfPartition=%v, err=%v",
+		len(messages), nextOffset, hwm, endOfPartition, err)
+
+	// Verify all offsets can be read
+	readOffsets := make(map[int64]bool)
+	for _, msg := range messages {
+		readOffsets[msg.Offset] = true
+	}
+
+	missingOffsets := []int64{}
+	for expectedOffset := int64(0); expectedOffset < 100; expectedOffset++ {
+		if !readOffsets[expectedOffset] {
+			missingOffsets = append(missingOffsets, expectedOffset)
+		}
+	}
+
+	if len(missingOffsets) > 0 {
+		t.Errorf("MISSING OFFSETS after flush: %d offsets cannot be read", len(missingOffsets))
+		if len(missingOffsets) <= 20 {
+			t.Errorf("Missing: %v", missingOffsets)
+		} else {
+			t.Errorf("Missing: %v ... and %d more", missingOffsets[:20], len(missingOffsets)-20)
+		}
+	} else {
+		t.Logf("All 100 offsets can be read after flush")
+	}
+}
+
+// TestFlushOffsetGap_ForceFlushAdvancesBuffer tests if ForceFlush
+// properly advances bufferStartOffset after flushing.
+func TestFlushOffsetGap_ForceFlushAdvancesBuffer(t *testing.T) {
+	flushedRanges := []struct{ min, max int64 }{}
+	var flushMu sync.Mutex
+
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+		flushMu.Lock()
+		flushedRanges = append(flushedRanges, struct{ min, max int64 }{minOffset, maxOffset})
+		flushMu.Unlock()
+		t.Logf("FLUSH: offsets %d-%d", minOffset, maxOffset)
+	}
+
+	logBuffer := NewLogBuffer("test", time.Hour, flushFn, nil, nil) // Long interval, manual flush only
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Send messages, flush, check state - repeat
+	for round := 0; round < 3; round++ {
+		t.Logf("\n=== ROUND %d ===", round)
+
+		// Check state before adding messages
+		logBuffer.RLock()
+		beforeOffset := logBuffer.offset
+		beforeStart := logBuffer.bufferStartOffset
+		logBuffer.RUnlock()
+
+		t.Logf("Before adding: offset=%d, bufferStartOffset=%d", beforeOffset, beforeStart)
+
+		// Add 10 messages
+		for i := 0; i < 10; i++ {
+			if err := logBuffer.AddToBuffer(&mq_pb.DataMessage{
+				Key:   []byte(fmt.Sprintf("round-%d-msg-%d", round, i)),
+				Value: []byte(fmt.Sprintf("data-%d-%d", round, i)),
+				TsNs:  time.Now().UnixNano(),
+			}); err != nil {
+				t.Fatalf("Failed to add buffer: %v", err)
+			}
+		}
+
+		// Check state after adding
+		logBuffer.RLock()
+		afterAddOffset := logBuffer.offset
+		afterAddStart := logBuffer.bufferStartOffset
+		logBuffer.RUnlock()
+
+		t.Logf("After adding: offset=%d, bufferStartOffset=%d", afterAddOffset, afterAddStart)
+
+		// Force flush
+		t.Logf("Forcing flush...")
+		logBuffer.ForceFlush()
+		time.Sleep(100 * time.Millisecond)
+
+		// Check state after flush
+		logBuffer.RLock()
+		afterFlushOffset := logBuffer.offset
+		afterFlushStart := logBuffer.bufferStartOffset
+		logBuffer.RUnlock()
+
+		t.Logf("After flush: offset=%d, bufferStartOffset=%d", afterFlushOffset, afterFlushStart)
+
+		// CRITICAL CHECK: bufferStartOffset should advance to where offset was before flush
+		if afterFlushStart != afterAddOffset {
+			t.Errorf("FLUSH BUG: bufferStartOffset did NOT advance correctly!")
+			t.Errorf("   Expected bufferStartOffset=%d (= offset after add)", afterAddOffset)
+			t.Errorf("   Actual bufferStartOffset=%d", afterFlushStart)
+			t.Errorf("   Gap: %d offsets WILL BE LOST", afterAddOffset-afterFlushStart)
+		} else {
+			t.Logf("bufferStartOffset correctly advanced to %d", afterFlushStart)
+		}
+	}
+
+	// Final verification: check all offset ranges are continuous
+	flushMu.Lock()
+	t.Logf("\n=== FLUSHED RANGES ===")
+	for i, r := range flushedRanges {
+		t.Logf("Flush #%d: offsets %d-%d", i, r.min, r.max)
+
+		// Check continuity with previous flush
+		if i > 0 {
+			prevMax := flushedRanges[i-1].max
+			currentMin := r.min
+			gap := currentMin - (prevMax + 1)
+
+			if gap > 0 {
+				t.Errorf("GAP between flush #%d and #%d: %d offsets missing!", i-1, i, gap)
+			} else if gap < 0 {
+				t.Errorf("OVERLAP between flush #%d and #%d: %d offsets duplicated!", i-1, i, -gap)
+			} else {
+				t.Logf("  Continuous with previous flush")
+			}
+		}
+	}
+	flushMu.Unlock()
+}

weed/util/log_buffer/log_buffer_oversized_test.go

@@ -0,0 +1,66 @@
+package log_buffer
+
+import (
+	"testing"
+	"time"
+)
+
+// An entry larger than BufferSize grows the window array to hold it. Window
+// arrays cycle through SealBuffer rather than being freed, so the grown one has
+// to be let go once it comes back around -- otherwise every later window is
+// allocated and snapshotted at the oversized width.
+func TestOversizedEntryDoesNotInflateBufferForever(t *testing.T) {
+	lb := NewLogBuffer("oversized", time.Hour, func(_ *LogBuffer, _, _ time.Time, _ []byte, _, _ int64) {}, nil, func() {})
+	defer lb.ShutdownLogBuffer()
+
+	if err := lb.AddDataToBuffer(nil, make([]byte, BufferSize+1), time.Now().UnixNano()); err != nil {
+		t.Fatalf("add oversized entry: %v", err)
+	}
+	if len(lb.buf) <= BufferSize {
+		t.Fatalf("oversized entry did not grow the buffer: %d bytes", len(lb.buf))
+	}
+
+	// One seal per window: the grown array lands in the newest sealed slot and
+	// needs a full trip through the rotation to come back as the current buffer.
+	for i := 0; i < PreviousBufferCount+2; i++ {
+		lb.ForceFlush()
+		if err := lb.AddDataToBuffer(nil, []byte("small"), time.Now().UnixNano()); err != nil {
+			t.Fatalf("add small entry: %v", err)
+		}
+	}
+
+	if len(lb.buf) > BufferSize {
+		t.Errorf("current buffer still %d bytes, want at most %d", len(lb.buf), BufferSize)
+	}
+	for i, sealed := range lb.prevBuffers.buffers {
+		if len(sealed.buf) > BufferSize {
+			t.Errorf("sealed buffer %d still %d bytes, want at most %d", i, len(sealed.buf), BufferSize)
+		}
+	}
+}
+
+// The oversized entry itself still has to survive the round trip.
+func TestOversizedEntryStillFlushes(t *testing.T) {
+	var flushed [][]byte
+	lb := NewLogBuffer("oversized-flush", time.Hour, func(_ *LogBuffer, _, _ time.Time, buf []byte, _, _ int64) {
+		flushed = append(flushed, append([]byte(nil), buf...))
+	}, nil, func() {})
+	defer lb.ShutdownLogBuffer()
+
+	payload := make([]byte, BufferSize+1)
+	for i := range payload {
+		payload[i] = byte(i)
+	}
+	if err := lb.AddDataToBuffer(nil, payload, time.Now().UnixNano()); err != nil {
+		t.Fatalf("add oversized entry: %v", err)
+	}
+	lb.ForceFlush()
+
+	var total int
+	for _, buf := range flushed {
+		total += len(buf)
+	}
+	if total < len(payload) {
+		t.Errorf("flushed %d bytes, want at least the %d byte payload", total, len(payload))
+	}
+}

weed/util/log_buffer/log_buffer_queryability_test.go

@@ -0,0 +1,303 @@
+package log_buffer
+
+import (
+	"bytes"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/proto"
+)
+
+// TestBufferQueryability tests that data written to the buffer can be immediately queried
+func TestBufferQueryability(t *testing.T) {
+	// Create a log buffer with a long flush interval to prevent premature flushing
+	logBuffer := NewLogBuffer("test-buffer", 10*time.Minute,
+		func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+			// Mock flush function - do nothing to keep data in memory
+		},
+		func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+			// Mock read from disk function
+			return startPosition, false, nil
+		},
+		func() {
+			// Mock notify function
+		})
+
+	// Test data similar to schema registry messages
+	testKey := []byte(`{"keytype":"SCHEMA","subject":"test-topic-value","version":1,"magic":1}`)
+	testValue := []byte(`{"subject":"test-topic-value","version":1,"id":1,"schemaType":"AVRO","schema":"\"string\"","deleted":false}`)
+
+	// Create a LogEntry with offset (simulating the schema registry scenario)
+	logEntry := &filer_pb.LogEntry{
+		TsNs:             time.Now().UnixNano(),
+		PartitionKeyHash: 12345,
+		Data:             testValue,
+		Key:              testKey,
+		Offset:           1,
+	}
+
+	// Add the entry to the buffer
+	if err := logBuffer.AddLogEntryToBuffer(logEntry); err != nil {
+		t.Fatalf("Failed to add log entry: %v", err)
+	}
+
+	// Verify the buffer has data
+	if logBuffer.pos == 0 {
+		t.Fatal("Buffer should have data after adding entry")
+	}
+
+	// Test immediate queryability - read from buffer starting from beginning
+	startPosition := NewMessagePosition(0, 0) // Start from beginning
+	bufferCopy, batchIndex, _, err := logBuffer.ReadFromBuffer(startPosition)
+
+	if err != nil {
+		t.Fatalf("ReadFromBuffer failed: %v", err)
+	}
+
+	if bufferCopy == nil {
+		t.Fatal("ReadFromBuffer returned nil buffer - data should be queryable immediately")
+	}
+
+	if batchIndex != 1 {
+		t.Errorf("Expected batchIndex=1, got %d", batchIndex)
+	}
+
+	// Verify we can read the data back
+	buf := bufferCopy.Bytes()
+	if len(buf) == 0 {
+		t.Fatal("Buffer copy is empty")
+	}
+
+	// Parse the first entry from the buffer
+	if len(buf) < 4 {
+		t.Fatal("Buffer too small to contain entry size")
+	}
+
+	size := util.BytesToUint32(buf[0:4])
+	if len(buf) < 4+int(size) {
+		t.Fatalf("Buffer too small to contain entry data: need %d, have %d", 4+int(size), len(buf))
+	}
+
+	entryData := buf[4 : 4+int(size)]
+
+	// Unmarshal and verify the entry
+	retrievedEntry := &filer_pb.LogEntry{}
+	if err := proto.Unmarshal(entryData, retrievedEntry); err != nil {
+		t.Fatalf("Failed to unmarshal retrieved entry: %v", err)
+	}
+
+	// Verify the data matches
+	if !bytes.Equal(retrievedEntry.Key, testKey) {
+		t.Errorf("Key mismatch: expected %s, got %s", string(testKey), string(retrievedEntry.Key))
+	}
+
+	if !bytes.Equal(retrievedEntry.Data, testValue) {
+		t.Errorf("Value mismatch: expected %s, got %s", string(testValue), string(retrievedEntry.Data))
+	}
+
+	if retrievedEntry.Offset != 1 {
+		t.Errorf("Offset mismatch: expected 1, got %d", retrievedEntry.Offset)
+	}
+
+	t.Logf("Buffer queryability test passed - data is immediately readable")
+}
+
+// TestMultipleEntriesQueryability tests querying multiple entries from buffer
+func TestMultipleEntriesQueryability(t *testing.T) {
+	logBuffer := NewLogBuffer("test-multi-buffer", 10*time.Minute,
+		func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+			// Mock flush function
+		},
+		func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+			return startPosition, false, nil
+		},
+		func() {})
+
+	// Add multiple entries
+	for i := 1; i <= 3; i++ {
+		logEntry := &filer_pb.LogEntry{
+			TsNs:             time.Now().UnixNano() + int64(i*1000), // Ensure different timestamps
+			PartitionKeyHash: int32(i),
+			Data:             []byte("test-data-" + string(rune('0'+i))),
+			Key:              []byte("test-key-" + string(rune('0'+i))),
+			Offset:           int64(i),
+		}
+		if err := logBuffer.AddLogEntryToBuffer(logEntry); err != nil {
+			t.Fatalf("Failed to add log entry: %v", err)
+		}
+	}
+
+	// Read all entries
+	startPosition := NewMessagePosition(0, 0)
+	bufferCopy, batchIndex, _, err := logBuffer.ReadFromBuffer(startPosition)
+
+	if err != nil {
+		t.Fatalf("ReadFromBuffer failed: %v", err)
+	}
+
+	if bufferCopy == nil {
+		t.Fatal("ReadFromBuffer returned nil buffer")
+	}
+
+	if batchIndex != 3 {
+		t.Errorf("Expected batchIndex=3, got %d", batchIndex)
+	}
+
+	// Count entries in buffer
+	buf := bufferCopy.Bytes()
+	entryCount := 0
+	pos := 0
+
+	for pos+4 < len(buf) {
+		size := util.BytesToUint32(buf[pos : pos+4])
+		if pos+4+int(size) > len(buf) {
+			break
+		}
+
+		entryData := buf[pos+4 : pos+4+int(size)]
+		entry := &filer_pb.LogEntry{}
+		if err := proto.Unmarshal(entryData, entry); err != nil {
+			t.Fatalf("Failed to unmarshal entry %d: %v", entryCount+1, err)
+		}
+
+		entryCount++
+		pos += 4 + int(size)
+
+		t.Logf("Entry %d: Key=%s, Data=%s, Offset=%d", entryCount, string(entry.Key), string(entry.Data), entry.Offset)
+	}
+
+	if entryCount != 3 {
+		t.Errorf("Expected 3 entries, found %d", entryCount)
+	}
+
+	t.Logf("Multiple entries queryability test passed - found %d entries", entryCount)
+}
+
+// TestSchemaRegistryScenario tests the specific scenario that was failing
+func TestSchemaRegistryScenario(t *testing.T) {
+	logBuffer := NewLogBuffer("_schemas", 10*time.Minute,
+		func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+			// Mock flush function - simulate what happens in real scenario
+			t.Logf("FLUSH: startTime=%v, stopTime=%v, bufSize=%d, minOffset=%d, maxOffset=%d",
+				startTime, stopTime, len(buf), minOffset, maxOffset)
+		},
+		func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+			return startPosition, false, nil
+		},
+		func() {})
+
+	// Simulate schema registry message
+	schemaKey := []byte(`{"keytype":"SCHEMA","subject":"test-schema-value","version":1,"magic":1}`)
+	schemaValue := []byte(`{"subject":"test-schema-value","version":1,"id":12,"schemaType":"AVRO","schema":"\"string\"","deleted":false}`)
+
+	logEntry := &filer_pb.LogEntry{
+		TsNs:             time.Now().UnixNano(),
+		PartitionKeyHash: 12345,
+		Data:             schemaValue,
+		Key:              schemaKey,
+		Offset:           0, // First message
+	}
+
+	// Add to buffer
+	if err := logBuffer.AddLogEntryToBuffer(logEntry); err != nil {
+		t.Fatalf("Failed to add log entry: %v", err)
+	}
+
+	// Simulate the SQL query scenario - read from offset 0
+	startPosition := NewMessagePosition(0, 0)
+	bufferCopy, _, _, err := logBuffer.ReadFromBuffer(startPosition)
+
+	if err != nil {
+		t.Fatalf("Schema registry scenario failed: %v", err)
+	}
+
+	if bufferCopy == nil {
+		t.Fatal("Schema registry scenario: ReadFromBuffer returned nil - this is the bug!")
+	}
+
+	// Verify schema data is readable
+	buf := bufferCopy.Bytes()
+	if len(buf) < 4 {
+		t.Fatal("Buffer too small")
+	}
+
+	size := util.BytesToUint32(buf[0:4])
+	entryData := buf[4 : 4+int(size)]
+
+	retrievedEntry := &filer_pb.LogEntry{}
+	if err := proto.Unmarshal(entryData, retrievedEntry); err != nil {
+		t.Fatalf("Failed to unmarshal schema entry: %v", err)
+	}
+
+	// Verify schema value is preserved
+	if !bytes.Equal(retrievedEntry.Data, schemaValue) {
+		t.Errorf("Schema value lost! Expected: %s, Got: %s", string(schemaValue), string(retrievedEntry.Data))
+	}
+
+	if len(retrievedEntry.Data) != len(schemaValue) {
+		t.Errorf("Schema value length mismatch! Expected: %d, Got: %d", len(schemaValue), len(retrievedEntry.Data))
+	}
+
+	t.Logf("Schema registry scenario test passed - schema value preserved: %d bytes", len(retrievedEntry.Data))
+}
+
+// TestTimeBasedFirstReadBeforeEarliest ensures starting slightly before earliest memory
+// does not force a disk resume and returns in-memory data (regression test)
+func TestTimeBasedFirstReadBeforeEarliest(t *testing.T) {
+	flushed := false
+	logBuffer := NewLogBuffer("local", 10*time.Minute,
+		func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+			// keep in memory; we just want earliest time populated
+			_ = buf
+		},
+		func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+			// disk should not be consulted in this regression path
+			return startPosition, false, nil
+		},
+		func() {})
+
+	// Seed one entry so earliestTime is set
+	baseTs := time.Now().Add(-time.Second)
+	entry := &filer_pb.LogEntry{TsNs: baseTs.UnixNano(), Data: []byte("x"), Key: []byte("k"), Offset: 0}
+	if err := logBuffer.AddLogEntryToBuffer(entry); err != nil {
+		t.Fatalf("Failed to add log entry: %v", err)
+	}
+	_ = flushed
+
+	// Start read 1ns before earliest memory, with offset sentinel (-2)
+	startPos := NewMessagePosition(baseTs.Add(-time.Nanosecond).UnixNano(), -2)
+	buf, _, _, err := logBuffer.ReadFromBuffer(startPos)
+	if err != nil {
+		t.Fatalf("ReadFromBuffer returned err: %v", err)
+	}
+	if buf == nil {
+		t.Fatalf("Expected in-memory data, got nil buffer")
+	}
+}
+
+// TestEarliestTimeExactRead ensures starting exactly at earliest time returns first entry (no skip)
+func TestEarliestTimeExactRead(t *testing.T) {
+	logBuffer := NewLogBuffer("local", 10*time.Minute,
+		func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {},
+		func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+			return startPosition, false, nil
+		},
+		func() {})
+
+	ts := time.Now()
+	entry := &filer_pb.LogEntry{TsNs: ts.UnixNano(), Data: []byte("a"), Key: []byte("k"), Offset: 0}
+	if err := logBuffer.AddLogEntryToBuffer(entry); err != nil {
+		t.Fatalf("Failed to add log entry: %v", err)
+	}
+
+	startPos := NewMessagePosition(ts.UnixNano(), -2)
+	buf, _, _, err := logBuffer.ReadFromBuffer(startPos)
+	if err != nil {
+		t.Fatalf("ReadFromBuffer err: %v", err)
+	}
+	if buf == nil || buf.Len() == 0 {
+		t.Fatalf("Expected data at earliest time, got nil/empty")
+	}
+}

weed/util/log_buffer/log_buffer_test.go

@@ -0,0 +1,599 @@
+package log_buffer
+
+import (
+	"bytes"
+	"crypto/rand"
+	"fmt"
+	"io"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func TestNewLogBufferFirstBuffer(t *testing.T) {
+	flushInterval := time.Second
+	lb := NewLogBuffer("test", flushInterval, func(logBuffer *LogBuffer, startTime time.Time, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+		fmt.Printf("flush from %v to %v %d bytes\n", startTime, stopTime, len(buf))
+	}, nil, func() {
+	})
+
+	startTime := MessagePosition{Time: time.Now()}
+
+	messageSize := 1024
+	messageCount := 5000
+
+	receivedMessageCount := 0
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		lastProcessedTime, isDone, err := lb.LoopProcessLogData("test", startTime, 0, func() bool {
+			// stop if no more messages
+			return receivedMessageCount < messageCount
+		}, func(logEntry *filer_pb.LogEntry) (isDone bool, err error) {
+			receivedMessageCount++
+			if receivedMessageCount >= messageCount {
+				println("processed all messages")
+				return true, io.EOF
+			}
+			return false, nil
+		})
+
+		fmt.Printf("before flush: sent %d received %d\n", messageCount, receivedMessageCount)
+		fmt.Printf("lastProcessedTime %v isDone %v err: %v\n", lastProcessedTime, isDone, err)
+		if err != nil && err != io.EOF {
+			t.Errorf("unexpected error %v", err)
+		}
+	}()
+
+	var buf = make([]byte, messageSize)
+	for i := 0; i < messageCount; i++ {
+		rand.Read(buf)
+		if err := lb.AddToBuffer(&mq_pb.DataMessage{
+			Key:   nil,
+			Value: buf,
+			TsNs:  0,
+		}); err != nil {
+			t.Fatalf("Failed to add buffer: %v", err)
+		}
+	}
+	wg.Wait()
+
+	if receivedMessageCount != messageCount {
+		t.Errorf("expect %d messages, but got %d", messageCount, receivedMessageCount)
+	}
+}
+
+func TestReadFromBufferTimestampBased_AfterFlushReturnsNewerData(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	payload := bytes.Repeat([]byte("x"), 4096)
+	var sealed *MemBuffer
+
+	for i := 0; i < 5000; i++ {
+		if err := lb.AddDataToBuffer([]byte("k"), payload, int64(i+1)); err != nil {
+			t.Fatalf("AddDataToBuffer(%d): %v", i, err)
+		}
+		candidate := lb.prevBuffers.buffers[len(lb.prevBuffers.buffers)-1]
+		if candidate.size > 0 {
+			sealed = &MemBuffer{
+				size:      candidate.size,
+				startTime: candidate.startTime,
+				stopTime:  candidate.stopTime,
+				offset:    candidate.offset,
+			}
+			break
+		}
+	}
+
+	if sealed == nil {
+		t.Fatal("expected first buffer flush to produce a sealed buffer")
+	}
+
+	for i := 5000; i < 5100; i++ {
+		if err := lb.AddDataToBuffer([]byte("k"), payload, int64(i+1)); err != nil {
+			t.Fatalf("AddDataToBuffer(%d): %v", i, err)
+		}
+	}
+
+	buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(sealed.stopTime.UnixNano(), sealed.offset))
+	if err != nil {
+		t.Fatalf("ReadFromBuffer returned error: %v", err)
+	}
+	if buf == nil || buf.Len() == 0 {
+		t.Fatalf("expected newer data after the first sealed buffer, got %v", buf)
+	}
+}
+
+// TestReadFromBuffer_OldOffsetReturnsResumeFromDiskError tests that requesting an old offset
+// that has been flushed to disk properly returns ResumeFromDiskError instead of hanging forever.
+// This reproduces the bug where Schema Registry couldn't read the _schemas topic.
+func TestReadFromBuffer_OldOffsetReturnsResumeFromDiskError(t *testing.T) {
+	tests := []struct {
+		name              string
+		bufferStartOffset int64
+		currentOffset     int64
+		requestedOffset   int64
+		hasData           bool
+		expectError       error
+		description       string
+	}{
+		{
+			name:              "Request offset 0 when buffer starts at 4 (Schema Registry bug scenario)",
+			bufferStartOffset: 4,
+			currentOffset:     10,
+			requestedOffset:   0,
+			hasData:           true,
+			expectError:       ResumeFromDiskError,
+			description:       "When Schema Registry tries to read from offset 0, but data has been flushed to disk",
+		},
+		{
+			name:              "Request offset before buffer start with empty buffer",
+			bufferStartOffset: 10,
+			currentOffset:     10,
+			requestedOffset:   5,
+			hasData:           false,
+			expectError:       ResumeFromDiskError,
+			description:       "Old offset with no data in memory should trigger disk read",
+		},
+		{
+			name:              "Request offset before buffer start with data",
+			bufferStartOffset: 100,
+			currentOffset:     150,
+			requestedOffset:   50,
+			hasData:           true,
+			expectError:       ResumeFromDiskError,
+			description:       "Old offset with current data in memory should still trigger disk read",
+		},
+		{
+			name:              "Request current offset (no disk read needed)",
+			bufferStartOffset: 4,
+			currentOffset:     10,
+			requestedOffset:   10,
+			hasData:           true,
+			expectError:       nil,
+			description:       "Current offset should return data from memory without error",
+		},
+		{
+			name:              "Request offset within buffer range",
+			bufferStartOffset: 4,
+			currentOffset:     10,
+			requestedOffset:   7,
+			hasData:           true,
+			expectError:       nil,
+			description:       "Offset within buffer range should return data without error",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Create a LogBuffer with minimal configuration
+			lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+
+			// Simulate data that has been flushed to disk by setting bufferStartOffset
+			lb.bufferStartOffset = tt.bufferStartOffset
+			lb.offset = tt.currentOffset
+
+			// CRITICAL: Mark this as an offset-based buffer
+			lb.hasOffsets = true
+
+			// Add some data to the buffer if needed (at current offset position)
+			if tt.hasData {
+				testData := []byte("test message")
+				// Use AddLogEntryToBuffer to preserve offset information
+				if err := lb.AddLogEntryToBuffer(&filer_pb.LogEntry{
+					TsNs:   time.Now().UnixNano(),
+					Key:    []byte("key"),
+					Data:   testData,
+					Offset: tt.currentOffset, // Add data at current offset
+				}); err != nil {
+					t.Fatalf("Failed to add log entry: %v", err)
+				}
+			}
+
+			// Create an offset-based position for the requested offset
+			requestPosition := NewMessagePositionFromOffset(tt.requestedOffset)
+
+			// Try to read from the buffer
+			buf, batchIdx, _, err := lb.ReadFromBuffer(requestPosition)
+
+			// Verify the error matches expectations
+			if tt.expectError != nil {
+				if err != tt.expectError {
+					t.Errorf("%s\nExpected error: %v\nGot error: %v\nbuf=%v, batchIdx=%d",
+						tt.description, tt.expectError, err, buf != nil, batchIdx)
+				} else {
+					t.Logf("✓ %s: correctly returned %v", tt.description, err)
+				}
+			} else {
+				if err != nil {
+					t.Errorf("%s\nExpected no error but got: %v\nbuf=%v, batchIdx=%d",
+						tt.description, err, buf != nil, batchIdx)
+				} else {
+					t.Logf("✓ %s: correctly returned data without error", tt.description)
+				}
+			}
+		})
+	}
+}
+
+// TestReadFromBuffer_OldOffsetWithNoPrevBuffers specifically tests the bug fix
+// where requesting an old offset would return nil instead of ResumeFromDiskError
+func TestReadFromBuffer_OldOffsetWithNoPrevBuffers(t *testing.T) {
+	// This is the exact scenario that caused the Schema Registry to hang:
+	// 1. Data was published to _schemas topic (offsets 0, 1, 2, 3)
+	// 2. Data was flushed to disk
+	// 3. LogBuffer's bufferStartOffset was updated to 4
+	// 4. Schema Registry tried to read from offset 0
+	// 5. ReadFromBuffer would return (nil, offset, nil) instead of ResumeFromDiskError
+	// 6. The subscriber would wait forever for data that would never come from memory
+
+	lb := NewLogBuffer("_schemas", time.Hour, nil, nil, func() {})
+
+	// Simulate the state after data has been flushed to disk:
+	// - bufferStartOffset = 10 (data 0-9 has been flushed)
+	// - offset = 15 (next offset to assign, current buffer has 10-14)
+	// - pos = 100 (some data in current buffer)
+	// Set prevBuffers to have non-overlapping ranges to avoid the safety check at line 420-428
+	lb.bufferStartOffset = 10
+	lb.offset = 15
+	lb.pos = 100
+
+	// Modify prevBuffers to have non-zero offset ranges that DON'T include the requested offset
+	// This bypasses the safety check and exposes the real bug
+	for i := range lb.prevBuffers.buffers {
+		lb.prevBuffers.buffers[i].startOffset = 20 + int64(i)*10 // 20, 30, 40, etc.
+		lb.prevBuffers.buffers[i].offset = 25 + int64(i)*10      // 25, 35, 45, etc.
+		lb.prevBuffers.buffers[i].size = 0                       // Empty (flushed)
+	}
+
+	// Schema Registry requests offset 5 (which is before bufferStartOffset=10)
+	requestPosition := NewMessagePositionFromOffset(5)
+
+	// Before the fix, this would return (nil, offset, nil) causing an infinite wait
+	// After the fix, this should return ResumeFromDiskError
+	buf, batchIdx, _, err := lb.ReadFromBuffer(requestPosition)
+
+	t.Logf("DEBUG: ReadFromBuffer returned: buf=%v, batchIdx=%d, err=%v", buf != nil, batchIdx, err)
+	t.Logf("DEBUG: Buffer state: bufferStartOffset=%d, offset=%d, pos=%d",
+		lb.bufferStartOffset, lb.offset, lb.pos)
+	t.Logf("DEBUG: Requested offset 5, prevBuffers[0] range: [%d-%d]",
+		lb.prevBuffers.buffers[0].startOffset, lb.prevBuffers.buffers[0].offset)
+
+	if err != ResumeFromDiskError {
+		t.Errorf("CRITICAL BUG REPRODUCED: Expected ResumeFromDiskError but got err=%v, buf=%v, batchIdx=%d\n"+
+			"This causes Schema Registry to hang indefinitely waiting for data that's on disk!",
+			err, buf != nil, batchIdx)
+		t.Errorf("The buggy code falls through without returning ResumeFromDiskError!")
+	} else {
+		t.Logf("✓ BUG FIX VERIFIED: Correctly returns ResumeFromDiskError when requesting old offset 5")
+		t.Logf("  This allows the subscriber to read from disk instead of waiting forever")
+	}
+}
+
+// TestReadFromBuffer_EmptyBufferAtCurrentOffset tests Bug #2
+// where an empty buffer at the current offset would return empty data instead of ResumeFromDiskError
+func TestReadFromBuffer_EmptyBufferAtCurrentOffset(t *testing.T) {
+	lb := NewLogBuffer("_schemas", time.Hour, nil, nil, func() {})
+
+	// Simulate buffer state where data 0-3 was published and flushed, but buffer NOT advanced yet:
+	// - bufferStartOffset = 0 (buffer hasn't been advanced after flush)
+	// - offset = 4 (next offset to assign - data 0-3 exists)
+	// - pos = 0 (buffer is empty after flush)
+	// This happens in the window between flush and buffer advancement
+	lb.bufferStartOffset = 0
+	lb.offset = 4
+	lb.pos = 0
+
+	// Schema Registry requests offset 0 (which appears to be in range [0, 4])
+	requestPosition := NewMessagePositionFromOffset(0)
+
+	// BUG: Without fix, this returns empty buffer instead of checking disk
+	// FIX: Should return ResumeFromDiskError because buffer is empty (pos=0) despite valid range
+	buf, batchIdx, _, err := lb.ReadFromBuffer(requestPosition)
+
+	t.Logf("DEBUG: ReadFromBuffer returned: buf=%v, batchIdx=%d, err=%v", buf != nil, batchIdx, err)
+	t.Logf("DEBUG: Buffer state: bufferStartOffset=%d, offset=%d, pos=%d",
+		lb.bufferStartOffset, lb.offset, lb.pos)
+
+	if err != ResumeFromDiskError {
+		if buf == nil || len(buf.Bytes()) == 0 {
+			t.Errorf("CRITICAL BUG #2 REPRODUCED: Empty buffer should return ResumeFromDiskError, got err=%v, buf=%v\n"+
+				"Without the fix, Schema Registry gets empty data instead of reading from disk!",
+				err, buf != nil)
+		}
+	} else {
+		t.Logf("✓ BUG #2 FIX VERIFIED: Empty buffer correctly returns ResumeFromDiskError to check disk")
+	}
+}
+
+// TestReadFromBuffer_OffsetRanges tests various offset range scenarios
+func TestReadFromBuffer_OffsetRanges(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+
+	// Setup: buffer contains offsets 10-20
+	lb.bufferStartOffset = 10
+	lb.offset = 20
+	lb.pos = 100 // some data in buffer
+
+	testCases := []struct {
+		name            string
+		requestedOffset int64
+		expectedError   error
+		description     string
+	}{
+		{
+			name:            "Before buffer start",
+			requestedOffset: 5,
+			expectedError:   ResumeFromDiskError,
+			description:     "Offset 5 < bufferStartOffset 10 → read from disk",
+		},
+		{
+			name:            "At buffer start",
+			requestedOffset: 10,
+			expectedError:   nil,
+			description:     "Offset 10 == bufferStartOffset 10 → read from buffer",
+		},
+		{
+			name:            "Within buffer range",
+			requestedOffset: 15,
+			expectedError:   nil,
+			description:     "Offset 15 is within [10, 20] → read from buffer",
+		},
+		{
+			name:            "At buffer end",
+			requestedOffset: 20,
+			expectedError:   nil,
+			description:     "Offset 20 == offset 20 → read from buffer",
+		},
+		{
+			name:            "After buffer end",
+			requestedOffset: 25,
+			expectedError:   nil,
+			description:     "Offset 25 > offset 20 → future data, return nil without error",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			requestPosition := NewMessagePositionFromOffset(tc.requestedOffset)
+			_, _, _, err := lb.ReadFromBuffer(requestPosition)
+
+			if tc.expectedError != nil {
+				if err != tc.expectedError {
+					t.Errorf("%s\nExpected error: %v, got: %v", tc.description, tc.expectedError, err)
+				} else {
+					t.Logf("✓ %s", tc.description)
+				}
+			} else {
+				// For nil expectedError, we accept either nil or no error condition
+				// (future offsets return nil without error)
+				if err != nil && err != ResumeFromDiskError {
+					t.Errorf("%s\nExpected no ResumeFromDiskError, got: %v", tc.description, err)
+				} else {
+					t.Logf("✓ %s", tc.description)
+				}
+			}
+		})
+	}
+}
+
+// TestReadFromBuffer_InitializedFromDisk tests Bug #3
+// where bufferStartOffset was incorrectly set to 0 after InitializeOffsetFromExistingData,
+// causing reads for old offsets to return new data instead of triggering a disk read.
+func TestReadFromBuffer_InitializedFromDisk(t *testing.T) {
+	// This reproduces the real Schema Registry bug scenario:
+	// 1. Broker restarts, finds 4 messages on disk (offsets 0-3)
+	// 2. InitializeOffsetFromExistingData sets offset=4
+	//    - BUG: bufferStartOffset=0 (wrong!)
+	//    - FIX: bufferStartOffset=4 (correct!)
+	// 3. First new message is written (offset 4)
+	// 4. Schema Registry reads offset 0
+	// 5. With FIX: requestedOffset=0 < bufferStartOffset=4 → ResumeFromDiskError (correct!)
+	// 6. Without FIX: requestedOffset=0 in range [0, 5] → returns wrong data (bug!)
+
+	lb := NewLogBuffer("_schemas", time.Hour, nil, nil, func() {})
+
+	// Use the actual InitializeOffsetFromExistingData to test the fix
+	err := lb.InitializeOffsetFromExistingData(func() (int64, error) {
+		return 3, nil // Simulate 4 messages on disk (offsets 0-3, highest=3)
+	})
+	if err != nil {
+		t.Fatalf("InitializeOffsetFromExistingData failed: %v", err)
+	}
+
+	t.Logf("After InitializeOffsetFromExistingData(highestOffset=3):")
+	t.Logf("  offset=%d (should be 4), bufferStartOffset=%d (FIX: should be 4, not 0)",
+		lb.offset, lb.bufferStartOffset)
+
+	// Now write a new message at offset 4
+	if err := lb.AddToBuffer(&mq_pb.DataMessage{
+		Key:   []byte("new-key"),
+		Value: []byte("new-message-at-offset-4"),
+		TsNs:  time.Now().UnixNano(),
+	}); err != nil {
+		t.Fatalf("Failed to add buffer: %v", err)
+	}
+	// After AddToBuffer: offset=5, pos>0
+
+	// Schema Registry tries to read offset 0 (should be on disk)
+	requestPosition := NewMessagePositionFromOffset(0)
+
+	buf, batchIdx, _, err := lb.ReadFromBuffer(requestPosition)
+
+	t.Logf("After writing new message:")
+	t.Logf("  bufferStartOffset=%d, offset=%d, pos=%d", lb.bufferStartOffset, lb.offset, lb.pos)
+	t.Logf("  Requested offset 0, got: buf=%v, batchIdx=%d, err=%v", buf != nil, batchIdx, err)
+
+	// EXPECTED BEHAVIOR (with fix):
+	// bufferStartOffset=4 after initialization, so requestedOffset=0 < bufferStartOffset=4
+	// → returns ResumeFromDiskError
+
+	// BUGGY BEHAVIOR (without fix):
+	// bufferStartOffset=0 after initialization, so requestedOffset=0 is in range [0, 5]
+	// → returns the NEW message (offset 4) instead of reading from disk!
+
+	if err != ResumeFromDiskError {
+		t.Errorf("CRITICAL BUG #3 REPRODUCED: Reading offset 0 after initialization from disk should return ResumeFromDiskError\n"+
+			"Instead got: err=%v, buf=%v, batchIdx=%d\n"+
+			"This means Schema Registry would receive WRONG data (offset 4) when requesting offset 0!",
+			err, buf != nil, batchIdx)
+		t.Errorf("Root cause: bufferStartOffset=%d should be 4 after InitializeOffsetFromExistingData(highestOffset=3)",
+			lb.bufferStartOffset)
+	} else {
+		t.Logf("✓ BUG #3 FIX VERIFIED: Reading old offset 0 correctly returns ResumeFromDiskError")
+		t.Logf("  This ensures Schema Registry reads correct data from disk instead of getting new messages")
+	}
+}
+
+// TestLoopProcessLogDataWithOffset_DiskReadRetry tests that when a subscriber
+// reads from disk before data is available, it continues to retry disk reads
+// and eventually finds the data once it appears.  This reproduces the Schema
+// Registry timeout where the first read happened before the data landed on
+// disk and the loop never retried.
+//
+// The data is delivered exclusively through the disk-read path (no in-memory
+// AddToBuffer) so the test deterministically asserts that retries happen,
+// rather than racing the in-memory delivery path.
+func TestLoopProcessLogDataWithOffset_DiskReadRetry(t *testing.T) {
+	diskReadCallCount := 0
+	diskReadMu := sync.Mutex{}
+	dataReady := false
+	mockEntry := &filer_pb.LogEntry{
+		Key:    []byte("key-0"),
+		Data:   []byte("message-0"),
+		TsNs:   time.Now().UnixNano(),
+		Offset: 0,
+	}
+
+	readFromDiskFn := func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+		diskReadMu.Lock()
+		diskReadCallCount++
+		callNum := diskReadCallCount
+		ready := dataReady
+		diskReadMu.Unlock()
+
+		t.Logf("DISK READ #%d: startOffset=%d, dataReady=%v", callNum, startPosition.Offset, ready)
+
+		if !ready {
+			t.Logf("  → No data on disk yet")
+			return startPosition, false, nil
+		}
+
+		if mockEntry.Offset >= startPosition.Offset {
+			isDone, err := eachLogEntryFn(mockEntry)
+			if err != nil || isDone {
+				return NewMessagePositionFromOffset(mockEntry.Offset + 1), isDone, err
+			}
+		}
+		return NewMessagePositionFromOffset(mockEntry.Offset + 1), false, nil
+	}
+
+	logBuffer := NewLogBuffer("test", 1*time.Minute, nil, readFromDiskFn, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	receivedMessages := 0
+	mu := sync.Mutex{}
+	maxIterations := 50
+	iterationCount := 0
+
+	waitForDataFn := func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		iterationCount++
+		return receivedMessages == 0 && iterationCount < maxIterations
+	}
+
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+		mu.Lock()
+		receivedMessages++
+		mu.Unlock()
+		t.Logf("✉️  RECEIVED: offset=%d key=%s", offset, string(logEntry.Key))
+		return true, nil
+	}
+
+	var readerWg sync.WaitGroup
+	readerWg.Add(1)
+	go func() {
+		defer readerWg.Done()
+		startPosition := NewMessagePositionFromOffset(0)
+		_, isDone, err := logBuffer.LoopProcessLogDataWithOffset("test-subscriber", startPosition, 0, waitForDataFn, eachLogEntryFn)
+		t.Logf("📋 Reader finished: isDone=%v, err=%v", isDone, err)
+	}()
+
+	// Wait until the reader has issued at least one no-op disk read so we
+	// know it has entered the retry loop, then publish the data.
+	deadline := time.Now().Add(2 * time.Second)
+	for {
+		diskReadMu.Lock()
+		seen := diskReadCallCount
+		diskReadMu.Unlock()
+		if seen >= 1 {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("reader never called readFromDiskFn")
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+
+	t.Logf("➕ Marking data as ready on disk")
+	diskReadMu.Lock()
+	dataReady = true
+	diskReadMu.Unlock()
+
+	readerWg.Wait()
+
+	diskReadMu.Lock()
+	finalDiskReadCount := diskReadCallCount
+	diskReadMu.Unlock()
+
+	mu.Lock()
+	finalReceivedMessages := receivedMessages
+	finalIterations := iterationCount
+	mu.Unlock()
+
+	t.Logf("\nRESULTS:")
+	t.Logf("  Disk reads: %d", finalDiskReadCount)
+	t.Logf("  Received messages: %d", finalReceivedMessages)
+	t.Logf("  Loop iterations: %d", finalIterations)
+
+	if finalDiskReadCount < 2 {
+		t.Errorf("CRITICAL BUG REPRODUCED: Disk read was only called %d time(s)", finalDiskReadCount)
+		t.Errorf("Expected: Multiple disk reads as the loop continues after data lands")
+		t.Errorf("This is why Schema Registry times out - it reads once before data is available, never re-reads")
+	}
+
+	if finalReceivedMessages == 0 {
+		t.Errorf("SCHEMA REGISTRY TIMEOUT REPRODUCED: No messages received even after data landed")
+		t.Errorf("The subscriber is stuck because disk reads are not retried")
+	} else {
+		t.Logf("✓ SUCCESS: Message received after %d disk read attempts", finalDiskReadCount)
+	}
+}
+
+// A buffer shut down long before its flush interval elapses must not keep its
+// loops - and the tens of megabytes of slabs they reach - alive until the next
+// tick would have fired.
+func TestShutdownStopsGoroutinesPromptly(t *testing.T) {
+	lb := NewLogBuffer("shutdown", time.Hour, nil, nil, nil)
+	// let both loops start and park in their waits before shutting down
+	time.Sleep(100 * time.Millisecond)
+	lb.ShutdownLogBuffer()
+
+	done := make(chan struct{})
+	go func() {
+		lb.loopsDone.Wait()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("log buffer goroutines still running 5s after shutdown")
+	}
+}

weed/util/log_buffer/log_read.go

@@ -0,0 +1,597 @@
+package log_buffer
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var (
+	ResumeError         = fmt.Errorf("resume")
+	ResumeFromDiskError = fmt.Errorf("resumeFromDisk")
+	// StopReadingError, wrapped by an eachLogDataFn's error, marks a read the
+	// callback ended on purpose - control flow, not a failure - so the loop
+	// hands it back to the caller without logging it.
+	StopReadingError = fmt.Errorf("stopReading")
+)
+
+// notificationHealthCheckInterval bounds how long an idle subscriber blocks
+// on the notification channel before re-checking state (client disconnect via
+// waitForDataFn, LogBuffer shutdown, timestamp advancement). notifyChan is the
+// primary wakeup path when new data arrives or a flush lands; this timeout is
+// the safety net for any missed notification and also caps the latency to
+// notice that the subscriber should exit. Balances idle CPU and log noise
+// against client-disconnect detection latency.
+const notificationHealthCheckInterval = 250 * time.Millisecond
+
+// caughtUpDiskPollInterval is the (longer) timeout used once a subscriber has
+// already proven via ReadFromDiskFn that there is no data past its current
+// position on disk. In that state, notifyChan is the primary wakeup path for
+// in-memory writes; disk may still gain data from external writers (e.g.
+// Schema Registry's external disk-write path), so we still re-probe on
+// timeout, just at a much lower cadence than the 250ms health tick.
+const caughtUpDiskPollInterval = 2 * time.Second
+
+type MessagePosition struct {
+	Time          time.Time // timestamp of the message
+	Offset        int64     // Kafka offset for offset-based positioning, or batch index for timestamp-based
+	IsOffsetBased bool      // true if this position is offset-based, false if timestamp-based
+}
+
+func NewMessagePosition(tsNs int64, offset int64) MessagePosition {
+	return MessagePosition{
+		Time:          time.Unix(0, tsNs).UTC(),
+		Offset:        offset,
+		IsOffsetBased: false, // timestamp-based by default
+	}
+}
+
+// NewMessagePositionFromOffset creates a MessagePosition that represents a specific offset
+func NewMessagePositionFromOffset(offset int64) MessagePosition {
+	return MessagePosition{
+		Time:          time.Time{}, // Zero time for offset-based positions
+		Offset:        offset,
+		IsOffsetBased: true,
+	}
+}
+
+// GetOffset extracts the offset from an offset-based MessagePosition
+func (mp MessagePosition) GetOffset() int64 {
+	if !mp.IsOffsetBased {
+		return -1 // Not an offset-based position
+	}
+	return mp.Offset // Offset is stored directly
+}
+
+// awaitNotificationOrTimeout blocks until one of:
+//   - a new-data / flush notification arrives on notifyChan (returns true)
+//   - the LogBuffer is shut down via ShutdownLogBuffer (returns true; callers
+//     re-check IsStopping() and exit)
+//   - notificationHealthCheckInterval elapses (returns false; caller
+//     re-checks client-disconnect and other state)
+func (logBuffer *LogBuffer) awaitNotificationOrTimeout(notifyChan <-chan struct{}) bool {
+	return logBuffer.awaitNotificationOrTimeoutFor(notifyChan, notificationHealthCheckInterval)
+}
+
+// awaitNotificationOrTimeoutFor is awaitNotificationOrTimeout with a caller-supplied
+// timeout. Used by the ResumeFromDiskError path to back off to
+// caughtUpDiskPollInterval once the disk has been proven empty past the
+// subscriber's current position.
+func (logBuffer *LogBuffer) awaitNotificationOrTimeoutFor(notifyChan <-chan struct{}, timeout time.Duration) bool {
+	timer := time.NewTimer(timeout)
+	defer timer.Stop()
+
+	select {
+	case <-notifyChan:
+		return true
+	case <-logBuffer.shutdownCh:
+		return true
+	case <-timer.C:
+		return false
+	}
+}
+
+func (logBuffer *LogBuffer) LoopProcessLogData(readerName string, startPosition MessagePosition, stopTsNs int64,
+	waitForDataFn func() bool, eachLogDataFn EachLogEntryFuncType) (lastReadPosition MessagePosition, isDone bool, err error) {
+
+	// Register for instant notifications (<1ms latency)
+	notifyChan := logBuffer.RegisterSubscriber(readerName)
+	defer logBuffer.UnregisterSubscriber(readerName)
+
+	// loop through all messages
+	var bytesBuf *bytes.Buffer
+	var bytesBufPooled bool
+	var batchIndex int64
+	lastReadPosition = startPosition
+	var entryCounter int64
+	// caughtUpToDiskHead is set when ReadFromDiskFn last returned without
+	// advancing lastReadPosition, i.e. the disk has no data past our current
+	// position. In that steady state, the loop still re-probes buffer and disk
+	// (external writers can land data on disk without notifying subscribers),
+	// but at caughtUpDiskPollInterval instead of the 250ms health-check tick.
+	// Any progress (disk advance or notification) clears the flag, so the
+	// responsive 250ms cadence resumes for active readers.
+	caughtUpToDiskHead := false
+	defer func() {
+		if bytesBuf != nil && bytesBufPooled {
+			logBuffer.ReleaseMemory(bytesBuf)
+		}
+		// println("LoopProcessLogData", readerName, "sent messages total", entryCounter)
+	}()
+
+	for {
+
+		if bytesBuf != nil && bytesBufPooled {
+			logBuffer.ReleaseMemory(bytesBuf)
+			bytesBuf = nil // keep the deferred release from double-freeing if ReadFromBuffer panics
+		}
+		bytesBuf, batchIndex, bytesBufPooled, err = logBuffer.ReadFromBuffer(lastReadPosition)
+		if err == ResumeFromDiskError {
+			// Try to read from disk if readFromDiskFn is available
+			if logBuffer.ReadFromDiskFn != nil {
+				prevReadPosition := lastReadPosition
+				lastReadPosition, isDone, err = logBuffer.ReadFromDiskFn(lastReadPosition, stopTsNs, eachLogDataFn)
+				if err != nil {
+					return lastReadPosition, isDone, err
+				}
+				if isDone {
+					return lastReadPosition, isDone, nil
+				}
+				if lastReadPosition != prevReadPosition {
+					caughtUpToDiskHead = false
+					continue
+				}
+				if !caughtUpToDiskHead {
+					caughtUpToDiskHead = true
+					glog.V(4).Infof("%s: Caught up to disk head, backing off to %s poll", readerName, caughtUpDiskPollInterval)
+				}
+			} else if logBuffer.HasData() {
+				// HasData() and ReadFromBuffer lock separately, so a racing write can make
+				// HasData() see data the empty-buffer read missed. Re-read; only bail if the
+				// position is genuinely behind the in-memory window (flushed to disk).
+				reBuf, _, rePooled, reErr := logBuffer.ReadFromBuffer(lastReadPosition)
+				if reErr == ResumeFromDiskError {
+					return lastReadPosition, isDone, ResumeFromDiskError
+				}
+				if reBuf != nil && rePooled {
+					logBuffer.ReleaseMemory(reBuf)
+				}
+				continue
+			}
+
+			// Nothing readable anywhere for this cursor: a bounded subscription is
+			// done, same as the caught-up path below. Without this a bounded
+			// subscriber on a buffer that never took a write (empty since process
+			// start, ReadFromDiskFn nil) parks here forever while heartbeats keep
+			// the stream looking alive. The nil error matters: err still holds
+			// ResumeFromDiskError, which callers treat as retry-the-disk-pass.
+			if stopTsNs != 0 {
+				return lastReadPosition, true, nil
+			}
+
+			// CRITICAL: Check if client is still connected
+			if !waitForDataFn() {
+				// Client disconnected - exit cleanly
+				glog.V(4).Infof("%s: Client disconnected after disk read attempt", readerName)
+				return lastReadPosition, true, nil
+			}
+
+			// Wait for notification or timeout. While caught up to disk head,
+			// use the longer poll interval so idle readers don't spin every
+			// 250ms re-probing an empty buffer and disk.
+			waitTimeout := notificationHealthCheckInterval
+			if caughtUpToDiskHead {
+				waitTimeout = caughtUpDiskPollInterval
+			}
+			if logBuffer.awaitNotificationOrTimeoutFor(notifyChan, waitTimeout) {
+				glog.V(3).Infof("%s: Woke up from notification after ResumeFromDiskError", readerName)
+				caughtUpToDiskHead = false
+			}
+			// Silent on timeout while caught up: the re-probe cadence alone is
+			// the signal; no new log line per tick.
+
+			// If the LogBuffer is shutting down, exit cleanly instead of looping
+			// on ResumeFromDiskError. awaitNotificationOrTimeout returns true on
+			// shutdown (shutdownCh closed), which would otherwise spin here
+			// because ReadFromBuffer keeps returning ResumeFromDiskError.
+			if logBuffer.IsStopping() {
+				isDone = true
+				return
+			}
+
+			// Continue to next iteration (don't return ResumeFromDiskError)
+			continue
+		}
+		if err != nil {
+			// Check for buffer corruption error
+			if errors.Is(err, ErrBufferCorrupted) {
+				glog.Errorf("%s: Buffer corruption detected: %v", readerName, err)
+				return lastReadPosition, true, fmt.Errorf("buffer corruption: %w", err)
+			}
+			// Other errors
+			glog.Errorf("%s: ReadFromBuffer error: %v", readerName, err)
+			return lastReadPosition, true, err
+		}
+		readSize := 0
+		if bytesBuf != nil {
+			readSize = bytesBuf.Len()
+		}
+		glog.V(4).Infof("%s ReadFromBuffer at %v offset %d. Read bytes %v batchIndex %d", readerName, lastReadPosition, lastReadPosition.Offset, readSize, batchIndex)
+		if bytesBuf == nil {
+			if batchIndex >= 0 {
+				lastReadPosition = NewMessagePosition(lastReadPosition.Time.UnixNano(), batchIndex)
+			}
+			if stopTsNs != 0 {
+				isDone = true
+				return
+			}
+			lastTsNs := logBuffer.LastTsNs.Load()
+
+			for lastTsNs == logBuffer.LastTsNs.Load() {
+				if !waitForDataFn() {
+					isDone = true
+					return
+				}
+				// Wait for notification or timeout (instant wake-up when data arrives)
+				if logBuffer.awaitNotificationOrTimeout(notifyChan) {
+					glog.V(3).Infof("%s: Woke up from notification (LoopProcessLogData)", readerName)
+				} else if lastTsNs != logBuffer.LastTsNs.Load() {
+					break
+				} else {
+					glog.V(4).Infof("%s: Notification timeout (LoopProcessLogData), rechecking state", readerName)
+				}
+				// Exit the wait loop on shutdown so we don't spin against a
+				// closed shutdownCh.
+				if logBuffer.IsStopping() {
+					isDone = true
+					return
+				}
+			}
+			if logBuffer.IsStopping() {
+				isDone = true
+				return
+			}
+			continue
+		}
+
+		buf := bytesBuf.Bytes()
+		// fmt.Printf("ReadFromBuffer %s by %v size %d\n", readerName, lastReadPosition, len(buf))
+
+		batchSize := 0
+
+		for pos := 0; pos+4 < len(buf); {
+
+			size := util.BytesToUint32(buf[pos : pos+4])
+			if pos+4+int(size) > len(buf) {
+				err = ResumeError
+				glog.Errorf("LoopProcessLogData: %s read buffer %v read %d entries [%d,%d) from [0,%d)", readerName, lastReadPosition, batchSize, pos, pos+int(size)+4, len(buf))
+				return
+			}
+			entryData := buf[pos+4 : pos+4+int(size)]
+
+			logEntry := &filer_pb.LogEntry{}
+			// data/key alias entryData; valid only within eachLogDataFn below.
+			if err = unmarshalLogEntryAliased(entryData, logEntry); err != nil {
+				glog.Errorf("unexpected unmarshal mq_pb.Message: %v", err)
+				pos += 4 + int(size)
+				continue
+			}
+
+			// Handle offset-based filtering for offset-based start positions
+			if startPosition.IsOffsetBased {
+				startOffset := startPosition.GetOffset()
+				if logEntry.Offset < startOffset {
+					// Skip entries before the starting offset
+					pos += 4 + int(size)
+					batchSize++
+					continue
+				}
+			}
+
+			if stopTsNs != 0 && logEntry.TsNs > stopTsNs {
+				isDone = true
+				// println("stopTsNs", stopTsNs, "logEntry.TsNs", logEntry.TsNs)
+				return
+			}
+			lastReadPosition = NewMessagePosition(logEntry.TsNs, batchIndex)
+
+			if isDone, err = eachLogDataFn(logEntry); err != nil {
+				if !errors.Is(err, StopReadingError) {
+					glog.Errorf("LoopProcessLogData: %s process log entry %d key:%q ts_ns:%d offset:%d size:%d: %v", readerName, batchSize+1, logEntry.Key, logEntry.TsNs, logEntry.Offset, len(logEntry.Data), err)
+				}
+				return
+			}
+			if isDone {
+				glog.V(0).Infof("LoopProcessLogData2: %s process log entry %d", readerName, batchSize+1)
+				return
+			}
+
+			pos += 4 + int(size)
+			batchSize++
+			entryCounter++
+
+		}
+
+		glog.V(4).Infof("%s sent messages ts[%+v,%+v] size %d\n", readerName, startPosition, lastReadPosition, batchSize)
+	}
+
+}
+
+// LoopProcessLogDataWithOffset is similar to LoopProcessLogData but provides offset to the callback
+func (logBuffer *LogBuffer) LoopProcessLogDataWithOffset(readerName string, startPosition MessagePosition, stopTsNs int64,
+	waitForDataFn func() bool, eachLogDataFn EachLogEntryWithOffsetFuncType) (lastReadPosition MessagePosition, isDone bool, err error) {
+	glog.V(4).Infof("LoopProcessLogDataWithOffset started for %s, startPosition=%v", readerName, startPosition)
+
+	// Register for instant notifications (<1ms latency)
+	notifyChan := logBuffer.RegisterSubscriber(readerName)
+	defer logBuffer.UnregisterSubscriber(readerName)
+
+	// loop through all messages
+	var bytesBuf *bytes.Buffer
+	var bytesBufPooled bool
+	var offset int64
+	lastReadPosition = startPosition
+	var entryCounter int64
+	// See LoopProcessLogData for the caughtUpToDiskHead invariant.
+	caughtUpToDiskHead := false
+	defer func() {
+		if bytesBuf != nil && bytesBufPooled {
+			logBuffer.ReleaseMemory(bytesBuf)
+		}
+		// println("LoopProcessLogDataWithOffset", readerName, "sent messages total", entryCounter)
+	}()
+
+	for {
+		// Check stopTsNs at the beginning of each iteration
+		// This ensures we exit immediately if the stop time is in the past
+		if stopTsNs != 0 && time.Now().UnixNano() > stopTsNs {
+			isDone = true
+			return
+		}
+
+		if bytesBuf != nil && bytesBufPooled {
+			logBuffer.ReleaseMemory(bytesBuf)
+			bytesBuf = nil // keep the deferred release from double-freeing if ReadFromBuffer panics
+		}
+		bytesBuf, offset, bytesBufPooled, err = logBuffer.ReadFromBuffer(lastReadPosition)
+		glog.V(4).Infof("ReadFromBuffer for %s returned bytesBuf=%v, offset=%d, err=%v", readerName, bytesBuf != nil, offset, err)
+
+		// Check for buffer corruption error before other error handling
+		if err != nil && errors.Is(err, ErrBufferCorrupted) {
+			glog.Errorf("%s: Buffer corruption detected: %v", readerName, err)
+			return lastReadPosition, true, fmt.Errorf("buffer corruption: %w", err)
+		}
+
+		if err == ResumeFromDiskError {
+			// Try to read from disk if readFromDiskFn is available
+			if logBuffer.ReadFromDiskFn != nil {
+				prevReadPosition := lastReadPosition
+				// Wrap eachLogDataFn to match the expected signature
+				diskReadFn := func(logEntry *filer_pb.LogEntry) (bool, error) {
+					return eachLogDataFn(logEntry, logEntry.Offset)
+				}
+				lastReadPosition, isDone, err = logBuffer.ReadFromDiskFn(lastReadPosition, stopTsNs, diskReadFn)
+				if err != nil {
+					return lastReadPosition, isDone, err
+				}
+				if isDone {
+					return lastReadPosition, isDone, nil
+				}
+				if lastReadPosition != prevReadPosition {
+					caughtUpToDiskHead = false
+					continue
+				}
+				if !caughtUpToDiskHead {
+					caughtUpToDiskHead = true
+					glog.V(4).Infof("%s: Caught up to disk head, backing off to %s poll", readerName, caughtUpDiskPollInterval)
+				}
+			} else if logBuffer.HasData() {
+				// HasData() and ReadFromBuffer lock separately, so a racing write can make
+				// HasData() see data the empty-buffer read missed. Re-read; only bail if the
+				// position is genuinely behind the in-memory window (flushed to disk).
+				reBuf, _, rePooled, reErr := logBuffer.ReadFromBuffer(lastReadPosition)
+				if reErr == ResumeFromDiskError {
+					return lastReadPosition, isDone, ResumeFromDiskError
+				}
+				if reBuf != nil && rePooled {
+					logBuffer.ReleaseMemory(reBuf)
+				}
+				continue
+			}
+
+			// CRITICAL: Check if client is still connected after disk read
+			if !waitForDataFn() {
+				// Client disconnected - exit cleanly
+				glog.V(4).Infof("%s: Client disconnected after disk read", readerName)
+				return lastReadPosition, true, nil
+			}
+
+			// Wait for notification or timeout. While caught up to disk head,
+			// use the longer poll interval so idle readers don't spin every
+			// 250ms re-probing an empty buffer and disk.
+			waitTimeout := notificationHealthCheckInterval
+			if caughtUpToDiskHead {
+				waitTimeout = caughtUpDiskPollInterval
+			}
+			if logBuffer.awaitNotificationOrTimeoutFor(notifyChan, waitTimeout) {
+				glog.V(3).Infof("%s: Woke up from notification after disk read", readerName)
+				caughtUpToDiskHead = false
+			}
+			// Silent on timeout while caught up: the re-probe cadence alone is
+			// the signal; no new log line per tick.
+
+			// Exit cleanly on shutdown so we don't loop on ResumeFromDiskError.
+			if logBuffer.IsStopping() {
+				return lastReadPosition, true, nil
+			}
+
+			// Continue to next iteration (don't return ResumeFromDiskError)
+			continue
+		}
+		readSize := 0
+		if bytesBuf != nil {
+			readSize = bytesBuf.Len()
+		}
+		glog.V(4).Infof("%s ReadFromBuffer at %v posOffset %d. Read bytes %v bufferOffset %d", readerName, lastReadPosition, lastReadPosition.Offset, readSize, offset)
+		if bytesBuf == nil {
+			// CRITICAL: Check if subscription is still active BEFORE waiting
+			// This prevents infinite loops when client has disconnected
+			if !waitForDataFn() {
+				glog.V(4).Infof("%s: waitForDataFn returned false, subscription ending", readerName)
+				return lastReadPosition, true, nil
+			}
+
+			if offset >= 0 {
+				lastReadPosition = NewMessagePosition(lastReadPosition.Time.UnixNano(), offset)
+			}
+			if stopTsNs != 0 {
+				isDone = true
+				return
+			}
+
+			// If we're reading offset-based and there's no data in LogBuffer,
+			// return ResumeFromDiskError to let Subscribe try reading from disk again.
+			// This prevents infinite blocking when all data is on disk (e.g., after restart).
+			if startPosition.IsOffsetBased {
+				glog.V(4).Infof("%s: No data in LogBuffer for offset-based read at %v, checking if client still connected", readerName, lastReadPosition)
+				// Check if client is still connected before busy-looping
+				if !waitForDataFn() {
+					glog.V(4).Infof("%s: Client disconnected, stopping offset-based read", readerName)
+					return lastReadPosition, true, nil
+				}
+				// Wait for notification or timeout (instant wake-up when data arrives)
+				if logBuffer.awaitNotificationOrTimeout(notifyChan) {
+					glog.V(3).Infof("%s: Woke up from notification for offset-based read", readerName)
+				} else {
+					glog.V(4).Infof("%s: Notification timeout for offset-based, rechecking state", readerName)
+				}
+				// On shutdown, exit cleanly instead of returning ResumeFromDiskError.
+				if logBuffer.IsStopping() {
+					return lastReadPosition, true, nil
+				}
+				return lastReadPosition, isDone, ResumeFromDiskError
+			}
+
+			lastTsNs := logBuffer.LastTsNs.Load()
+
+			for lastTsNs == logBuffer.LastTsNs.Load() {
+				if !waitForDataFn() {
+					glog.V(4).Infof("%s: Client disconnected during timestamp wait", readerName)
+					return lastReadPosition, true, nil
+				}
+				// Wait for notification or timeout (instant wake-up when data arrives)
+				if logBuffer.awaitNotificationOrTimeout(notifyChan) {
+					glog.V(3).Infof("%s: Woke up from notification (main loop)", readerName)
+				} else if lastTsNs != logBuffer.LastTsNs.Load() {
+					break
+				} else {
+					glog.V(4).Infof("%s: Notification timeout (main loop), rechecking state", readerName)
+				}
+				// Exit the wait loop on shutdown so we don't spin against a
+				// closed shutdownCh.
+				if logBuffer.IsStopping() {
+					glog.V(4).Infof("%s: LogBuffer is stopping", readerName)
+					return lastReadPosition, true, nil
+				}
+			}
+			if logBuffer.IsStopping() {
+				glog.V(4).Infof("%s: LogBuffer is stopping", readerName)
+				return lastReadPosition, true, nil
+			}
+			continue
+		}
+
+		buf := bytesBuf.Bytes()
+		// fmt.Printf("ReadFromBuffer %s by %v size %d\n", readerName, lastReadPosition, len(buf))
+		glog.V(4).Infof("Processing buffer with %d bytes for %s", len(buf), readerName)
+
+		// If buffer is empty, check if client is still connected before looping
+		if len(buf) == 0 {
+			glog.V(4).Infof("Empty buffer for %s, checking if client still connected", readerName)
+			if !waitForDataFn() {
+				glog.V(4).Infof("%s: Client disconnected on empty buffer", readerName)
+				return lastReadPosition, true, nil
+			}
+			if logBuffer.awaitNotificationOrTimeout(notifyChan) {
+				glog.V(3).Infof("%s: Woke up from notification on empty buffer", readerName)
+			} else {
+				glog.V(4).Infof("%s: Empty buffer timeout, rechecking state", readerName)
+			}
+			// Exit cleanly on shutdown to avoid an idle spin on the empty buffer.
+			if logBuffer.IsStopping() {
+				return lastReadPosition, true, nil
+			}
+			continue
+		}
+
+		batchSize := 0
+
+		for pos := 0; pos+4 < len(buf); {
+
+			size := util.BytesToUint32(buf[pos : pos+4])
+			if pos+4+int(size) > len(buf) {
+				err = ResumeError
+				glog.Errorf("LoopProcessLogDataWithOffset: %s read buffer %v read %d entries [%d,%d) from [0,%d)", readerName, lastReadPosition, batchSize, pos, pos+int(size)+4, len(buf))
+				return
+			}
+			entryData := buf[pos+4 : pos+4+int(size)]
+
+			logEntry := &filer_pb.LogEntry{}
+			// data/key alias entryData; valid only within eachLogDataFn below.
+			if err = unmarshalLogEntryAliased(entryData, logEntry); err != nil {
+				glog.Errorf("unexpected unmarshal mq_pb.Message: %v", err)
+				pos += 4 + int(size)
+				continue
+			}
+
+			glog.V(4).Infof("Unmarshaled log entry %d: TsNs=%d, Offset=%d, Key=%s", batchSize+1, logEntry.TsNs, logEntry.Offset, string(logEntry.Key))
+
+			// Handle offset-based filtering for offset-based start positions
+			if startPosition.IsOffsetBased {
+				startOffset := startPosition.GetOffset()
+				glog.V(4).Infof("Offset-based filtering: logEntry.Offset=%d, startOffset=%d", logEntry.Offset, startOffset)
+				if logEntry.Offset < startOffset {
+					// Skip entries before the starting offset
+					glog.V(4).Infof("Skipping entry due to offset filter")
+					pos += 4 + int(size)
+					batchSize++
+					continue
+				}
+			}
+
+			if stopTsNs != 0 && logEntry.TsNs > stopTsNs {
+				glog.V(4).Infof("Stopping due to stopTsNs")
+				isDone = true
+				// println("stopTsNs", stopTsNs, "logEntry.TsNs", logEntry.TsNs)
+				return
+			}
+			// Use logEntry.Offset + 1 to move PAST the current entry
+			// This prevents infinite loops where we keep requesting the same offset
+			lastReadPosition = NewMessagePosition(logEntry.TsNs, logEntry.Offset+1)
+
+			glog.V(4).Infof("Calling eachLogDataFn for entry at offset %d, next position will be %d", logEntry.Offset, logEntry.Offset+1)
+			if isDone, err = eachLogDataFn(logEntry, logEntry.Offset); err != nil {
+				if !errors.Is(err, StopReadingError) {
+					glog.Errorf("LoopProcessLogDataWithOffset: %s process log entry %d key:%q ts_ns:%d offset:%d size:%d: %v", readerName, batchSize+1, logEntry.Key, logEntry.TsNs, logEntry.Offset, len(logEntry.Data), err)
+				}
+				return
+			}
+			if isDone {
+				glog.V(0).Infof("LoopProcessLogDataWithOffset: %s process log entry %d", readerName, batchSize+1)
+				return
+			}
+
+			pos += 4 + int(size)
+			batchSize++
+			entryCounter++
+
+		}
+
+	}
+
+}

weed/util/log_buffer/log_read_integration_test.go

@@ -0,0 +1,363 @@
+package log_buffer
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+// TestConcurrentProducerConsumer simulates the integration test scenario:
+// - One producer writing messages continuously
+// - Multiple consumers reading from different offsets
+// - Consumers reading sequentially (like Kafka consumers)
+func TestConcurrentProducerConsumer(t *testing.T) {
+	lb := NewLogBuffer("integration-test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	const numMessages = 1000
+	const numConsumers = 2
+	const messagesPerConsumer = numMessages / numConsumers
+
+	// Start producer
+	producerDone := make(chan bool)
+	go func() {
+		for i := 0; i < numMessages; i++ {
+			entry := &filer_pb.LogEntry{
+				TsNs:   time.Now().UnixNano(),
+				Key:    []byte("key"),
+				Data:   []byte("value"),
+				Offset: int64(i),
+			}
+			if err := lb.AddLogEntryToBuffer(entry); err != nil {
+				t.Errorf("Failed to add log entry: %v", err)
+				return
+			}
+			time.Sleep(1 * time.Millisecond) // Simulate production rate
+		}
+		producerDone <- true
+	}()
+
+	// Start consumers
+	consumerWg := sync.WaitGroup{}
+	consumerErrors := make(chan error, numConsumers)
+	consumedCounts := make([]int64, numConsumers)
+
+	for consumerID := 0; consumerID < numConsumers; consumerID++ {
+		consumerWg.Add(1)
+		go func(id int, startOffset int64, endOffset int64) {
+			defer consumerWg.Done()
+
+			currentOffset := startOffset
+			for currentOffset < endOffset {
+				// Read 10 messages at a time (like integration test)
+				messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(currentOffset, 10, 10240)
+				if err != nil {
+					consumerErrors <- err
+					return
+				}
+
+				if len(messages) == 0 {
+					// No data yet, wait a bit
+					time.Sleep(5 * time.Millisecond)
+					continue
+				}
+
+				// Count only messages in this consumer's assigned range
+				messagesInRange := 0
+				for i, msg := range messages {
+					if msg.Offset >= startOffset && msg.Offset < endOffset {
+						messagesInRange++
+						expectedOffset := currentOffset + int64(i)
+						if msg.Offset != expectedOffset {
+							t.Errorf("Consumer %d: Expected offset %d, got %d", id, expectedOffset, msg.Offset)
+						}
+					}
+				}
+
+				atomic.AddInt64(&consumedCounts[id], int64(messagesInRange))
+				currentOffset = nextOffset
+			}
+		}(consumerID, int64(consumerID*messagesPerConsumer), int64((consumerID+1)*messagesPerConsumer))
+	}
+
+	// Wait for producer to finish
+	<-producerDone
+
+	// Wait for consumers (with timeout)
+	done := make(chan bool)
+	go func() {
+		consumerWg.Wait()
+		done <- true
+	}()
+
+	select {
+	case <-done:
+		// Success
+	case err := <-consumerErrors:
+		t.Fatalf("Consumer error: %v", err)
+	case <-time.After(10 * time.Second):
+		t.Fatal("Timeout waiting for consumers to finish")
+	}
+
+	// Verify all messages were consumed
+	totalConsumed := int64(0)
+	for i, count := range consumedCounts {
+		t.Logf("Consumer %d consumed %d messages", i, count)
+		totalConsumed += count
+	}
+
+	if totalConsumed != numMessages {
+		t.Errorf("Expected to consume %d messages, but consumed %d", numMessages, totalConsumed)
+	}
+}
+
+// TestBackwardSeeksWhileProducing simulates consumer rebalancing where
+// consumers seek backward to earlier offsets while producer is still writing
+func TestBackwardSeeksWhileProducing(t *testing.T) {
+	lb := NewLogBuffer("backward-seek-test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	const numMessages = 500
+	const numSeeks = 10
+
+	// Start producer
+	producerDone := make(chan bool)
+	go func() {
+		for i := 0; i < numMessages; i++ {
+			entry := &filer_pb.LogEntry{
+				TsNs:   time.Now().UnixNano(),
+				Key:    []byte("key"),
+				Data:   []byte("value"),
+				Offset: int64(i),
+			}
+			if err := lb.AddLogEntryToBuffer(entry); err != nil {
+				t.Errorf("Failed to add log entry: %v", err)
+				return
+			}
+			time.Sleep(1 * time.Millisecond)
+		}
+		producerDone <- true
+	}()
+
+	// Consumer that seeks backward periodically
+	consumerDone := make(chan bool)
+	readOffsets := make(map[int64]int) // Track how many times each offset was read
+
+	go func() {
+		currentOffset := int64(0)
+		seeksRemaining := numSeeks
+
+		for currentOffset < numMessages {
+			// Read some messages
+			messages, nextOffset, _, endOfPartition, err := lb.ReadMessagesAtOffset(currentOffset, 10, 10240)
+			if err != nil {
+				// For stateless reads, "offset out of range" means data not in memory yet
+				// This is expected when reading historical data or before production starts
+				time.Sleep(5 * time.Millisecond)
+				continue
+			}
+
+			if len(messages) == 0 {
+				// No data available yet or caught up to producer
+				if !endOfPartition {
+					// Data might be coming, wait
+					time.Sleep(5 * time.Millisecond)
+				} else {
+					// At end of partition, wait for more production
+					time.Sleep(5 * time.Millisecond)
+				}
+				continue
+			}
+
+			// Track read offsets
+			for _, msg := range messages {
+				readOffsets[msg.Offset]++
+			}
+
+			// Periodically seek backward (simulating rebalancing)
+			if seeksRemaining > 0 && nextOffset > 50 && nextOffset%100 == 0 {
+				seekOffset := nextOffset - 20
+				t.Logf("Seeking backward from %d to %d", nextOffset, seekOffset)
+				currentOffset = seekOffset
+				seeksRemaining--
+			} else {
+				currentOffset = nextOffset
+			}
+		}
+
+		consumerDone <- true
+	}()
+
+	// Wait for both
+	<-producerDone
+	<-consumerDone
+
+	// Verify each offset was read at least once
+	for i := int64(0); i < numMessages; i++ {
+		if readOffsets[i] == 0 {
+			t.Errorf("Offset %d was never read", i)
+		}
+	}
+
+	t.Logf("Total unique offsets read: %d out of %d", len(readOffsets), numMessages)
+}
+
+// TestHighConcurrencyReads simulates multiple consumers reading from
+// different offsets simultaneously (stress test)
+func TestHighConcurrencyReads(t *testing.T) {
+	lb := NewLogBuffer("high-concurrency-test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	const numMessages = 1000
+	const numReaders = 10
+
+	// Pre-populate buffer
+	for i := 0; i < numMessages; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		if err := lb.AddLogEntryToBuffer(entry); err != nil {
+			t.Fatalf("Failed to add log entry: %v", err)
+		}
+	}
+
+	// Start many concurrent readers at different offsets
+	wg := sync.WaitGroup{}
+	errors := make(chan error, numReaders)
+
+	for reader := 0; reader < numReaders; reader++ {
+		wg.Add(1)
+		go func(startOffset int64) {
+			defer wg.Done()
+
+			// Read 100 messages from this offset
+			currentOffset := startOffset
+			readCount := 0
+
+			for readCount < 100 && currentOffset < numMessages {
+				messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(currentOffset, 10, 10240)
+				if err != nil {
+					errors <- err
+					return
+				}
+
+				// Verify offsets are sequential
+				for i, msg := range messages {
+					expected := currentOffset + int64(i)
+					if msg.Offset != expected {
+						t.Errorf("Reader at %d: expected offset %d, got %d", startOffset, expected, msg.Offset)
+					}
+				}
+
+				readCount += len(messages)
+				currentOffset = nextOffset
+			}
+		}(int64(reader * 10))
+	}
+
+	// Wait with timeout
+	done := make(chan bool)
+	go func() {
+		wg.Wait()
+		done <- true
+	}()
+
+	select {
+	case <-done:
+		// Success
+	case err := <-errors:
+		t.Fatalf("Reader error: %v", err)
+	case <-time.After(10 * time.Second):
+		t.Fatal("Timeout waiting for readers")
+	}
+}
+
+// TestRepeatedReadsAtSameOffset simulates what happens when Kafka
+// consumer re-fetches the same offset multiple times (due to timeouts or retries)
+func TestRepeatedReadsAtSameOffset(t *testing.T) {
+	lb := NewLogBuffer("repeated-reads-test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	const numMessages = 100
+
+	// Pre-populate buffer
+	for i := 0; i < numMessages; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		if err := lb.AddLogEntryToBuffer(entry); err != nil {
+			t.Fatalf("Failed to add log entry: %v", err)
+		}
+	}
+
+	// Read the same offset multiple times concurrently
+	const numReads = 10
+	const testOffset = int64(50)
+
+	wg := sync.WaitGroup{}
+	results := make([][]*filer_pb.LogEntry, numReads)
+
+	for i := 0; i < numReads; i++ {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			messages, _, _, _, err := lb.ReadMessagesAtOffset(testOffset, 10, 10240)
+			if err != nil {
+				t.Errorf("Read %d error: %v", idx, err)
+				return
+			}
+			results[idx] = messages
+		}(i)
+	}
+
+	wg.Wait()
+
+	// Verify all reads returned the same data
+	firstRead := results[0]
+	for i := 1; i < numReads; i++ {
+		if len(results[i]) != len(firstRead) {
+			t.Errorf("Read %d returned %d messages, expected %d", i, len(results[i]), len(firstRead))
+		}
+
+		for j := range results[i] {
+			if results[i][j].Offset != firstRead[j].Offset {
+				t.Errorf("Read %d message %d has offset %d, expected %d",
+					i, j, results[i][j].Offset, firstRead[j].Offset)
+			}
+		}
+	}
+}
+
+// TestEmptyPartitionPolling simulates consumers polling empty partitions
+// waiting for data (common in Kafka)
+func TestEmptyPartitionPolling(t *testing.T) {
+	lb := NewLogBuffer("empty-partition-test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+	lb.bufferStartOffset = 0
+	lb.offset = 0
+
+	// Try to read from empty partition
+	messages, nextOffset, _, endOfPartition, err := lb.ReadMessagesAtOffset(0, 10, 10240)
+
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	if len(messages) != 0 {
+		t.Errorf("Expected 0 messages, got %d", len(messages))
+	}
+	if nextOffset != 0 {
+		t.Errorf("Expected nextOffset=0, got %d", nextOffset)
+	}
+	if !endOfPartition {
+		t.Error("Expected endOfPartition=true for future offset")
+	}
+}

weed/util/log_buffer/log_read_stateless.go

@@ -0,0 +1,524 @@
+package log_buffer
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/proto"
+)
+
+// ReadMessagesAtOffset provides Kafka-style stateless reads from LogBuffer
+// Each call is completely independent - no state maintained between calls
+// Thread-safe for concurrent reads at different offsets
+//
+// This is the recommended API for stateless clients like Kafka gateway
+// Unlike Subscribe loops, this:
+// 1. Returns immediately with available data (or empty if none)
+// 2. Does not maintain any session state
+// 3. Safe for concurrent calls
+// 4. No cancellation/restart complexity
+//
+// Returns:
+// - messages: Array of messages starting at startOffset
+// - nextOffset: Offset to use for next fetch
+// - highWaterMark: Highest offset available in partition
+// - endOfPartition: True if no more data available
+// - err: Any error encountered
+func (logBuffer *LogBuffer) ReadMessagesAtOffset(startOffset int64, maxMessages int, maxBytes int) (
+	messages []*filer_pb.LogEntry,
+	nextOffset int64,
+	highWaterMark int64,
+	endOfPartition bool,
+	err error,
+) {
+	// Quick validation
+	if maxMessages <= 0 {
+		maxMessages = 100 // Default reasonable batch size
+	}
+	if maxBytes <= 0 {
+		maxBytes = 4 * 1024 * 1024 // 4MB default
+	}
+
+	messages = make([]*filer_pb.LogEntry, 0, maxMessages)
+	nextOffset = startOffset
+
+	// Try to read from in-memory buffers first (hot path)
+	logBuffer.RLock()
+	currentBufferEnd := logBuffer.offset
+	bufferStartOffset := logBuffer.bufferStartOffset
+	highWaterMark = currentBufferEnd
+
+	// Special case: empty buffer (no data written yet)
+	if currentBufferEnd == 0 && bufferStartOffset == 0 && logBuffer.pos == 0 {
+		logBuffer.RUnlock()
+		// Return empty result - partition exists but has no data yet
+		// Preserve the requested offset in nextOffset
+		return messages, startOffset, 0, true, nil
+	}
+
+	// Check if requested offset is in current buffer
+	if startOffset >= bufferStartOffset && startOffset < currentBufferEnd {
+		// Read from current buffer
+		glog.V(4).Infof("[StatelessRead] Reading from current buffer: start=%d, end=%d",
+			bufferStartOffset, currentBufferEnd)
+
+		if logBuffer.pos > 0 {
+			// Make a copy of the buffer to avoid concurrent modification
+			bufCopy := make([]byte, logBuffer.pos)
+			copy(bufCopy, logBuffer.buf[:logBuffer.pos])
+			logBuffer.RUnlock() // Release lock early
+
+			// Parse messages from buffer copy
+			messages, nextOffset, _, err = parseMessagesFromBuffer(
+				bufCopy, startOffset, maxMessages, maxBytes)
+
+			if err != nil {
+				return nil, startOffset, highWaterMark, false, err
+			}
+
+			glog.V(4).Infof("[StatelessRead] Read %d messages from current buffer, nextOffset=%d",
+				len(messages), nextOffset)
+
+			// Check if we reached the end
+			endOfPartition = (nextOffset >= currentBufferEnd) && (len(messages) == 0 || len(messages) < maxMessages)
+			return messages, nextOffset, highWaterMark, endOfPartition, nil
+		}
+
+		// Buffer is empty but offset is in range - check previous buffers
+		logBuffer.RUnlock()
+
+		// Try previous buffers
+		logBuffer.RLock()
+		for _, prevBuf := range logBuffer.prevBuffers.buffers {
+			if startOffset >= prevBuf.startOffset && startOffset <= prevBuf.offset {
+				if prevBuf.size > 0 {
+					// Found in previous buffer
+					bufCopy := make([]byte, prevBuf.size)
+					copy(bufCopy, prevBuf.buf[:prevBuf.size])
+					logBuffer.RUnlock()
+
+					messages, nextOffset, _, err = parseMessagesFromBuffer(
+						bufCopy, startOffset, maxMessages, maxBytes)
+
+					if err != nil {
+						return nil, startOffset, highWaterMark, false, err
+					}
+
+					glog.V(4).Infof("[StatelessRead] Read %d messages from previous buffer, nextOffset=%d",
+						len(messages), nextOffset)
+
+					endOfPartition = false // More data might be in current buffer
+					return messages, nextOffset, highWaterMark, endOfPartition, nil
+				}
+				// Empty previous buffer means data was flushed to disk - fall through to disk read
+				glog.V(2).Infof("[StatelessRead] Data at offset %d was flushed, attempting disk read", startOffset)
+				break
+			}
+		}
+		logBuffer.RUnlock()
+
+		// Data not in memory - attempt disk read if configured
+		// Don't return error here - data may be on disk!
+		// Fall through to disk read logic below
+		glog.V(2).Infof("[StatelessRead] Data at offset %d not in memory (buffer: %d-%d), attempting disk read",
+			startOffset, bufferStartOffset, currentBufferEnd)
+		// Don't return error - continue to disk read check below
+	} else {
+		// Offset is not in current buffer - check previous buffers FIRST before going to disk
+		// This handles the case where data was just flushed but is still in prevBuffers
+
+		for _, prevBuf := range logBuffer.prevBuffers.buffers {
+			if startOffset >= prevBuf.startOffset && startOffset <= prevBuf.offset {
+				if prevBuf.size > 0 {
+					// Found in previous buffer!
+					bufCopy := make([]byte, prevBuf.size)
+					copy(bufCopy, prevBuf.buf[:prevBuf.size])
+					logBuffer.RUnlock()
+
+					messages, nextOffset, _, err = parseMessagesFromBuffer(
+						bufCopy, startOffset, maxMessages, maxBytes)
+
+					if err != nil {
+						return nil, startOffset, highWaterMark, false, err
+					}
+
+					endOfPartition = false // More data might exist
+					return messages, nextOffset, highWaterMark, endOfPartition, nil
+				}
+				// Empty previous buffer - data was flushed to disk
+				glog.V(2).Infof("[StatelessRead] Found empty previous buffer for offset %d, will try disk", startOffset)
+				break
+			}
+		}
+		logBuffer.RUnlock()
+	}
+
+	// If we get here, unlock if not already unlocked
+	// (Note: logBuffer.RUnlock() was called above in all paths)
+
+	// Data not in memory - try disk read
+	// This handles two cases:
+	// 1. startOffset < bufferStartOffset: Historical data
+	// 2. startOffset in buffer range but not in memory: Data was flushed (from fall-through above)
+	if startOffset < currentBufferEnd {
+		// Historical data or flushed data - try to read from disk if ReadFromDiskFn is configured
+		if startOffset < bufferStartOffset {
+			glog.Errorf("[StatelessRead] CASE 1: Historical data - offset %d < bufferStart %d",
+				startOffset, bufferStartOffset)
+		} else {
+			glog.Errorf("[StatelessRead] CASE 2: Flushed data - offset %d in range [%d, %d) but not in memory",
+				startOffset, bufferStartOffset, currentBufferEnd)
+		}
+
+		// Check if disk read function is configured
+		if logBuffer.ReadFromDiskFn == nil {
+			glog.Errorf("[StatelessRead] CRITICAL: ReadFromDiskFn is NIL! Cannot read from disk.")
+			if startOffset < bufferStartOffset {
+				return messages, startOffset, highWaterMark, false, fmt.Errorf("offset %d too old (earliest in-memory: %d), and ReadFromDiskFn is nil",
+					startOffset, bufferStartOffset)
+			}
+			return messages, startOffset, highWaterMark, false, fmt.Errorf("offset %d not in memory (buffer: %d-%d), and ReadFromDiskFn is nil",
+				startOffset, bufferStartOffset, currentBufferEnd)
+		}
+
+		// Read from disk (this is async/non-blocking if the ReadFromDiskFn is properly implemented)
+		// The ReadFromDiskFn should handle its own timeouts and not block indefinitely
+		diskMessages, diskNextOffset, diskErr := readHistoricalDataFromDisk(
+			logBuffer, startOffset, maxMessages, maxBytes, highWaterMark)
+
+		if diskErr != nil {
+			glog.Errorf("[StatelessRead] CRITICAL: Disk read FAILED for offset %d: %v", startOffset, diskErr)
+			// IMPORTANT: Return retryable error instead of silently returning empty!
+			return messages, startOffset, highWaterMark, false, fmt.Errorf("disk read failed for offset %d: %v", startOffset, diskErr)
+		}
+
+		if len(diskMessages) == 0 {
+			glog.Errorf("[StatelessRead] WARNING: Disk read returned 0 messages for offset %d (HWM=%d, bufferStart=%d)",
+				startOffset, highWaterMark, bufferStartOffset)
+		}
+
+		// Return disk data
+		endOfPartition = diskNextOffset >= bufferStartOffset && len(diskMessages) < maxMessages
+		return diskMessages, diskNextOffset, highWaterMark, endOfPartition, nil
+	}
+
+	// startOffset >= currentBufferEnd - future offset, no data available yet
+	glog.V(4).Infof("[StatelessRead] Future offset %d >= buffer end %d, no data available",
+		startOffset, currentBufferEnd)
+	return messages, startOffset, highWaterMark, true, nil
+}
+
+// readHistoricalDataFromDisk reads messages from disk for historical offsets
+// This is called when the requested offset is older than what's in memory
+// Uses an in-memory cache to avoid repeated disk I/O for the same chunks
+func readHistoricalDataFromDisk(
+	logBuffer *LogBuffer,
+	startOffset int64,
+	maxMessages int,
+	maxBytes int,
+	highWaterMark int64,
+) (messages []*filer_pb.LogEntry, nextOffset int64, err error) {
+	const chunkSize = 1000 // Size of each cached chunk
+
+	// Calculate chunk start offset (aligned to chunkSize boundary)
+	chunkStartOffset := (startOffset / chunkSize) * chunkSize
+
+	// Try to get from cache first
+	cachedMessages, cacheHit := getCachedDiskChunk(logBuffer, chunkStartOffset)
+
+	if cacheHit {
+		// Found in cache - extract requested messages
+		result, nextOff, err := extractMessagesFromCache(cachedMessages, startOffset, maxMessages, maxBytes)
+
+		if err != nil {
+			// CRITICAL: Cache extraction failed because requested offset is BEYOND cached chunk
+			// This means disk files only contain partial data (e.g., 1000-1763) and the
+			// requested offset (e.g., 1764) is in a gap between disk and memory.
+			//
+			// SOLUTION: Return empty result with NO ERROR to let ReadMessagesAtOffset
+			// continue to check memory buffers. The data might be in memory even though
+			// it's not on disk.
+			glog.Errorf("[DiskCache] Offset %d is beyond cached chunk (start=%d, size=%d)",
+				startOffset, chunkStartOffset, len(cachedMessages))
+
+			// Return empty but NO ERROR - this signals "not on disk, try memory"
+			return nil, startOffset, nil
+		}
+
+		// Success - return cached data
+		return result, nextOff, nil
+	}
+
+	// Not in cache - read entire chunk from disk for caching
+	chunkMessages := make([]*filer_pb.LogEntry, 0, chunkSize)
+	chunkNextOffset := chunkStartOffset
+
+	// Create a position for the chunk start
+	chunkPosition := MessagePosition{
+		IsOffsetBased: true,
+		Offset:        chunkStartOffset,
+	}
+
+	// Define callback to collect the entire chunk
+	eachMessageFn := func(logEntry *filer_pb.LogEntry) (isDone bool, err error) {
+		// Read up to chunkSize messages for caching
+		if len(chunkMessages) >= chunkSize {
+			return true, nil
+		}
+
+		chunkMessages = append(chunkMessages, logEntry)
+		chunkNextOffset++
+
+		// Continue reading the chunk
+		return false, nil
+	}
+
+	// Read chunk from disk
+	_, _, readErr := logBuffer.ReadFromDiskFn(chunkPosition, 0, eachMessageFn)
+
+	if readErr != nil {
+		glog.Errorf("[DiskRead] CRITICAL: ReadFromDiskFn returned ERROR: %v", readErr)
+		return nil, startOffset, fmt.Errorf("failed to read from disk: %w", readErr)
+	}
+
+	// Cache the chunk for future reads
+	if len(chunkMessages) > 0 {
+		cacheDiskChunk(logBuffer, chunkStartOffset, chunkNextOffset-1, chunkMessages)
+	} else {
+		glog.Errorf("[DiskRead] WARNING: ReadFromDiskFn returned 0 messages for chunkStart=%d", chunkStartOffset)
+	}
+
+	// Extract requested messages from the chunk
+	result, resNextOffset, resErr := extractMessagesFromCache(chunkMessages, startOffset, maxMessages, maxBytes)
+	return result, resNextOffset, resErr
+}
+
+// getCachedDiskChunk retrieves a cached disk chunk if available
+func getCachedDiskChunk(logBuffer *LogBuffer, chunkStartOffset int64) ([]*filer_pb.LogEntry, bool) {
+	logBuffer.diskChunkCache.mu.RLock()
+	defer logBuffer.diskChunkCache.mu.RUnlock()
+
+	if chunk, exists := logBuffer.diskChunkCache.chunks[chunkStartOffset]; exists {
+		// Update last access time
+		chunk.lastAccess = time.Now()
+		return chunk.messages, true
+	}
+
+	return nil, false
+}
+
+// cacheDiskChunk stores a disk chunk in the cache with LRU eviction
+func cacheDiskChunk(logBuffer *LogBuffer, startOffset, endOffset int64, messages []*filer_pb.LogEntry) {
+	logBuffer.diskChunkCache.mu.Lock()
+	defer logBuffer.diskChunkCache.mu.Unlock()
+
+	// Check if we need to evict old chunks (LRU policy)
+	if len(logBuffer.diskChunkCache.chunks) >= logBuffer.diskChunkCache.maxChunks {
+		// Find least recently used chunk
+		var oldestOffset int64
+		var oldestTime time.Time
+		first := true
+
+		for offset, chunk := range logBuffer.diskChunkCache.chunks {
+			if first || chunk.lastAccess.Before(oldestTime) {
+				oldestOffset = offset
+				oldestTime = chunk.lastAccess
+				first = false
+			}
+		}
+
+		// Evict oldest chunk
+		delete(logBuffer.diskChunkCache.chunks, oldestOffset)
+		glog.V(4).Infof("[DiskCache] Evicted chunk at offset %d (LRU)", oldestOffset)
+	}
+
+	// Store new chunk
+	logBuffer.diskChunkCache.chunks[startOffset] = &CachedDiskChunk{
+		startOffset: startOffset,
+		endOffset:   endOffset,
+		messages:    messages,
+		lastAccess:  time.Now(),
+	}
+}
+
+// extractMessagesFromCache extracts requested messages from a cached chunk
+// chunkMessages contains messages starting from the chunk's aligned start offset
+// We need to skip to the requested startOffset within the chunk
+func extractMessagesFromCache(chunkMessages []*filer_pb.LogEntry, startOffset int64, maxMessages, maxBytes int) ([]*filer_pb.LogEntry, int64, error) {
+	const chunkSize = 1000
+	chunkStartOffset := (startOffset / chunkSize) * chunkSize
+
+	// Calculate position within chunk
+	positionInChunk := int(startOffset - chunkStartOffset)
+
+	// Check if requested offset is within the chunk
+	if positionInChunk < 0 {
+		glog.Errorf("[DiskCache] CRITICAL: Requested offset %d is BEFORE chunk start %d (positionInChunk=%d < 0)",
+			startOffset, chunkStartOffset, positionInChunk)
+		return nil, startOffset, fmt.Errorf("offset %d before chunk start %d", startOffset, chunkStartOffset)
+	}
+
+	if positionInChunk >= len(chunkMessages) {
+		// Requested offset is beyond the cached chunk
+		// This happens when disk files only contain partial data
+		// The requested offset might be in the gap between disk and memory
+
+		// Return empty (data not on disk) - caller will check memory buffers
+		return nil, startOffset, nil
+	}
+
+	// Extract messages starting from the requested position
+	messages := make([]*filer_pb.LogEntry, 0, maxMessages)
+	nextOffset := startOffset
+	totalBytes := 0
+
+	for i := positionInChunk; i < len(chunkMessages) && len(messages) < maxMessages; i++ {
+		entry := chunkMessages[i]
+		entrySize := proto.Size(entry)
+
+		// Check byte limit
+		if totalBytes > 0 && totalBytes+entrySize > maxBytes {
+			break
+		}
+
+		messages = append(messages, entry)
+		totalBytes += entrySize
+		nextOffset++
+	}
+
+	glog.V(4).Infof("[DiskCache] Extracted %d messages from cache (offset %d-%d, bytes=%d)",
+		len(messages), startOffset, nextOffset-1, totalBytes)
+
+	return messages, nextOffset, nil
+}
+
+// parseMessagesFromBuffer parses messages from a buffer byte slice
+// This is thread-safe as it operates on a copy of the buffer
+func parseMessagesFromBuffer(buf []byte, startOffset int64, maxMessages int, maxBytes int) (
+	messages []*filer_pb.LogEntry,
+	nextOffset int64,
+	totalBytes int,
+	err error,
+) {
+	messages = make([]*filer_pb.LogEntry, 0, maxMessages)
+	nextOffset = startOffset
+	totalBytes = 0
+	foundStart := false
+
+	messagesInBuffer := 0
+	for pos := 0; pos+4 < len(buf) && len(messages) < maxMessages && totalBytes < maxBytes; {
+		// Read message size
+		size := util.BytesToUint32(buf[pos : pos+4])
+		if pos+4+int(size) > len(buf) {
+			// Incomplete message at end of buffer
+			glog.V(4).Infof("[parseMessages] Incomplete message at pos %d, size %d, bufLen %d",
+				pos, size, len(buf))
+			break
+		}
+
+		// Parse message
+		entryData := buf[pos+4 : pos+4+int(size)]
+		logEntry := &filer_pb.LogEntry{}
+		if err = logEntry.UnmarshalVT(entryData); err != nil {
+			glog.Warningf("[parseMessages] Failed to unmarshal message: %v", err)
+			pos += 4 + int(size)
+			continue
+		}
+
+		messagesInBuffer++
+
+		// Initialize foundStart from first message
+		if !foundStart {
+			// Find the first message at or after startOffset
+			if logEntry.Offset >= startOffset {
+				foundStart = true
+				nextOffset = logEntry.Offset
+			} else {
+				// Skip messages before startOffset
+				glog.V(3).Infof("[parseMessages] Skipping message at offset %d (before startOffset %d)", logEntry.Offset, startOffset)
+				pos += 4 + int(size)
+				continue
+			}
+		}
+
+		// Check if this message matches expected offset
+		if foundStart && logEntry.Offset >= startOffset {
+			glog.V(3).Infof("[parseMessages] Adding message at offset %d (count=%d)", logEntry.Offset, len(messages)+1)
+			messages = append(messages, logEntry)
+			totalBytes += 4 + int(size)
+			nextOffset = logEntry.Offset + 1
+		}
+
+		pos += 4 + int(size)
+	}
+
+	glog.V(4).Infof("[parseMessages] Parsed %d messages, nextOffset=%d, totalBytes=%d",
+		len(messages), nextOffset, totalBytes)
+
+	return messages, nextOffset, totalBytes, nil
+}
+
+// GetHighWaterMark returns the highest offset available in this partition
+// This is a lightweight operation for clients to check partition state
+func (logBuffer *LogBuffer) GetHighWaterMark() int64 {
+	logBuffer.RLock()
+	defer logBuffer.RUnlock()
+	return logBuffer.offset
+}
+
+// GetLogStartOffset returns the earliest offset available (either in memory or on disk)
+// This is useful for clients to know the valid offset range
+func (logBuffer *LogBuffer) GetLogStartOffset() int64 {
+	logBuffer.RLock()
+	defer logBuffer.RUnlock()
+
+	// Check if we have offset information
+	if !logBuffer.hasOffsets {
+		return 0
+	}
+
+	// Return the current buffer start offset - this is the earliest offset in memory RIGHT NOW
+	// For stateless fetch, we only return what's currently available in memory
+	// We don't check prevBuffers because they may be stale or getting flushed
+	return logBuffer.bufferStartOffset
+}
+
+// WaitForDataWithTimeout waits up to maxWaitMs for data to be available at startOffset
+// Returns true if data became available, false if timeout
+// This allows "long poll" behavior for real-time consumers
+func (logBuffer *LogBuffer) WaitForDataWithTimeout(startOffset int64, maxWaitMs int) bool {
+	if maxWaitMs <= 0 {
+		return false
+	}
+
+	timeout := time.NewTimer(time.Duration(maxWaitMs) * time.Millisecond)
+	defer timeout.Stop()
+
+	// Register for notifications
+	notifyChan := logBuffer.RegisterSubscriber(fmt.Sprintf("fetch-%d", startOffset))
+	defer logBuffer.UnregisterSubscriber(fmt.Sprintf("fetch-%d", startOffset))
+
+	// Check if data is already available
+	logBuffer.RLock()
+	currentEnd := logBuffer.offset
+	logBuffer.RUnlock()
+
+	if currentEnd >= startOffset {
+		return true
+	}
+
+	// Wait for notification or timeout
+	select {
+	case <-notifyChan:
+		// Data might be available now
+		logBuffer.RLock()
+		currentEnd := logBuffer.offset
+		logBuffer.RUnlock()
+		return currentEnd >= startOffset
+	case <-timeout.C:
+		return false
+	}
+}

weed/util/log_buffer/log_read_stateless_test.go

@@ -0,0 +1,390 @@
+package log_buffer
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func TestReadMessagesAtOffset_EmptyBuffer(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+	lb.bufferStartOffset = 0
+	lb.offset = 0 // Empty buffer
+
+	messages, nextOffset, hwm, endOfPartition, err := lb.ReadMessagesAtOffset(100, 10, 1024)
+
+	// Reading from future offset (100) when buffer is at 0
+	// Should return empty, no error
+	if err != nil {
+		t.Errorf("Expected no error for future offset, got %v", err)
+	}
+	if len(messages) != 0 {
+		t.Errorf("Expected 0 messages, got %d", len(messages))
+	}
+	if nextOffset != 100 {
+		t.Errorf("Expected nextOffset=100, got %d", nextOffset)
+	}
+	if !endOfPartition {
+		t.Error("Expected endOfPartition=true for future offset")
+	}
+	if hwm != 0 {
+		t.Errorf("Expected highWaterMark=0, got %d", hwm)
+	}
+}
+
+func TestReadMessagesAtOffset_SingleMessage(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add a message
+	entry := &filer_pb.LogEntry{
+		TsNs:   time.Now().UnixNano(),
+		Key:    []byte("key1"),
+		Data:   []byte("value1"),
+		Offset: 0,
+	}
+	if err := lb.AddLogEntryToBuffer(entry); err != nil {
+		t.Fatalf("Failed to add log entry to buffer: %v", err)
+	}
+
+	// Read from offset 0
+	messages, nextOffset, _, endOfPartition, err := lb.ReadMessagesAtOffset(0, 10, 1024)
+
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+	if len(messages) != 1 {
+		t.Errorf("Expected 1 message, got %d", len(messages))
+	}
+	if nextOffset != 1 {
+		t.Errorf("Expected nextOffset=1, got %d", nextOffset)
+	}
+	if !endOfPartition {
+		t.Error("Expected endOfPartition=true after reading all messages")
+	}
+	if messages[0].Offset != 0 {
+		t.Errorf("Expected message offset=0, got %d", messages[0].Offset)
+	}
+	if string(messages[0].Key) != "key1" {
+		t.Errorf("Expected key='key1', got '%s'", string(messages[0].Key))
+	}
+}
+
+func TestReadMessagesAtOffset_MultipleMessages(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add 5 messages
+	for i := 0; i < 5; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		if err := lb.AddLogEntryToBuffer(entry); err != nil {
+			t.Fatalf("Failed to add log entry to buffer: %v", err)
+		}
+	}
+
+	// Read from offset 0, max 3 messages
+	messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(0, 3, 10240)
+
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+	if len(messages) != 3 {
+		t.Errorf("Expected 3 messages, got %d", len(messages))
+	}
+	if nextOffset != 3 {
+		t.Errorf("Expected nextOffset=3, got %d", nextOffset)
+	}
+
+	// Verify offsets are sequential
+	for i, msg := range messages {
+		if msg.Offset != int64(i) {
+			t.Errorf("Message %d: expected offset=%d, got %d", i, i, msg.Offset)
+		}
+	}
+}
+
+func TestReadMessagesAtOffset_StartFromMiddle(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add 10 messages (0-9)
+	for i := 0; i < 10; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		if err := lb.AddLogEntryToBuffer(entry); err != nil {
+			t.Fatalf("Failed to add log entry to buffer: %v", err)
+		}
+	}
+
+	// Read from offset 5
+	messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(5, 3, 10240)
+
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+	if len(messages) != 3 {
+		t.Errorf("Expected 3 messages, got %d", len(messages))
+	}
+	if nextOffset != 8 {
+		t.Errorf("Expected nextOffset=8, got %d", nextOffset)
+	}
+
+	// Verify we got messages 5, 6, 7
+	expectedOffsets := []int64{5, 6, 7}
+	for i, msg := range messages {
+		if msg.Offset != expectedOffsets[i] {
+			t.Errorf("Message %d: expected offset=%d, got %d", i, expectedOffsets[i], msg.Offset)
+		}
+	}
+}
+
+func TestReadMessagesAtOffset_MaxBytesLimit(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add messages with 100 bytes each
+	for i := 0; i < 10; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   make([]byte, 100), // 100 bytes
+			Offset: int64(i),
+		}
+		if err := lb.AddLogEntryToBuffer(entry); err != nil {
+			t.Fatalf("Failed to add log entry to buffer: %v", err)
+		}
+	}
+
+	// Request with max 250 bytes (should get ~2 messages)
+	messages, _, _, _, err := lb.ReadMessagesAtOffset(0, 100, 250)
+
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+
+	// Should get at least 1 message, but likely 2
+	if len(messages) == 0 {
+		t.Error("Expected at least 1 message")
+	}
+	if len(messages) > 3 {
+		t.Errorf("Expected max 3 messages with 250 byte limit, got %d", len(messages))
+	}
+}
+
+func TestReadMessagesAtOffset_ConcurrentReads(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add 100 messages
+	for i := 0; i < 100; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		if err := lb.AddLogEntryToBuffer(entry); err != nil {
+			t.Fatalf("Failed to add log entry to buffer: %v", err)
+		}
+	}
+
+	// Start 10 concurrent readers at different offsets
+	done := make(chan bool, 10)
+
+	for reader := 0; reader < 10; reader++ {
+		startOffset := int64(reader * 10)
+		go func(offset int64) {
+			messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(offset, 5, 10240)
+
+			if err != nil {
+				t.Errorf("Reader at offset %d: unexpected error: %v", offset, err)
+			}
+			if len(messages) != 5 {
+				t.Errorf("Reader at offset %d: expected 5 messages, got %d", offset, len(messages))
+			}
+			if nextOffset != offset+5 {
+				t.Errorf("Reader at offset %d: expected nextOffset=%d, got %d", offset, offset+5, nextOffset)
+			}
+
+			// Verify sequential offsets
+			for i, msg := range messages {
+				expectedOffset := offset + int64(i)
+				if msg.Offset != expectedOffset {
+					t.Errorf("Reader at offset %d: message %d has offset %d, expected %d",
+						offset, i, msg.Offset, expectedOffset)
+				}
+			}
+
+			done <- true
+		}(startOffset)
+	}
+
+	// Wait for all readers
+	for i := 0; i < 10; i++ {
+		<-done
+	}
+}
+
+func TestReadMessagesAtOffset_FutureOffset(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add 5 messages (0-4)
+	for i := 0; i < 5; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		if err := lb.AddLogEntryToBuffer(entry); err != nil {
+			t.Fatalf("Failed to add log entry to buffer: %v", err)
+		}
+	}
+
+	// Try to read from offset 10 (future)
+	messages, nextOffset, _, endOfPartition, err := lb.ReadMessagesAtOffset(10, 10, 10240)
+
+	if err != nil {
+		t.Errorf("Expected no error for future offset, got %v", err)
+	}
+	if len(messages) != 0 {
+		t.Errorf("Expected 0 messages for future offset, got %d", len(messages))
+	}
+	if nextOffset != 10 {
+		t.Errorf("Expected nextOffset=10, got %d", nextOffset)
+	}
+	if !endOfPartition {
+		t.Error("Expected endOfPartition=true for future offset")
+	}
+}
+
+func TestWaitForDataWithTimeout_DataAvailable(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add message at offset 0
+	entry := &filer_pb.LogEntry{
+		TsNs:   time.Now().UnixNano(),
+		Key:    []byte("key"),
+		Data:   []byte("value"),
+		Offset: 0,
+	}
+	if err := lb.AddLogEntryToBuffer(entry); err != nil {
+		t.Fatalf("Failed to add log entry to buffer: %v", err)
+	}
+
+	// Wait for data at offset 0 (should return immediately)
+	dataAvailable := lb.WaitForDataWithTimeout(0, 100)
+
+	if !dataAvailable {
+		t.Error("Expected data to be available at offset 0")
+	}
+}
+
+func TestWaitForDataWithTimeout_NoData(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+	lb.bufferStartOffset = 0
+	lb.offset = 0
+
+	// Don't add any messages, wait for offset 10
+
+	// Wait for data at offset 10 with short timeout
+	start := time.Now()
+	dataAvailable := lb.WaitForDataWithTimeout(10, 50)
+	elapsed := time.Since(start)
+
+	if dataAvailable {
+		t.Error("Expected no data to be available")
+	}
+	// Note: Actual wait time may be shorter if subscriber mechanism
+	// returns immediately. Just verify no data was returned.
+	t.Logf("Waited %v for timeout", elapsed)
+}
+
+func TestWaitForDataWithTimeout_DataArrives(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Start waiting in background
+	done := make(chan bool)
+	var dataAvailable bool
+
+	go func() {
+		dataAvailable = lb.WaitForDataWithTimeout(0, 500)
+		done <- true
+	}()
+
+	// Add data after 50ms
+	time.Sleep(50 * time.Millisecond)
+	entry := &filer_pb.LogEntry{
+		TsNs:   time.Now().UnixNano(),
+		Key:    []byte("key"),
+		Data:   []byte("value"),
+		Offset: 0,
+	}
+	if err := lb.AddLogEntryToBuffer(entry); err != nil {
+		t.Fatalf("Failed to add log entry to buffer: %v", err)
+	}
+
+	// Wait for result
+	<-done
+
+	if !dataAvailable {
+		t.Error("Expected data to become available after being added")
+	}
+}
+
+func TestGetHighWaterMark(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Initially should be 0
+	hwm := lb.GetHighWaterMark()
+	if hwm != 0 {
+		t.Errorf("Expected initial HWM=0, got %d", hwm)
+	}
+
+	// Add messages (offsets 0-4)
+	for i := 0; i < 5; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		if err := lb.AddLogEntryToBuffer(entry); err != nil {
+			t.Fatalf("Failed to add log entry to buffer: %v", err)
+		}
+	}
+
+	// HWM should be 5 (next offset to write, not last written offset)
+	// This matches Kafka semantics where HWM = last offset + 1
+	hwm = lb.GetHighWaterMark()
+	if hwm != 5 {
+		t.Errorf("Expected HWM=5 after adding 5 messages (0-4), got %d", hwm)
+	}
+}
+
+func TestGetLogStartOffset(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+	lb.bufferStartOffset = 10
+
+	lso := lb.GetLogStartOffset()
+	if lso != 10 {
+		t.Errorf("Expected LSO=10, got %d", lso)
+	}
+}

weed/util/log_buffer/log_read_test.go

@@ -0,0 +1,664 @@
+package log_buffer
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+)
+
+// TestLoopProcessLogDataWithOffset_ClientDisconnect tests that the loop exits
+// when the client disconnects (waitForDataFn returns false)
+func TestLoopProcessLogDataWithOffset_ClientDisconnect(t *testing.T) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Simulate client disconnect after 100ms
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	waitForDataFn := func() bool {
+		select {
+		case <-ctx.Done():
+			return false // Client disconnected
+		default:
+			return true
+		}
+	}
+
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+		return true, nil
+	}
+
+	startPosition := NewMessagePositionFromOffset(0)
+	startTime := time.Now()
+
+	// This should exit within 200ms (100ms timeout + some buffer)
+	_, isDone, _ := logBuffer.LoopProcessLogDataWithOffset("test-client", startPosition, 0, waitForDataFn, eachLogEntryFn)
+
+	elapsed := time.Since(startTime)
+
+	if !isDone {
+		t.Errorf("Expected isDone=true when client disconnects, got false")
+	}
+
+	if elapsed > 500*time.Millisecond {
+		t.Errorf("Loop took too long to exit: %v (expected < 500ms)", elapsed)
+	}
+
+	t.Logf("Loop exited cleanly in %v after client disconnect", elapsed)
+}
+
+// TestLoopProcessLogDataWithOffset_EmptyBuffer tests that the loop doesn't
+// busy-wait when the buffer is empty
+func TestLoopProcessLogDataWithOffset_EmptyBuffer(t *testing.T) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	callCount := 0
+	maxCalls := 4
+	mu := sync.Mutex{}
+
+	waitForDataFn := func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		callCount++
+		// Disconnect after maxCalls to prevent infinite loop
+		return callCount < maxCalls
+	}
+
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+		return true, nil
+	}
+
+	startPosition := NewMessagePositionFromOffset(0)
+	startTime := time.Now()
+
+	_, isDone, _ := logBuffer.LoopProcessLogDataWithOffset("test-client", startPosition, 0, waitForDataFn, eachLogEntryFn)
+
+	elapsed := time.Since(startTime)
+
+	if !isDone {
+		t.Errorf("Expected isDone=true when waitForDataFn returns false, got false")
+	}
+
+	minExpectedTime := time.Duration(maxCalls-1) * notificationHealthCheckInterval
+	if elapsed < minExpectedTime {
+		t.Errorf("Loop exited too quickly (%v), expected at least %v (suggests busy-waiting)", elapsed, minExpectedTime)
+	}
+
+	maxExpectedTime := time.Duration(maxCalls+1) * notificationHealthCheckInterval
+	if elapsed > maxExpectedTime {
+		t.Errorf("Loop took too long: %v (expected < %v)", elapsed, maxExpectedTime)
+	}
+
+	mu.Lock()
+	finalCallCount := callCount
+	mu.Unlock()
+
+	if finalCallCount != maxCalls {
+		t.Errorf("Expected exactly %d calls to waitForDataFn, got %d", maxCalls, finalCallCount)
+	}
+
+	t.Logf("Loop exited cleanly in %v after %d iterations (no busy-waiting detected)", elapsed, finalCallCount)
+}
+
+// TestLoopProcessLogDataWithOffset_NoDataResumeFromDisk tests that the loop
+// properly handles ResumeFromDiskError without busy-waiting
+func TestLoopProcessLogDataWithOffset_NoDataResumeFromDisk(t *testing.T) {
+	readFromDiskFn := func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (lastReadPosition MessagePosition, isDone bool, err error) {
+		// No data on disk
+		return startPosition, false, nil
+	}
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, readFromDiskFn, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	callCount := 0
+	maxCalls := 3
+	mu := sync.Mutex{}
+
+	waitForDataFn := func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		callCount++
+		// Disconnect after maxCalls
+		return callCount < maxCalls
+	}
+
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+		return true, nil
+	}
+
+	startPosition := NewMessagePositionFromOffset(0)
+	startTime := time.Now()
+
+	_, isDone, _ := logBuffer.LoopProcessLogDataWithOffset("test-client", startPosition, 0, waitForDataFn, eachLogEntryFn)
+
+	elapsed := time.Since(startTime)
+
+	if !isDone {
+		t.Errorf("Expected isDone=true when waitForDataFn returns false, got false")
+	}
+
+	minExpectedTime := time.Duration(maxCalls-1) * notificationHealthCheckInterval
+	if elapsed < minExpectedTime {
+		t.Errorf("Loop exited too quickly (%v), expected at least %v (suggests missing wait)", elapsed, minExpectedTime)
+	}
+
+	t.Logf("Loop exited cleanly in %v after %d iterations (proper sleep detected)", elapsed, callCount)
+}
+
+// TestLoopFlush_NotifiesSubscribersAfterFlush is a regression test for the
+// issue #9007 fix: loopFlush must call notifySubscribers() after processing a
+// flush so that readers parked on notifyChan wake up when a flush lands. The
+// classic bug scenario is a reader that got ResumeFromDiskError, did a disk
+// read that raced the flush and found nothing, and is now blocked on
+// notifyChan waiting for the data that just hit disk.
+//
+// We drain the AddToBuffer notification first, then ForceFlush, and assert a
+// new notification is delivered on notifyChan well before the fallback
+// timeout. If the loopFlush notification is removed, this test fails by
+// hitting the fallback.
+func TestLoopFlush_NotifiesSubscribersAfterFlush(t *testing.T) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+	}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	notifyChan := logBuffer.RegisterSubscriber("flush-notify-test")
+	defer logBuffer.UnregisterSubscriber("flush-notify-test")
+
+	if err := logBuffer.AddToBuffer(&mq_pb.DataMessage{
+		Key:   []byte("k"),
+		Value: []byte("v"),
+		TsNs:  time.Now().UnixNano(),
+	}); err != nil {
+		t.Fatalf("AddToBuffer: %v", err)
+	}
+
+	// Consume the AddToBuffer notification so the channel starts empty.
+	select {
+	case <-notifyChan:
+	case <-time.After(100 * time.Millisecond):
+		t.Fatal("expected a notification from AddToBuffer")
+	}
+
+	// ForceFlush waits for loopFlush to process the flush. After it returns,
+	// loopFlush must have called notifySubscribers() again.
+	start := time.Now()
+	logBuffer.ForceFlush()
+
+	select {
+	case <-notifyChan:
+		elapsed := time.Since(start)
+		// The fallback timeout is notificationHealthCheckInterval; the flush
+		// notification should arrive well before that.
+		if elapsed >= notificationHealthCheckInterval {
+			t.Errorf("flush notification too slow: %v (>= fallback %v)", elapsed, notificationHealthCheckInterval)
+		}
+		t.Logf("flush notification delivered in %v", elapsed)
+	case <-time.After(notificationHealthCheckInterval):
+		t.Fatalf("loopFlush did not notify subscribers within %v", notificationHealthCheckInterval)
+	}
+}
+
+// TestLoopProcessLogDataWithOffset_WakesOnDataArrival drives a real
+// LoopProcessLogDataWithOffset reader from an empty buffer (readFromDiskFn
+// returns nothing, forcing the reader to park on notifyChan after the
+// ResumeFromDiskError branch), then adds data from another goroutine and
+// asserts the reader completes well before the fallback timeout would fire.
+// This protects the end-to-end wake-up path; the loopFlush-specific
+// notification is covered by TestLoopFlush_NotifiesSubscribersAfterFlush.
+func TestLoopProcessLogDataWithOffset_WakesOnDataArrival(t *testing.T) {
+	readFromDiskFn := func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+		// No data on disk; return unchanged so the reader parks on notifyChan.
+		return startPosition, false, nil
+	}
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+	}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, readFromDiskFn, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	received := make(chan struct{})
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+		close(received)
+		return true, nil // isDone
+	}
+
+	waitForDataFn := func() bool { return true }
+
+	startPosition := NewMessagePositionFromOffset(0)
+
+	readerDone := make(chan struct{})
+	go func() {
+		_, _, _ = logBuffer.LoopProcessLogDataWithOffset(
+			"wake-test", startPosition, 0, waitForDataFn, eachLogEntryFn)
+		close(readerDone)
+	}()
+
+	// Give the reader time to reach awaitNotificationOrTimeout. Both wake
+	// paths under test (notifyChan via AddToBuffer and shutdownCh via
+	// ShutdownLogBuffer) are race-free even if the reader hasn't parked yet
+	// — the notification stays buffered / shutdownCh stays closed — but a
+	// generous head start makes it likelier we exercise the actual park-then-
+	// wake path rather than the already-pending fast path. 50ms is well below
+	// notificationHealthCheckInterval (250ms) and tolerates slow CI.
+	time.Sleep(50 * time.Millisecond)
+
+	start := time.Now()
+	if err := logBuffer.AddToBuffer(&mq_pb.DataMessage{
+		Key:   []byte("k"),
+		Value: []byte("v"),
+		TsNs:  time.Now().UnixNano(),
+	}); err != nil {
+		t.Fatalf("AddToBuffer: %v", err)
+	}
+
+	select {
+	case <-received:
+	case <-time.After(notificationHealthCheckInterval):
+		t.Fatalf("reader did not process the entry within %v (fallback timeout)", notificationHealthCheckInterval)
+	}
+	<-readerDone
+	elapsed := time.Since(start)
+
+	if elapsed >= notificationHealthCheckInterval {
+		t.Errorf("reader wake too slow: %v (>= fallback %v)", elapsed, notificationHealthCheckInterval)
+	}
+	t.Logf("reader processed the entry in %v after AddToBuffer", elapsed)
+}
+
+// TestLoopProcessLogDataWithOffset_WakesOnShutdown verifies that a reader
+// parked inside awaitNotificationOrTimeout via the ResumeFromDiskError branch
+// exits promptly when ShutdownLogBuffer is called, without waiting for the
+// 250ms health-check fallback. Regression guard for the IsStopping() shutdown
+// path: if awaitNotificationOrTimeout returns true via shutdownCh and the
+// caller does not check IsStopping(), the reader either spins against the
+// closed shutdownCh or returns ResumeFromDiskError instead of exiting.
+func TestLoopProcessLogDataWithOffset_WakesOnShutdown(t *testing.T) {
+	readFromDiskFn := func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (MessagePosition, bool, error) {
+		// No data on disk; return unchanged so the reader parks on notifyChan.
+		return startPosition, false, nil
+	}
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+	}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, readFromDiskFn, nil)
+	// Note: not deferring ShutdownLogBuffer; we trigger it explicitly below.
+
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+		return false, nil
+	}
+	waitForDataFn := func() bool { return true }
+	startPosition := NewMessagePositionFromOffset(0)
+
+	type result struct {
+		isDone bool
+		err    error
+	}
+	resultCh := make(chan result, 1)
+	go func() {
+		_, isDone, err := logBuffer.LoopProcessLogDataWithOffset(
+			"shutdown-test", startPosition, 0, waitForDataFn, eachLogEntryFn)
+		resultCh <- result{isDone: isDone, err: err}
+	}()
+
+	// Give the reader time to reach awaitNotificationOrTimeout. Both wake
+	// paths under test (notifyChan via AddToBuffer and shutdownCh via
+	// ShutdownLogBuffer) are race-free even if the reader hasn't parked yet
+	// — the notification stays buffered / shutdownCh stays closed — but a
+	// generous head start makes it likelier we exercise the actual park-then-
+	// wake path rather than the already-pending fast path. 50ms is well below
+	// notificationHealthCheckInterval (250ms) and tolerates slow CI.
+	time.Sleep(50 * time.Millisecond)
+
+	start := time.Now()
+	logBuffer.ShutdownLogBuffer()
+
+	select {
+	case r := <-resultCh:
+		elapsed := time.Since(start)
+		if elapsed >= notificationHealthCheckInterval {
+			t.Errorf("reader did not wake on shutdown: %v (>= fallback %v)", elapsed, notificationHealthCheckInterval)
+		}
+		if !r.isDone {
+			t.Errorf("expected isDone=true on shutdown, got false")
+		}
+		if r.err != nil {
+			t.Errorf("expected err=nil on shutdown, got %v", r.err)
+		}
+		t.Logf("reader exited in %v after ShutdownLogBuffer", elapsed)
+	case <-time.After(2 * notificationHealthCheckInterval):
+		t.Fatalf("reader did not exit within %v after ShutdownLogBuffer", 2*notificationHealthCheckInterval)
+	}
+}
+
+// TestLoopProcessLogDataWithOffset_WithData tests normal operation with data
+func TestLoopProcessLogDataWithOffset_WithData(t *testing.T) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Add some test data to the buffer
+	testMessages := []*mq_pb.DataMessage{
+		{Key: []byte("key1"), Value: []byte("message1"), TsNs: 1},
+		{Key: []byte("key2"), Value: []byte("message2"), TsNs: 2},
+		{Key: []byte("key3"), Value: []byte("message3"), TsNs: 3},
+	}
+
+	for _, msg := range testMessages {
+		if err := logBuffer.AddToBuffer(msg); err != nil {
+			t.Fatalf("Failed to add message to buffer: %v", err)
+		}
+	}
+
+	receivedCount := 0
+	mu := sync.Mutex{}
+
+	// Disconnect after receiving at least 1 message to test that data processing works
+	waitForDataFn := func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		return receivedCount == 0 // Disconnect after first message
+	}
+
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+		mu.Lock()
+		receivedCount++
+		mu.Unlock()
+		return true, nil // Continue processing
+	}
+
+	startPosition := NewMessagePositionFromOffset(0)
+	startTime := time.Now()
+
+	_, isDone, _ := logBuffer.LoopProcessLogDataWithOffset("test-client", startPosition, 0, waitForDataFn, eachLogEntryFn)
+
+	elapsed := time.Since(startTime)
+
+	if !isDone {
+		t.Errorf("Expected isDone=true after client disconnect, got false")
+	}
+
+	mu.Lock()
+	finalCount := receivedCount
+	mu.Unlock()
+
+	if finalCount < 1 {
+		t.Errorf("Expected to receive at least 1 message, got %d", finalCount)
+	}
+
+	// Should complete quickly since data is available
+	if elapsed > 1*time.Second {
+		t.Errorf("Processing took too long: %v (expected < 1s)", elapsed)
+	}
+
+	t.Logf("Successfully processed %d message(s) in %v", finalCount, elapsed)
+}
+
+// TestLoopProcessLogDataWithOffset_ConcurrentDisconnect tests that the loop
+// handles concurrent client disconnects without panicking
+func TestLoopProcessLogDataWithOffset_ConcurrentDisconnect(t *testing.T) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	numClients := 10
+	var wg sync.WaitGroup
+
+	for i := 0; i < numClients; i++ {
+		wg.Add(1)
+		go func(clientID int) {
+			defer wg.Done()
+
+			ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+			defer cancel()
+
+			waitForDataFn := func() bool {
+				select {
+				case <-ctx.Done():
+					return false
+				default:
+					return true
+				}
+			}
+
+			eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+				return true, nil
+			}
+
+			startPosition := NewMessagePositionFromOffset(0)
+			_, _, _ = logBuffer.LoopProcessLogDataWithOffset("test-client", startPosition, 0, waitForDataFn, eachLogEntryFn)
+		}(i)
+	}
+
+	// Wait for all clients to finish with a timeout
+	done := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		t.Logf("All %d concurrent clients exited cleanly", numClients)
+	case <-time.After(5 * time.Second):
+		t.Errorf("Timeout waiting for concurrent clients to exit (possible deadlock or stuck loop)")
+	}
+}
+
+// TestLoopProcessLogDataWithOffset_StopTime tests that the loop respects stopTsNs
+func TestLoopProcessLogDataWithOffset_StopTime(t *testing.T) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	callCount := 0
+	waitForDataFn := func() bool {
+		callCount++
+		// Prevent infinite loop in case of test failure
+		return callCount < 10
+	}
+
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+		t.Errorf("Should not process any entries when stopTsNs is in the past")
+		return false, nil
+	}
+
+	startPosition := NewMessagePositionFromOffset(0)
+	stopTsNs := time.Now().Add(-1 * time.Hour).UnixNano() // Stop time in the past
+
+	startTime := time.Now()
+	_, isDone, _ := logBuffer.LoopProcessLogDataWithOffset("test-client", startPosition, stopTsNs, waitForDataFn, eachLogEntryFn)
+	elapsed := time.Since(startTime)
+
+	if !isDone {
+		t.Errorf("Expected isDone=true when stopTsNs is in the past, got false")
+	}
+
+	if elapsed > 1*time.Second {
+		t.Errorf("Loop should exit quickly when stopTsNs is in the past, took %v", elapsed)
+	}
+
+	t.Logf("Loop correctly exited for past stopTsNs in %v (waitForDataFn called %d times)", elapsed, callCount)
+}
+
+// TestLoopProcessLogData_BoundedEmptyBufferTerminates: a bounded subscription
+// (stopTsNs != 0) against a buffer that has never received a write - so
+// ReadFromBuffer returns ResumeFromDiskError - with no ReadFromDiskFn must
+// terminate like the caught-up path does, not park on the notification loop
+// forever. This is the filer SubscribeMetadata configuration: both meta log
+// buffers are built with ReadFromDiskFn == nil, and a freshly restarted filer
+// that has serviced zero metadata writes wedges every UntilNs-bounded
+// subscriber while heartbeats keep the stream looking alive.
+func TestLoopProcessLogData_BoundedEmptyBufferTerminates(t *testing.T) {
+	logBuffer := NewLogBuffer("test", time.Minute, nil, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	// Client stays connected, like the filer's heartbeat-sending closure.
+	waitForDataFn := func() bool { return true }
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry) (bool, error) {
+		t.Error("no entries should be delivered from an empty buffer")
+		return false, nil
+	}
+
+	startPosition := NewMessagePosition(time.Now().Add(-time.Hour).UnixNano(), EvictionGatedOffset)
+	stopTsNs := time.Now().UnixNano()
+
+	done := make(chan struct{})
+	var isDone bool
+	var err error
+	go func() {
+		_, isDone, err = logBuffer.LoopProcessLogData("bounded-empty", startPosition, stopTsNs, waitForDataFn, eachLogEntryFn)
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		if !isDone {
+			t.Errorf("expected isDone=true for a bounded read of an empty buffer, got false")
+		}
+		// A leaked ResumeFromDiskError would send the filer's outer loop into
+		// its gap machinery, which parks the bounded subscriber all over again.
+		if err != nil {
+			t.Errorf("expected err=nil for a bounded read of an empty buffer, got %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("bounded LoopProcessLogData wedged on an empty buffer instead of terminating")
+	}
+}
+
+func TestLoopProcessLogData_SlowConsumerFallsBehind(t *testing.T) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	baseTime := time.Now()
+	for i := 0; i < 1000; i++ {
+		ts := baseTime.Add(time.Duration(i) * time.Millisecond)
+		if err := logBuffer.AddDataToBuffer([]byte("key"), []byte("value"), ts.UnixNano()); err != nil {
+			t.Fatalf("AddDataToBuffer(%d): %v", i, err)
+		}
+	}
+
+	oldPosition := NewMessagePosition(baseTime.Add(-10*time.Second).UnixNano(), 1)
+
+	waitForDataFn := func() bool {
+		t.Errorf("waitForDataFn should not be called for a slow consumer that has fallen behind")
+		return false
+	}
+
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry) (bool, error) {
+		return false, nil
+	}
+
+	done := make(chan struct{})
+	var err error
+	go func() {
+		_, _, err = logBuffer.LoopProcessLogData("slow-consumer", oldPosition, 0, waitForDataFn, eachLogEntryFn)
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		if err != ResumeFromDiskError {
+			t.Fatalf("expected ResumeFromDiskError, got %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("LoopProcessLogData blocked instead of returning ResumeFromDiskError")
+	}
+}
+
+// TestLoopProcessLogData_DuplicateReaderLeaving is a regression test for issue
+// #10810: an idle metadata subscriber pinning a full core inside
+// awaitNotificationOrTimeoutFor. Two readers registered under the same name
+// share one notification channel, which happens whenever a client reconnects
+// before its previous stream noticed the disconnect. When the departing one
+// closed that channel, the surviving reader's select won on it instantly on
+// every pass, so the loop ran flat out - allocating a timer per iteration -
+// until the client went away.
+func TestLoopProcessLogData_DuplicateReaderLeaving(t *testing.T) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	const readerName = "localMeta:s3@"
+	startPosition := NewMessagePosition(time.Now().UnixNano(), -2)
+	eachLogEntryFn := func(logEntry *filer_pb.LogEntry) (bool, error) { return false, nil }
+
+	// startReader runs a reader and reports how many times it went round the
+	// loop; the loop calls waitForDataFn exactly once per pass.
+	startReader := func(iterations *atomic.Int64, stop *atomic.Bool) chan struct{} {
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			logBuffer.LoopProcessLogData(readerName, startPosition, 0, func() bool {
+				iterations.Add(1)
+				return !stop.Load()
+			}, eachLogEntryFn)
+		}()
+		for iterations.Load() == 0 {
+			time.Sleep(time.Millisecond)
+		}
+		return done
+	}
+
+	var leavingIterations, survivorIterations atomic.Int64
+	var stopLeaving, stopSurvivor atomic.Bool
+	leavingDone := startReader(&leavingIterations, &stopLeaving)
+	survivorDone := startReader(&survivorIterations, &stopSurvivor)
+
+	stopLeaving.Store(true)
+	<-leavingDone
+
+	const observeFor = 500 * time.Millisecond
+	before := survivorIterations.Load()
+	time.Sleep(observeFor)
+	spun := survivorIterations.Load() - before
+
+	// A low iteration count only means "not spinning" if the reader was still
+	// there to spin. Had LoopProcessLogData returned when its duplicate left,
+	// spun would be 0 and the count below would pass while proving nothing.
+	select {
+	case <-survivorDone:
+		t.Fatal("surviving reader returned while its duplicate unregistered; it must keep reading")
+	default:
+	}
+
+	stopSurvivor.Store(true)
+	<-survivorDone
+
+	maxIterations := int64(observeFor/notificationHealthCheckInterval) + 1
+	if spun > maxIterations {
+		t.Errorf("surviving reader looped %d times in %v, expected at most %d (suggests busy-waiting)", spun, observeFor, maxIterations)
+	}
+}
+
+// BenchmarkLoopProcessLogDataWithOffset_EmptyBuffer benchmarks the performance
+// of the loop with an empty buffer to ensure no busy-waiting
+func BenchmarkLoopProcessLogDataWithOffset_EmptyBuffer(b *testing.B) {
+	flushFn := func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {}
+	logBuffer := NewLogBuffer("test", 1*time.Minute, flushFn, nil, nil)
+	defer logBuffer.ShutdownLogBuffer()
+
+	for i := 0; i < b.N; i++ {
+		callCount := 0
+		waitForDataFn := func() bool {
+			callCount++
+			return callCount < 3 // Exit after 3 calls
+		}
+
+		eachLogEntryFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+			return true, nil
+		}
+
+		startPosition := NewMessagePositionFromOffset(0)
+		logBuffer.LoopProcessLogDataWithOffset("test-client", startPosition, 0, waitForDataFn, eachLogEntryFn)
+	}
+}

weed/util/log_buffer/read_ts_test.go

@@ -0,0 +1,230 @@
+package log_buffer
+
+import (
+	"bytes"
+	"math"
+	"math/rand"
+	"testing"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// readTsNs must return exactly what a full proto.Unmarshal would put in TsNs,
+// for every field combination, without decoding data/key.
+func TestReadTsNsMatchesUnmarshal(t *testing.T) {
+	cases := []*filer_pb.LogEntry{
+		{},                              // all defaults: ts_ns absent on the wire => 0
+		{TsNs: 1},                       // minimal
+		{TsNs: 1783396413597780505},     // realistic nanosecond timestamp
+		{TsNs: math.MaxInt64},           // 9-byte varint
+		{TsNs: -1},                      // negative int64 => 10-byte varint
+		{TsNs: math.MinInt64},           // extreme negative
+		{TsNs: 42, PartitionKeyHash: 7}, // ts followed by another varint field
+		{TsNs: 12345, Data: []byte("some-metadata-event-payload"), Key: []byte("/buckets/pvc-x/1.log")},
+		{TsNs: 99, Data: make([]byte, 64*1024), Key: []byte("k"), Offset: 555}, // large data, all fields
+		{Data: []byte("no-ts"), Key: []byte("k"), Offset: 9},                   // ts_ns == 0 with other fields present
+	}
+	for i, e := range cases {
+		data, err := proto.Marshal(e)
+		if err != nil {
+			t.Fatalf("case %d marshal: %v", i, err)
+		}
+		got, err := readTsNs(data)
+		if err != nil {
+			t.Fatalf("case %d readTsNs: %v", i, err)
+		}
+		if got != e.TsNs {
+			t.Errorf("case %d: readTsNs=%d want %d", i, got, e.TsNs)
+		}
+	}
+}
+
+// Fuzz against proto.Unmarshal with randomized entries.
+func TestReadTsNsFuzz(t *testing.T) {
+	r := rand.New(rand.NewSource(1))
+	for i := 0; i < 5000; i++ {
+		e := &filer_pb.LogEntry{
+			TsNs:             r.Int63() - r.Int63(),
+			PartitionKeyHash: r.Int31(),
+			Offset:           r.Int63(),
+		}
+		if r.Intn(2) == 0 {
+			e.Data = make([]byte, r.Intn(300))
+			r.Read(e.Data)
+		}
+		if r.Intn(2) == 0 {
+			e.Key = make([]byte, r.Intn(64))
+			r.Read(e.Key)
+		}
+		data, err := proto.Marshal(e)
+		if err != nil {
+			t.Fatalf("marshal: %v", err)
+		}
+		var want filer_pb.LogEntry
+		if err := proto.Unmarshal(data, &want); err != nil {
+			t.Fatalf("unmarshal: %v", err)
+		}
+		got, err := readTsNs(data)
+		if err != nil {
+			t.Fatalf("readTsNs iter %d: %v", i, err)
+		}
+		if got != want.TsNs {
+			t.Fatalf("iter %d: readTsNs=%d want %d", i, got, want.TsNs)
+		}
+	}
+}
+
+// readTsNs must not panic and must report an error on truncated/garbage input.
+func TestReadTsNsCorrupted(t *testing.T) {
+	inputs := [][]byte{
+		{0x08},                   // ts_ns tag with no value
+		{0x1a, 0xff, 0xff, 0xff}, // data field, length overruns buffer
+		{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f}, // overlong tag
+	}
+	for i, in := range inputs {
+		if _, err := readTsNs(in); err == nil {
+			t.Errorf("input %d: expected error, got nil", i)
+		}
+	}
+}
+
+// unmarshalLogEntryAliased must produce the same fields as proto.Unmarshal.
+func TestUnmarshalLogEntryAliasedMatchesUnmarshal(t *testing.T) {
+	r := rand.New(rand.NewSource(2))
+	for i := 0; i < 5000; i++ {
+		e := &filer_pb.LogEntry{
+			TsNs:             r.Int63() - r.Int63(),
+			PartitionKeyHash: r.Int31() - r.Int31(),
+			Offset:           r.Int63() - r.Int63(),
+		}
+		if r.Intn(2) == 0 {
+			e.Data = make([]byte, r.Intn(4096))
+			r.Read(e.Data)
+		}
+		if r.Intn(2) == 0 {
+			e.Key = make([]byte, r.Intn(128))
+			r.Read(e.Key)
+		}
+		data, err := proto.Marshal(e)
+		if err != nil {
+			t.Fatalf("marshal: %v", err)
+		}
+		var want, got filer_pb.LogEntry
+		if err := proto.Unmarshal(data, &want); err != nil {
+			t.Fatalf("unmarshal: %v", err)
+		}
+		if err := unmarshalLogEntryAliased(data, &got); err != nil {
+			t.Fatalf("aliased iter %d: %v", i, err)
+		}
+		if got.TsNs != want.TsNs || got.PartitionKeyHash != want.PartitionKeyHash ||
+			got.Offset != want.Offset || !bytes.Equal(got.Data, want.Data) || !bytes.Equal(got.Key, want.Key) {
+			t.Fatalf("iter %d mismatch:\n got=%+v\nwant=%+v", i, &got, &want)
+		}
+	}
+}
+
+// The decoded data/key must point into the source buffer, not a copy.
+func TestUnmarshalLogEntryAliasedAliases(t *testing.T) {
+	data, err := proto.Marshal(&filer_pb.LogEntry{TsNs: 5, Data: []byte("payload"), Key: []byte("path")})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var out filer_pb.LogEntry
+	if err := unmarshalLogEntryAliased(data, &out); err != nil {
+		t.Fatal(err)
+	}
+	if string(out.Data) != "payload" || string(out.Key) != "path" {
+		t.Fatalf("decoded data=%q key=%q", out.Data, out.Key)
+	}
+	for i := range data { // mutate source; an alias must observe it, a copy must not
+		data[i] ^= 0xFF
+	}
+	if string(out.Data) == "payload" || string(out.Key) == "path" {
+		t.Fatal("data/key were copied, expected an alias into the source buffer")
+	}
+}
+
+// sampleEntryData returns a marshaled LogEntry whose Data is a realistic
+// marshaled metadata event, as delivered on the subscribe path.
+func sampleEntryData(tb testing.TB) []byte {
+	event := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/buckets/pvc-bf458866-0413-4cca-99c6-3b7386a80855",
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name:       "object-000123",
+				Attributes: &filer_pb.FuseAttributes{FileSize: 4096, Mtime: 1783396413},
+				Chunks: []*filer_pb.FileChunk{
+					{FileId: "3,01637037d6", Size: 4096, Fid: &filer_pb.FileId{VolumeId: 3, FileKey: 23283, Cookie: 2015}},
+				},
+			},
+		},
+	}
+	payload, err := proto.Marshal(event)
+	if err != nil {
+		tb.Fatal(err)
+	}
+	data, err := proto.Marshal(&filer_pb.LogEntry{
+		TsNs: 1783396413597780505, PartitionKeyHash: 12345, Offset: 2013258,
+		Key: []byte("/buckets/pvc-bf458866/object-000123"), Data: payload,
+	})
+	if err != nil {
+		tb.Fatal(err)
+	}
+	return data
+}
+
+// Per-entry decode cost on the delivery path: the old full unmarshal copies the
+// data/key byte fields; the aliased decode does not. The delta is what the fix
+// saves for every entry a subscriber reads (and, for non-matching entries a
+// subscriber filters out, the whole saving with nothing decoded downstream).
+func BenchmarkLogEntryDecodeCopy(b *testing.B) {
+	data := sampleEntryData(b)
+	var out filer_pb.LogEntry
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := proto.Unmarshal(data, &out); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkLogEntryDecodeAlias(b *testing.B) {
+	data := sampleEntryData(b)
+	var out filer_pb.LogEntry
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := unmarshalLogEntryAliased(data, &out); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkReadTs(b *testing.B) {
+	e := &filer_pb.LogEntry{
+		TsNs:             1783396413597780505,
+		PartitionKeyHash: 12345,
+		Data:             make([]byte, 4*1024), // typical metadata event payload
+		Key:              []byte("/buckets/pvc-bf458866/1.log"),
+		Offset:           2013258,
+	}
+	data, err := proto.Marshal(e)
+	if err != nil {
+		b.Fatal(err)
+	}
+	buf := make([]byte, 4+len(data))
+	util.Uint32toBytes(buf, uint32(len(data)))
+	copy(buf[4:], data)
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, _, err := readTs(buf, 0); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

weed/util/log_buffer/sealed_buffer.go

@@ -0,0 +1,98 @@
+package log_buffer
+
+import (
+	"fmt"
+	"sync"
+	"time"
+)
+
+type MemBuffer struct {
+	buf         []byte
+	size        int
+	startTime   time.Time
+	stopTime    time.Time
+	startOffset int64 // First offset in this buffer
+	offset      int64 // Last offset in this buffer (endOffset)
+	// max pre-bump entry timestamp; stamped by the sealer after SealBuffer
+	maxOriginalTsNs int64
+
+	// snapshot is a GC-owned copy of buf[:size] shared by all readers of this
+	// sealed window, so N subscribers reading the same window cost one copy
+	// instead of N pooled copies. Created lazily by the first reader; travels
+	// with the window when SealBuffer shifts slots. Immutable once created.
+	snapMu   sync.Mutex
+	snapshot []byte
+}
+
+// sharedSnapshot returns the shared read-only copy of this sealed window,
+// creating it on first use. Callers must hold the LogBuffer read lock, which
+// keeps buf stable (SealBuffer mutates slots only under the write lock).
+func (mb *MemBuffer) sharedSnapshot() []byte {
+	mb.snapMu.Lock()
+	defer mb.snapMu.Unlock()
+	if mb.snapshot == nil {
+		mb.snapshot = append([]byte(nil), mb.buf[:mb.size]...)
+	}
+	return mb.snapshot
+}
+
+type SealedBuffers struct {
+	buffers []*MemBuffer
+}
+
+func newSealedBuffers(size int) *SealedBuffers {
+	sbs := &SealedBuffers{}
+
+	sbs.buffers = make([]*MemBuffer, size)
+	for i := 0; i < size; i++ {
+		sbs.buffers[i] = &MemBuffer{
+			buf: make([]byte, BufferSize),
+		}
+	}
+
+	return sbs
+}
+
+func (sbs *SealedBuffers) SealBuffer(startTime, stopTime time.Time, buf []byte, pos int, startOffset int64, endOffset int64) (newBuf []byte) {
+	oldBuf := sbs.buffers[0].buf
+	size := len(sbs.buffers)
+	for i := 0; i < size-1; i++ {
+		sbs.buffers[i].buf = sbs.buffers[i+1].buf
+		sbs.buffers[i].size = sbs.buffers[i+1].size
+		sbs.buffers[i].startTime = sbs.buffers[i+1].startTime
+		sbs.buffers[i].stopTime = sbs.buffers[i+1].stopTime
+		sbs.buffers[i].startOffset = sbs.buffers[i+1].startOffset
+		sbs.buffers[i].offset = sbs.buffers[i+1].offset
+		sbs.buffers[i].snapshot = sbs.buffers[i+1].snapshot // snapshot follows its window
+		sbs.buffers[i].maxOriginalTsNs = sbs.buffers[i+1].maxOriginalTsNs
+	}
+	sbs.buffers[size-1].buf = buf
+	sbs.buffers[size-1].size = pos
+	sbs.buffers[size-1].startTime = startTime
+	sbs.buffers[size-1].stopTime = stopTime
+	sbs.buffers[size-1].startOffset = startOffset
+	sbs.buffers[size-1].offset = endOffset
+	sbs.buffers[size-1].snapshot = nil
+	sbs.buffers[size-1].maxOriginalTsNs = 0
+	return oldBuf
+}
+
+func (mb *MemBuffer) locateByTs(lastReadTime time.Time) (pos int, err error) {
+	lastReadTs := lastReadTime.UnixNano()
+	for pos < mb.size {
+		size, t, readErr := readTs(mb.buf, pos)
+		if readErr != nil {
+			// Return error if buffer is corrupted
+			return 0, fmt.Errorf("locateByTs: buffer corruption at pos %d: %w", pos, readErr)
+		}
+		if t > lastReadTs {
+			return pos, nil
+		}
+		pos += size + 4
+	}
+	return mb.size, nil
+}
+
+func (mb *MemBuffer) String() string {
+	return fmt.Sprintf("[%v,%v] bytes:%d", mb.startTime, mb.stopTime, mb.size)
+}

weed/util/log_buffer/shared_snapshot_test.go

@@ -0,0 +1,275 @@
+package log_buffer
+
+import (
+	"fmt"
+	"runtime/debug"
+	"sync"
+	"testing"
+	"time"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func addEntries(tb testing.TB, lb *LogBuffer, n int, payloadSize int, tag string) {
+	payload := make([]byte, payloadSize)
+	for i := 0; i < n; i++ {
+		key := []byte(fmt.Sprintf("/%s/%d", tag, i))
+		if err := lb.AddDataToBuffer(key, payload, time.Now().UnixNano()); err != nil {
+			tb.Fatal(err)
+		}
+	}
+}
+
+// Sealed-window reads must hand every reader the same shared snapshot rather
+// than a fresh copy each.
+func TestSealedReadsShareOneSnapshot(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	addEntries(t, lb, 10, 1024, "a")
+	lb.ForceFlush() // seal the window
+	addEntries(t, lb, 1, 1024, "b")
+
+	pos := NewMessagePosition(1, -2) // far in the past => sealed window hit
+	buf1, _, pooled1, err1 := lb.ReadFromBuffer(pos)
+	buf2, _, pooled2, err2 := lb.ReadFromBuffer(pos)
+	if err1 != nil || err2 != nil {
+		t.Fatalf("read errors: %v %v", err1, err2)
+	}
+	if buf1 == nil || buf2 == nil {
+		t.Fatal("expected sealed data in memory")
+	}
+	if pooled1 || pooled2 {
+		t.Fatalf("sealed reads should not be pooled copies: %v %v", pooled1, pooled2)
+	}
+	b1, b2 := buf1.Bytes(), buf2.Bytes()
+	if len(b1) == 0 || len(b1) != len(b2) {
+		t.Fatalf("unexpected lengths %d %d", len(b1), len(b2))
+	}
+	if &b1[0] != &b2[0] {
+		t.Fatal("sealed reads made independent copies; expected one shared snapshot")
+	}
+}
+
+// A shared snapshot must stay intact after its window rotates out and the
+// backing array is recycled and overwritten by new writes.
+func TestSharedSnapshotSurvivesRecycle(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	addEntries(t, lb, 5, 2048, "keep")
+	lb.ForceFlush()
+
+	pos := NewMessagePosition(1, -2)
+	buf, _, pooled, err := lb.ReadFromBuffer(pos)
+	if err != nil || buf == nil || pooled {
+		t.Fatalf("expected shared sealed read, got buf=%v pooled=%v err=%v", buf != nil, pooled, err)
+	}
+	before := append([]byte(nil), buf.Bytes()...)
+
+	// Rotate the sealed window all the way out so its array is recycled and
+	// overwritten with different content.
+	for i := 0; i < PreviousBufferCount+1; i++ {
+		addEntries(t, lb, 5, 2048, "overwrite")
+		lb.ForceFlush()
+	}
+	addEntries(t, lb, 5, 2048, "overwrite")
+
+	if string(before) != string(buf.Bytes()) {
+		t.Fatal("shared snapshot content changed after its window was recycled")
+	}
+}
+
+// Race test: concurrent lagging readers against a writer that keeps sealing
+// and recycling windows. Every delivered entry must unmarshal cleanly with the
+// expected payload — corruption here means a reader saw recycled bytes.
+func TestSharedSnapshotConcurrentIntegrity(t *testing.T) {
+	// The writer below is deliberately unthrottled, so transient marshal garbage
+	// scales with host speed; on linux/386 a fast runner can hit the 4GB
+	// address-space ceiling. Cap the heap so the GC paces the writer instead.
+	prevLimit := debug.SetMemoryLimit(1 << 30)
+	defer debug.SetMemoryLimit(prevLimit)
+
+	lb := NewLogBuffer("test", time.Hour, func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	const payloadByte = 0x5A
+	payload := make([]byte, 8*1024)
+	for i := range payload {
+		payload[i] = payloadByte
+	}
+
+	var stop sync.WaitGroup
+	done := make(chan struct{})
+	stop.Add(1)
+	go func() { // writer: fill and seal aggressively so arrays recycle under the readers
+		defer stop.Done()
+		for i := 0; ; i++ {
+			select {
+			case <-done:
+				return
+			default:
+			}
+			if err := lb.AddDataToBuffer([]byte(fmt.Sprintf("/k/%d", i)), payload, time.Now().UnixNano()); err != nil {
+				t.Error(err)
+				return
+			}
+			if i%200 == 0 {
+				lb.ForceFlush()
+			}
+		}
+	}()
+
+	var readers sync.WaitGroup
+	errCh := make(chan error, 16)
+	for r := 0; r < 8; r++ {
+		readers.Add(1)
+		go func(r int) {
+			defer readers.Done()
+			deadline := time.Now().Add(2 * time.Second)
+			for time.Now().Before(deadline) {
+				start := NewMessagePosition(time.Now().Add(-500*time.Millisecond).UnixNano(), -2)
+				lb.LoopProcessLogData(fmt.Sprintf("r%d", r), start, 0,
+					func() bool { return time.Now().Before(deadline) },
+					func(le *filer_pb.LogEntry) (bool, error) {
+						if len(le.Data) != len(payload) {
+							err := fmt.Errorf("payload length %d, want %d", len(le.Data), len(payload))
+							select {
+							case errCh <- err:
+							default:
+							}
+							return true, err
+						}
+						for _, b := range le.Data {
+							if b != payloadByte {
+								err := fmt.Errorf("corrupted payload byte %x", b)
+								select {
+								case errCh <- err:
+								default:
+								}
+								return true, err
+							}
+						}
+						return false, nil
+					})
+			}
+		}(r)
+	}
+	readers.Wait()
+	close(done)
+	stop.Wait()
+
+	select {
+	case err := <-errCh:
+		t.Fatal(err)
+	default:
+	}
+}
+
+// Current-window reads must share the prefix snapshot: two behind-readers get
+// views of the same backing array, and the view content must stay intact while
+// the writer keeps appending and eventually seals and recycles the window.
+func TestCurrentWindowPrefixSharing(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	addEntries(t, lb, 8, 1024, "cur")
+
+	pos := NewMessagePosition(1, -2) // behind => full current-window read
+	buf1, _, pooled1, err1 := lb.ReadFromBuffer(pos)
+	buf2, _, pooled2, err2 := lb.ReadFromBuffer(pos)
+	if err1 != nil || err2 != nil || buf1 == nil || buf2 == nil {
+		t.Fatalf("reads: %v %v %v %v", buf1 != nil, err1, buf2 != nil, err2)
+	}
+	if pooled1 || pooled2 {
+		t.Fatalf("current-window behind-reads should be shared, got pooled %v %v", pooled1, pooled2)
+	}
+	b1, b2 := buf1.Bytes(), buf2.Bytes()
+	if len(b1) == 0 || len(b1) != len(b2) || &b1[0] != &b2[0] {
+		t.Fatalf("expected one shared prefix snapshot, lens %d %d", len(b1), len(b2))
+	}
+	before := append([]byte(nil), b1...)
+
+	// Keep appending: the prefix must be extended, not reallocated or mutated.
+	addEntries(t, lb, 8, 1024, "more")
+	buf3, _, _, err3 := lb.ReadFromBuffer(pos)
+	if err3 != nil || buf3 == nil {
+		t.Fatalf("read3: %v %v", buf3 != nil, err3)
+	}
+	if len(buf3.Bytes()) <= len(before) {
+		t.Fatalf("extended read %d should exceed first read %d", len(buf3.Bytes()), len(before))
+	}
+	if string(buf3.Bytes()[:len(before)]) != string(before) {
+		t.Fatal("prefix changed when the snapshot was extended")
+	}
+
+	// Seal + rotate the window fully out; earlier views must stay intact.
+	for i := 0; i < PreviousBufferCount+1; i++ {
+		lb.ForceFlush()
+		addEntries(t, lb, 4, 1024, "rotate")
+	}
+	if string(b1) != string(before) {
+		t.Fatal("shared current-window view changed after seal and recycle")
+	}
+}
+
+// A sealed window whose prefix snapshot was fully extended must reuse it
+// rather than re-copying on the first sealed read.
+func TestSealHandsOffCompleteSnapshot(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	addEntries(t, lb, 6, 512, "h")
+	pos := NewMessagePosition(1, -2)
+	buf1, _, _, err := lb.ReadFromBuffer(pos) // extends prefix snapshot to full window
+	if err != nil || buf1 == nil {
+		t.Fatalf("read1: %v %v", buf1 != nil, err)
+	}
+	lb.ForceFlush()
+	addEntries(t, lb, 1, 512, "next")
+
+	buf2, _, pooled, err := lb.ReadFromBuffer(pos) // sealed read of the same window
+	if err != nil || buf2 == nil || pooled {
+		t.Fatalf("read2: buf=%v pooled=%v err=%v", buf2 != nil, pooled, err)
+	}
+	b1, b2 := buf1.Bytes(), buf2.Bytes()
+	if len(b1) == 0 || len(b1) > len(b2) || &b1[0] != &b2[0] {
+		t.Fatalf("sealed read should reuse the handed-off snapshot (lens %d %d)", len(b1), len(b2))
+	}
+}
+
+// Sanity: sealed-read content must byte-match what a proto round-trip expects.
+func TestSharedSnapshotContentMatches(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, nil)
+	defer lb.ShutdownLogBuffer()
+
+	addEntries(t, lb, 3, 512, "x")
+	lb.ForceFlush()
+
+	buf, _, _, err := lb.ReadFromBuffer(NewMessagePosition(1, -2))
+	if err != nil || buf == nil {
+		t.Fatalf("read: buf=%v err=%v", buf != nil, err)
+	}
+	data := buf.Bytes()
+	count := 0
+	for pos := 0; pos+4 < len(data); {
+		size, _, err := readTs(data, pos)
+		if err != nil {
+			t.Fatalf("entry %d: %v", count, err)
+		}
+		var le filer_pb.LogEntry
+		if err := proto.Unmarshal(data[pos+4:pos+4+size], &le); err != nil {
+			t.Fatalf("entry %d unmarshal: %v", count, err)
+		}
+		if want := fmt.Sprintf("/x/%d", count); string(le.Key) != want {
+			t.Fatalf("entry %d key %q, want %q", count, le.Key, want)
+		}
+		pos += 4 + size
+		count++
+	}
+	if count != 3 {
+		t.Fatalf("read %d entries, want 3", count)
+	}
+}