snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/pb/filer_pb_direct_read_test.go

@@ -0,0 +1,477 @@
+package pb
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// buildLogFileData creates the on-disk log file format:
+// [4-byte size | protobuf LogEntry] repeated.
+func buildLogFileData(events []*filer_pb.SubscribeMetadataResponse) []byte {
+	var buf bytes.Buffer
+	for _, event := range events {
+		eventData, _ := proto.Marshal(event)
+		logEntry := &filer_pb.LogEntry{
+			TsNs: event.TsNs,
+			Data: eventData,
+			Key:  []byte(event.Directory),
+		}
+		entryData, _ := proto.Marshal(logEntry)
+		sizeBuf := make([]byte, 4)
+		util.Uint32toBytes(sizeBuf, uint32(len(entryData)))
+		buf.Write(sizeBuf)
+		buf.Write(entryData)
+	}
+	return buf.Bytes()
+}
+
+func makeSubEvent(dir, name string, tsNs int64) *filer_pb.SubscribeMetadataResponse {
+	return &filer_pb.SubscribeMetadataResponse{
+		Directory: dir,
+		TsNs:      tsNs,
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name:        name,
+				IsDirectory: false,
+			},
+		},
+	}
+}
+
+// delayedReader wraps data with a per-open latency to simulate volume server I/O.
+type delayedReader struct {
+	data     []byte
+	delay    time.Duration
+	openedAt time.Time
+}
+
+func (r *delayedReader) Read(p []byte) (int, error) {
+	if r.openedAt.IsZero() {
+		r.openedAt = time.Now()
+		time.Sleep(r.delay)
+	}
+	if len(r.data) == 0 {
+		return 0, io.EOF
+	}
+	n := copy(p, r.data)
+	r.data = r.data[n:]
+	return n, nil
+}
+
+func (r *delayedReader) Close() error { return nil }
+
+type testLogFiles struct {
+	refs      []*filer_pb.LogFileChunkRef
+	fileData  map[string][]byte // key: "filerId:fileTsNs" → raw log file bytes
+	fileDelay time.Duration
+}
+
+func newTestLogFiles(numFilers, filesPerFiler, eventsPerFile int, fileDelay time.Duration) *testLogFiles {
+	t := &testLogFiles{
+		fileData:  make(map[string][]byte),
+		fileDelay: fileDelay,
+	}
+
+	baseTs := time.Now().Add(-time.Hour).UnixNano()
+	tsCounter := int64(0)
+
+	for f := 0; f < numFilers; f++ {
+		filerId := fmt.Sprintf("filer%02d", f)
+		for file := 0; file < filesPerFiler; file++ {
+			fileTsNs := baseTs + int64(file)*int64(time.Minute)
+
+			events := make([]*filer_pb.SubscribeMetadataResponse, eventsPerFile)
+			for i := 0; i < eventsPerFile; i++ {
+				tsCounter++
+				ts := baseTs + tsCounter
+				events[i] = makeSubEvent(
+					fmt.Sprintf("/data/%s/dir%02d", filerId, file),
+					fmt.Sprintf("file%04d.txt", i),
+					ts,
+				)
+			}
+
+			data := buildLogFileData(events)
+			key := fmt.Sprintf("%s:%d", filerId, fileTsNs)
+			t.fileData[key] = data
+
+			t.refs = append(t.refs, &filer_pb.LogFileChunkRef{
+				Chunks: []*filer_pb.FileChunk{{
+					FileId: key,
+				}},
+				FileTsNs: fileTsNs,
+				FilerId:  filerId,
+			})
+		}
+	}
+	return t
+}
+
+func (t *testLogFiles) readerFn() LogFileReaderFn {
+	return func(chunks []*filer_pb.FileChunk) (io.ReadCloser, error) {
+		if len(chunks) == 0 {
+			return nil, fmt.Errorf("no chunks")
+		}
+		key := chunks[0].FileId
+		data, ok := t.fileData[key]
+		if !ok {
+			return nil, fmt.Errorf("file not found: %s", key)
+		}
+		dataCopy := make([]byte, len(data))
+		copy(dataCopy, data)
+		return &delayedReader{data: dataCopy, delay: t.fileDelay}, nil
+	}
+}
+
+func (t *testLogFiles) totalEvents() int {
+	total := 0
+	for _, data := range t.fileData {
+		pos := 0
+		for pos+4 <= len(data) {
+			size := int(util.BytesToUint32(data[pos : pos+4]))
+			pos += 4 + size
+			total++
+		}
+	}
+	return total
+}
+
+// assertOrderedReplay reads all refs and checks every event arrives exactly
+// once, in timestamp order.
+func assertOrderedReplay(t *testing.T, files *testLogFiles) {
+	t.Helper()
+
+	var timestamps []int64
+	_, err := ReadLogFileRefs(files.refs, files.readerFn(), 0, 0,
+		PathFilter{PathPrefix: "/"},
+		func(resp *filer_pb.SubscribeMetadataResponse) error {
+			timestamps = append(timestamps, resp.TsNs)
+			return nil
+		})
+	if err != nil {
+		t.Fatalf("ReadLogFileRefs: %v", err)
+	}
+
+	if got, want := len(timestamps), files.totalEvents(); got != want {
+		t.Fatalf("expected %d events, got %d", want, got)
+	}
+	for i := 1; i < len(timestamps); i++ {
+		if timestamps[i] < timestamps[i-1] {
+			t.Fatalf("out of order at index %d: ts[%d]=%d > ts[%d]=%d",
+				i, i-1, timestamps[i-1], i, timestamps[i])
+		}
+	}
+}
+
+// TestReadLogFileRefsMergeOrder verifies that entries from multiple filers are
+// delivered in correct timestamp order.
+func TestReadLogFileRefsMergeOrder(t *testing.T) {
+	assertOrderedReplay(t, newTestLogFiles(3, 2, 50, 0))
+}
+
+// TestReadLogFileRefsPathFilter verifies path filtering including system log exclusion.
+func TestReadLogFileRefsPathFilter(t *testing.T) {
+	files := newTestLogFiles(2, 2, 50, 0)
+	total := files.totalEvents()
+
+	var allCount, filteredCount int64
+	_, err := ReadLogFileRefs(files.refs, files.readerFn(), 0, 0,
+		PathFilter{PathPrefix: "/"},
+		func(resp *filer_pb.SubscribeMetadataResponse) error {
+			allCount++
+			return nil
+		})
+	if err != nil {
+		t.Fatalf("ReadLogFileRefs (all): %v", err)
+	}
+
+	_, err = ReadLogFileRefs(files.refs, files.readerFn(), 0, 0,
+		PathFilter{PathPrefix: "/data/filer00/"},
+		func(resp *filer_pb.SubscribeMetadataResponse) error {
+			filteredCount++
+			return nil
+		})
+	if err != nil {
+		t.Fatalf("ReadLogFileRefs (filtered): %v", err)
+	}
+
+	t.Logf("Total events: %d, matching /data/filer00/: %d", allCount, filteredCount)
+
+	if allCount != int64(total) {
+		t.Errorf("expected %d total events, got %d", total, allCount)
+	}
+	if filteredCount >= allCount {
+		t.Errorf("filter should reduce events: all=%d filtered=%d", allCount, filteredCount)
+	}
+	if filteredCount == 0 {
+		t.Errorf("filter matched zero events")
+	}
+}
+
+// TestDirectReadVsServerSideThroughput compares:
+//   - Server-side: sequential file read → gRPC send per event
+//   - Client direct-read: parallel filers + streaming + no gRPC
+func TestDirectReadVsServerSideThroughput(t *testing.T) {
+	const (
+		numFilers     = 3
+		filesPerFiler = 7
+		eventsPerFile = 300
+		fileReadDelay = 2 * time.Millisecond
+		sendDelay     = 20 * time.Microsecond
+	)
+
+	files := newTestLogFiles(numFilers, filesPerFiler, eventsPerFile, fileReadDelay)
+
+	var serverRate float64
+	t.Run("server_side_sequential", func(t *testing.T) {
+		var processed int64
+		start := time.Now()
+
+		for _, ref := range files.refs {
+			time.Sleep(fileReadDelay)
+			key := ref.Chunks[0].FileId
+			data := files.fileData[key]
+			pos := 0
+			for pos+4 <= len(data) {
+				size := int(util.BytesToUint32(data[pos : pos+4]))
+				pos += 4 + size
+				time.Sleep(sendDelay)
+				atomic.AddInt64(&processed, 1)
+			}
+		}
+		elapsed := time.Since(start)
+		serverRate = float64(processed) / elapsed.Seconds()
+		t.Logf("server-side: %d events  %v  %6.0f events/sec  (%d files sequential + %v send/event)",
+			processed, elapsed.Round(time.Millisecond), serverRate,
+			numFilers*filesPerFiler, sendDelay)
+	})
+
+	var directRate float64
+	t.Run("client_direct_read_parallel_streaming", func(t *testing.T) {
+		var processed int64
+		start := time.Now()
+
+		_, err := ReadLogFileRefs(files.refs, files.readerFn(), 0, 0,
+			PathFilter{PathPrefix: "/"},
+			func(resp *filer_pb.SubscribeMetadataResponse) error {
+				atomic.AddInt64(&processed, 1)
+				return nil
+			})
+		if err != nil {
+			t.Fatalf("ReadLogFileRefs: %v", err)
+		}
+		elapsed := time.Since(start)
+		directRate = float64(processed) / elapsed.Seconds()
+		t.Logf("direct-read: %d events  %v  %6.0f events/sec  (%d filers parallel + streaming, no gRPC)",
+			processed, elapsed.Round(time.Millisecond), directRate, numFilers)
+	})
+
+	if serverRate > 0 {
+		t.Logf("Speedup: %.1fx (parallel + streaming + no gRPC vs server-side sequential)", directRate/serverRate)
+	}
+}
+
+// failingReaderFn returns err for the given file key, delegating otherwise.
+func failingReaderFn(base LogFileReaderFn, failKey string, err error) LogFileReaderFn {
+	return func(chunks []*filer_pb.FileChunk) (io.ReadCloser, error) {
+		if len(chunks) > 0 && chunks[0].FileId == failKey {
+			return nil, err
+		}
+		return base(chunks)
+	}
+}
+
+// A real (non not-found) read error must fail the whole replay, not silently
+// drop the file and advance the cursor.
+func TestReadLogFileRefsMultiFilerGenuineErrorAborts(t *testing.T) {
+	files := newTestLogFiles(3, 2, 10, 0)
+	failKey := files.refs[2].Chunks[0].FileId // filer01's first file
+	readerFn := failingReaderFn(files.readerFn(), failKey, fmt.Errorf("failed to locate %s", failKey))
+
+	var count int64
+	_, err := ReadLogFileRefs(files.refs, readerFn, 0, 0,
+		PathFilter{PathPrefix: "/"},
+		func(resp *filer_pb.SubscribeMetadataResponse) error {
+			atomic.AddInt64(&count, 1)
+			return nil
+		})
+	if err == nil {
+		t.Fatalf("expected error from genuine read failure, got nil (delivered=%d)", count)
+	}
+}
+
+// A chunk-not-found error skips only that file (volume gone), not the replay.
+func TestReadLogFileRefsMultiFilerNotFoundSkips(t *testing.T) {
+	files := newTestLogFiles(3, 2, 10, 0)
+	skipKey := files.refs[2].Chunks[0].FileId // filer01's first file
+	readerFn := failingReaderFn(files.readerFn(), skipKey, fmt.Errorf("volume not found: %s", skipKey))
+
+	var count int64
+	_, err := ReadLogFileRefs(files.refs, readerFn, 0, 0,
+		PathFilter{PathPrefix: "/"},
+		func(resp *filer_pb.SubscribeMetadataResponse) error {
+			atomic.AddInt64(&count, 1)
+			return nil
+		})
+	if err != nil {
+		t.Fatalf("chunk-not-found should be skipped, got error: %v", err)
+	}
+	expected := int64(files.totalEvents() - 10) // one skipped file's events
+	if count != expected {
+		t.Fatalf("expected %d events after skipping one file, got %d", expected, count)
+	}
+}
+
+// TestReadLogFileRefsSingleFilerOrder covers the single-filer path: every
+// entry across all files, in order.
+func TestReadLogFileRefsSingleFilerOrder(t *testing.T) {
+	assertOrderedReplay(t, newTestLogFiles(1, 4, 50, 0))
+}
+
+// TestReadLogFileRefsSingleFilerProcessErrorStops verifies that the callback's
+// own error propagates and aborts the stream promptly, mid-file.
+func TestReadLogFileRefsSingleFilerProcessErrorStops(t *testing.T) {
+	files := newTestLogFiles(1, 3, 100, 0)
+
+	var count int
+	wantErr := fmt.Errorf("boom")
+	_, err := ReadLogFileRefs(files.refs, files.readerFn(), 0, 0,
+		PathFilter{PathPrefix: "/"},
+		func(resp *filer_pb.SubscribeMetadataResponse) error {
+			count++
+			if count == 5 {
+				return wantErr
+			}
+			return nil
+		})
+	if !errors.Is(err, wantErr) {
+		t.Fatalf("expected processing error to propagate, got: %v", err)
+	}
+	// Should stop near the failing event, not process the whole 300-event set.
+	if count > 20 {
+		t.Fatalf("expected prompt stop after error, processed %d events", count)
+	}
+}
+
+// A corrupt size prefix must fail the replay instead of allocating gigabytes.
+func TestReadLogFileRefsCorruptSizePrefix(t *testing.T) {
+	data := make([]byte, 4)
+	util.Uint32toBytes(data, 0xFFFFFFF0)
+	refs := []*filer_pb.LogFileChunkRef{{
+		Chunks:   []*filer_pb.FileChunk{{FileId: "corrupt"}},
+		FileTsNs: 1,
+		FilerId:  "filer00",
+	}}
+	readerFn := func(chunks []*filer_pb.FileChunk) (io.ReadCloser, error) {
+		return io.NopCloser(bytes.NewReader(data)), nil
+	}
+
+	_, err := ReadLogFileRefs(refs, readerFn, 0, 0, PathFilter{PathPrefix: "/"},
+		func(*filer_pb.SubscribeMetadataResponse) error { return nil })
+	if err == nil || !strings.Contains(err.Error(), "exceeds") {
+		t.Fatalf("expected size-cap error, got: %v", err)
+	}
+}
+
+// blockingReader blocks in Read until released.
+type blockingReader struct{ release chan struct{} }
+
+func (r *blockingReader) Read(p []byte) (int, error) { <-r.release; return 0, io.EOF }
+func (r *blockingReader) Close() error               { return nil }
+
+// An abort (fatal error on one filer) must not wait for another filer's
+// in-flight chunk read: the replay returns promptly and the wedged producer
+// exits on its own once its read completes.
+func TestReadLogFileRefsAbortDoesNotJoinWedgedReader(t *testing.T) {
+	files := newTestLogFiles(2, 1, 10, 0)
+	release := make(chan struct{})
+	t.Cleanup(func() { close(release) })
+
+	wedgedKey := files.refs[0].Chunks[0].FileId // filer00 wedges mid-read
+	failKey := files.refs[1].Chunks[0].FileId   // filer01 fails for real
+	base := files.readerFn()
+	readerFn := func(chunks []*filer_pb.FileChunk) (io.ReadCloser, error) {
+		switch chunks[0].FileId {
+		case wedgedKey:
+			return &blockingReader{release: release}, nil
+		case failKey:
+			return nil, fmt.Errorf("failed to locate %s", failKey)
+		}
+		return base(chunks)
+	}
+
+	done := make(chan error, 1)
+	go func() {
+		_, err := ReadLogFileRefs(files.refs, readerFn, 0, 0, PathFilter{PathPrefix: "/"},
+			func(*filer_pb.SubscribeMetadataResponse) error { return nil })
+		done <- err
+	}()
+
+	select {
+	case err := <-done:
+		if err == nil {
+			t.Fatalf("expected the fatal read error to propagate")
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatalf("ReadLogFileRefs did not return while a peer reader was wedged")
+	}
+}
+
+// TestReadLogFileRefsSingleFilerNotFoundSkips confirms a chunk-not-found on the
+// single-filer path skips just that file, not the whole replay.
+func TestReadLogFileRefsSingleFilerNotFoundSkips(t *testing.T) {
+	files := newTestLogFiles(1, 3, 10, 0)
+	skipKey := files.refs[1].Chunks[0].FileId // second file
+	readerFn := failingReaderFn(files.readerFn(), skipKey, fmt.Errorf("volume not found: %s", skipKey))
+
+	var count int
+	_, err := ReadLogFileRefs(files.refs, readerFn, 0, 0,
+		PathFilter{PathPrefix: "/"},
+		func(resp *filer_pb.SubscribeMetadataResponse) error {
+			count++
+			return nil
+		})
+	if err != nil {
+		t.Fatalf("chunk-not-found should be skipped, got error: %v", err)
+	}
+	if want := files.totalEvents() - 10; count != want {
+		t.Fatalf("expected %d events after skipping one file, got %d", want, count)
+	}
+}
+
+// TestProcessOneLogEntrySkipsInvalidUTF8 guards that an event whose string field
+// is not valid UTF-8 is rejected by proto.Unmarshal and never reaches path
+// filtering or the consumer. UnmarshalVT would accept it, so the decode here
+// must stay on proto.Unmarshal.
+func TestProcessOneLogEntrySkipsInvalidUTF8(t *testing.T) {
+	// SubscribeMetadataResponse{Directory: "\xff"}: field 1 (string), wire type 2,
+	// length 1, byte 0xff — a syntactically valid message with an invalid-UTF-8
+	// string, which proto3 forbids.
+	logEntry := &filer_pb.LogEntry{Data: []byte{0x0a, 0x01, 0xff}}
+
+	called := false
+	tsNs, err := processOneLogEntry(logEntry, PathFilter{PathPrefix: "/"}, func(resp *filer_pb.SubscribeMetadataResponse) error {
+		called = true
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("processOneLogEntry err = %v, want nil (corrupt entries are skipped)", err)
+	}
+	if called {
+		t.Fatal("consumer was invoked for a malformed (invalid UTF-8) event; validation was bypassed")
+	}
+	if tsNs != 0 {
+		t.Fatalf("tsNs = %d, want 0 for a skipped entry", tsNs)
+	}
+}