snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/mq/logstore/log_to_parquet.go

@@ -0,0 +1,609 @@
+package logstore
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/parquet-go/parquet-go"
+	"github.com/parquet-go/parquet-go/compress/zstd"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/mq"
+	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+	"google.golang.org/protobuf/proto"
+)
+
+const (
+	SW_COLUMN_NAME_TS     = "_ts_ns"
+	SW_COLUMN_NAME_KEY    = "_key"
+	SW_COLUMN_NAME_OFFSET = "_offset"
+	SW_COLUMN_NAME_VALUE  = "_value"
+)
+
+func CompactTopicPartitions(filerClient filer_pb.FilerClient, t topic.Topic, timeAgo time.Duration, recordType *schema_pb.RecordType, preference *operation.StoragePreference) error {
+	// list the topic partition versions
+	topicVersions, err := collectTopicVersions(filerClient, t, timeAgo)
+	if err != nil {
+		return fmt.Errorf("list topic files: %w", err)
+	}
+
+	// compact the partitions
+	for _, topicVersion := range topicVersions {
+		partitions, err := collectTopicVersionsPartitions(filerClient, t, topicVersion)
+		if err != nil {
+			return fmt.Errorf("list partitions %s/%s/%s: %v", t.Namespace, t.Name, topicVersion, err)
+		}
+		for _, partition := range partitions {
+			err := compactTopicPartition(filerClient, t, timeAgo, recordType, partition, preference)
+			if err != nil {
+				return fmt.Errorf("compact partition %s/%s/%s/%s: %v", t.Namespace, t.Name, topicVersion, partition, err)
+			}
+		}
+	}
+	return nil
+}
+
+func collectTopicVersions(filerClient filer_pb.FilerClient, t topic.Topic, timeAgo time.Duration) (partitionVersions []time.Time, err error) {
+	err = filer_pb.ReadDirAllEntries(context.Background(), filerClient, util.FullPath(t.Dir()), "", func(entry *filer_pb.Entry, isLast bool) error {
+		t, err := topic.ParseTopicVersion(entry.Name)
+		if err != nil {
+			// skip non-partition directories
+			return nil
+		}
+		if t.Unix() < time.Now().Unix()-int64(timeAgo/time.Second) {
+			partitionVersions = append(partitionVersions, t)
+		}
+		return nil
+	})
+	return
+}
+
+func collectTopicVersionsPartitions(filerClient filer_pb.FilerClient, t topic.Topic, topicVersion time.Time) (partitions []topic.Partition, err error) {
+	version := topicVersion.Format(topic.PartitionGenerationFormat)
+	err = filer_pb.ReadDirAllEntries(context.Background(), filerClient, util.FullPath(t.Dir()).Child(version), "", func(entry *filer_pb.Entry, isLast bool) error {
+		if !entry.IsDirectory {
+			return nil
+		}
+		start, stop := topic.ParsePartitionBoundary(entry.Name)
+		if start != stop {
+			partitions = append(partitions, topic.Partition{
+				RangeStart: start,
+				RangeStop:  stop,
+				RingSize:   topic.PartitionCount,
+				UnixTimeNs: topicVersion.UnixNano(),
+			})
+		}
+		return nil
+	})
+	return
+}
+
+func compactTopicPartition(filerClient filer_pb.FilerClient, t topic.Topic, timeAgo time.Duration, recordType *schema_pb.RecordType, partition topic.Partition, preference *operation.StoragePreference) error {
+	partitionDir := topic.PartitionDir(t, partition)
+
+	// compact the partition directory
+	return compactTopicPartitionDir(filerClient, t.Name, partitionDir, timeAgo, recordType, preference)
+}
+
+func compactTopicPartitionDir(filerClient filer_pb.FilerClient, topicName, partitionDir string, timeAgo time.Duration, recordType *schema_pb.RecordType, preference *operation.StoragePreference) error {
+	// read all existing parquet files
+	minTsNs, maxTsNs, err := readAllParquetFiles(filerClient, partitionDir)
+	if err != nil {
+		return err
+	}
+
+	// read all log files
+	logFiles, err := readAllLogFiles(filerClient, partitionDir, timeAgo, minTsNs, maxTsNs)
+	if err != nil {
+		return err
+	}
+	if len(logFiles) == 0 {
+		return nil
+	}
+
+	// divide log files into groups of 128MB
+	logFileGroups := groupFilesBySize(logFiles, 128*1024*1024)
+
+	// write to parquet file
+	parquetLevels, err := schema.ToParquetLevels(recordType)
+	if err != nil {
+		return fmt.Errorf("ToParquetLevels failed %+v: %v", recordType, err)
+	}
+
+	// create a parquet schema
+	parquetSchema, err := schema.ToParquetSchema(topicName, recordType)
+	if err != nil {
+		return fmt.Errorf("ToParquetSchema failed: %w", err)
+	}
+
+	// TODO parallelize the writing
+	for _, logFileGroup := range logFileGroups {
+		if err = writeLogFilesToParquet(filerClient, partitionDir, recordType, logFileGroup, parquetSchema, parquetLevels, preference); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func groupFilesBySize(logFiles []*filer_pb.Entry, maxGroupSize int64) (logFileGroups [][]*filer_pb.Entry) {
+	var logFileGroup []*filer_pb.Entry
+	var groupSize int64
+	for _, logFile := range logFiles {
+		if groupSize+int64(logFile.Attributes.FileSize) > maxGroupSize {
+			logFileGroups = append(logFileGroups, logFileGroup)
+			logFileGroup = nil
+			groupSize = 0
+		}
+		logFileGroup = append(logFileGroup, logFile)
+		groupSize += int64(logFile.Attributes.FileSize)
+	}
+	if len(logFileGroup) > 0 {
+		logFileGroups = append(logFileGroups, logFileGroup)
+	}
+	return
+}
+
+func readAllLogFiles(filerClient filer_pb.FilerClient, partitionDir string, timeAgo time.Duration, minTsNs, maxTsNs int64) (logFiles []*filer_pb.Entry, err error) {
+	err = filer_pb.ReadDirAllEntries(context.Background(), filerClient, util.FullPath(partitionDir), "", func(entry *filer_pb.Entry, isLast bool) error {
+		if strings.HasSuffix(entry.Name, ".parquet") {
+			return nil
+		}
+		if entry.Attributes.Crtime > time.Now().Unix()-int64(timeAgo/time.Second) {
+			return nil
+		}
+		logTime, err := time.Parse(topic.TIME_FORMAT, entry.Name)
+		if err != nil {
+			// glog.Warningf("parse log time %s: %v", entry.Name, err)
+			return nil
+		}
+		if maxTsNs > 0 && logTime.UnixNano() <= maxTsNs {
+			return nil
+		}
+		logFiles = append(logFiles, entry)
+		return nil
+	})
+	return
+}
+
+func readAllParquetFiles(filerClient filer_pb.FilerClient, partitionDir string) (minTsNs, maxTsNs int64, err error) {
+	err = filer_pb.ReadDirAllEntries(context.Background(), filerClient, util.FullPath(partitionDir), "", func(entry *filer_pb.Entry, isLast bool) error {
+		if !strings.HasSuffix(entry.Name, ".parquet") {
+			return nil
+		}
+		if len(entry.Extended) == 0 {
+			return nil
+		}
+
+		// read min ts
+		minTsBytes := entry.Extended[mq.ExtendedAttrTimestampMin]
+		if len(minTsBytes) != 8 {
+			return nil
+		}
+		minTs := int64(binary.BigEndian.Uint64(minTsBytes))
+		if minTsNs == 0 || minTs < minTsNs {
+			minTsNs = minTs
+		}
+
+		// read max ts
+		maxTsBytes := entry.Extended[mq.ExtendedAttrTimestampMax]
+		if len(maxTsBytes) != 8 {
+			return nil
+		}
+		maxTs := int64(binary.BigEndian.Uint64(maxTsBytes))
+		if maxTsNs == 0 || maxTs > maxTsNs {
+			maxTsNs = maxTs
+		}
+		return nil
+	})
+	return
+}
+
+// isSchemalessRecordType checks if the recordType represents a schema-less topic
+// Schema-less topics only have system fields: _ts_ns, _key, and _value
+func isSchemalessRecordType(recordType *schema_pb.RecordType) bool {
+	if recordType == nil {
+		return false
+	}
+
+	// Count only non-system data fields (exclude _ts_ns and _key which are always present)
+	// Schema-less topics should only have _value as the data field
+	hasValue := false
+	dataFieldCount := 0
+
+	for _, field := range recordType.Fields {
+		switch field.Name {
+		case SW_COLUMN_NAME_TS, SW_COLUMN_NAME_KEY, SW_COLUMN_NAME_OFFSET:
+			// System fields - ignore
+			continue
+		case SW_COLUMN_NAME_VALUE:
+			hasValue = true
+			dataFieldCount++
+		default:
+			// Any other field means it's not schema-less
+			dataFieldCount++
+		}
+	}
+
+	// Schema-less = only has _value field as the data field (plus system fields)
+	return hasValue && dataFieldCount == 1
+}
+
+func writeLogFilesToParquet(filerClient filer_pb.FilerClient, partitionDir string, recordType *schema_pb.RecordType, logFileGroups []*filer_pb.Entry, parquetSchema *parquet.Schema, parquetLevels *schema.ParquetLevels, preference *operation.StoragePreference) (err error) {
+
+	tempFile, err := os.CreateTemp(".", "t*.parquet")
+	if err != nil {
+		return fmt.Errorf("create temp file: %w", err)
+	}
+	defer func() {
+		tempFile.Close()
+		os.Remove(tempFile.Name())
+	}()
+
+	// Enable column statistics for fast aggregation queries
+	writer := parquet.NewWriter(tempFile, parquetSchema,
+		parquet.Compression(&zstd.Codec{Level: zstd.DefaultLevel}),
+		parquet.DataPageStatistics(true), // Enable column statistics
+	)
+	rowBuilder := parquet.NewRowBuilder(parquetSchema)
+
+	var startTsNs, stopTsNs int64
+	var minOffset, maxOffset int64
+	var hasOffsets bool
+	isSchemaless := isSchemalessRecordType(recordType)
+
+	for _, logFile := range logFileGroups {
+		var rows []parquet.Row
+		if err := iterateLogEntries(filerClient, logFile, func(entry *filer_pb.LogEntry) error {
+
+			// Skip control entries without actual data (same logic as read operations)
+			if isControlEntry(entry) {
+				return nil
+			}
+
+			if startTsNs == 0 {
+				startTsNs = entry.TsNs
+			}
+			stopTsNs = entry.TsNs
+
+			// Track offset ranges for Kafka integration
+			if entry.Offset > 0 {
+				if !hasOffsets {
+					minOffset = entry.Offset
+					maxOffset = entry.Offset
+					hasOffsets = true
+				} else {
+					if entry.Offset < minOffset {
+						minOffset = entry.Offset
+					}
+					if entry.Offset > maxOffset {
+						maxOffset = entry.Offset
+					}
+				}
+			}
+
+			// write to parquet file
+			rowBuilder.Reset()
+
+			record := &schema_pb.RecordValue{}
+
+			if isSchemaless {
+				// For schema-less topics, put raw entry.Data into _value field
+				record.Fields = make(map[string]*schema_pb.Value)
+				record.Fields[SW_COLUMN_NAME_VALUE] = &schema_pb.Value{
+					Kind: &schema_pb.Value_BytesValue{
+						BytesValue: entry.Data,
+					},
+				}
+			} else {
+				// For schematized topics, unmarshal entry.Data as RecordValue
+				if err := proto.Unmarshal(entry.Data, record); err != nil {
+					return fmt.Errorf("unmarshal record value: %w", err)
+				}
+
+				// Initialize Fields map if nil (prevents nil map assignment panic)
+				if record.Fields == nil {
+					record.Fields = make(map[string]*schema_pb.Value)
+				}
+
+				// Add offset field to parquet records for native offset support
+				// ASSUMPTION: LogEntry.Offset field is populated by broker during message publishing
+				record.Fields[SW_COLUMN_NAME_OFFSET] = &schema_pb.Value{
+					Kind: &schema_pb.Value_Int64Value{
+						Int64Value: entry.Offset,
+					},
+				}
+			}
+
+			// Add system columns (for both schematized and schema-less topics)
+			record.Fields[SW_COLUMN_NAME_TS] = &schema_pb.Value{
+				Kind: &schema_pb.Value_Int64Value{
+					Int64Value: entry.TsNs,
+				},
+			}
+
+			// Handle nil key bytes to prevent growslice panic in parquet-go
+			keyBytes := entry.Key
+			if keyBytes == nil {
+				keyBytes = []byte{} // Use empty slice instead of nil
+			}
+			record.Fields[SW_COLUMN_NAME_KEY] = &schema_pb.Value{
+				Kind: &schema_pb.Value_BytesValue{
+					BytesValue: keyBytes,
+				},
+			}
+
+			if err := schema.AddRecordValue(rowBuilder, recordType, parquetLevels, record); err != nil {
+				return fmt.Errorf("add record value: %w", err)
+			}
+
+			// Build row and normalize any nil ByteArray values to empty slices
+			row := rowBuilder.Row()
+			for i, value := range row {
+				if value.Kind() == parquet.ByteArray {
+					if value.ByteArray() == nil {
+						row[i] = parquet.ByteArrayValue([]byte{})
+					}
+				}
+			}
+
+			rows = append(rows, row)
+
+			return nil
+
+		}); err != nil {
+			return fmt.Errorf("iterate log entry %v/%v: %w", partitionDir, logFile.Name, err)
+		}
+
+		// Nil ByteArray handling is done during row creation
+
+		// Write all rows in a single call
+		if _, err := writer.WriteRows(rows); err != nil {
+			return fmt.Errorf("write rows: %w", err)
+		}
+	}
+
+	if err := writer.Close(); err != nil {
+		return fmt.Errorf("close writer: %w", err)
+	}
+
+	// write to parquet file to partitionDir
+	parquetFileName := fmt.Sprintf("%s.parquet", time.Unix(0, startTsNs).UTC().Format("2006-01-02-15-04-05"))
+
+	// Collect source log file names and buffer_start metadata for deduplication
+	var sourceLogFiles []string
+	var earliestBufferStart int64
+	for _, logFile := range logFileGroups {
+		sourceLogFiles = append(sourceLogFiles, logFile.Name)
+
+		// Extract buffer_start from log file metadata
+		if bufferStart := getBufferStartFromLogFile(logFile); bufferStart > 0 {
+			if earliestBufferStart == 0 || bufferStart < earliestBufferStart {
+				earliestBufferStart = bufferStart
+			}
+		}
+	}
+
+	if err := saveParquetFileToPartitionDir(filerClient, tempFile, partitionDir, parquetFileName, preference, startTsNs, stopTsNs, sourceLogFiles, earliestBufferStart, minOffset, maxOffset, hasOffsets); err != nil {
+		return fmt.Errorf("save parquet file %s: %v", parquetFileName, err)
+	}
+
+	return nil
+
+}
+
+func saveParquetFileToPartitionDir(filerClient filer_pb.FilerClient, sourceFile *os.File, partitionDir, parquetFileName string, preference *operation.StoragePreference, startTsNs, stopTsNs int64, sourceLogFiles []string, earliestBufferStart int64, minOffset, maxOffset int64, hasOffsets bool) error {
+	uploader, err := operation.NewUploader()
+	if err != nil {
+		return fmt.Errorf("new uploader: %w", err)
+	}
+
+	// get file size
+	fileInfo, err := sourceFile.Stat()
+	if err != nil {
+		return fmt.Errorf("stat source file: %w", err)
+	}
+
+	// upload file in chunks
+	chunkSize := int64(4 * 1024 * 1024)
+	chunkCount := (fileInfo.Size() + chunkSize - 1) / chunkSize
+	entry := &filer_pb.Entry{
+		Name: parquetFileName,
+		Attributes: &filer_pb.FuseAttributes{
+			Crtime:   time.Now().Unix(),
+			Mtime:    time.Now().Unix(),
+			FileMode: uint32(os.FileMode(0644)),
+			FileSize: uint64(fileInfo.Size()),
+			Mime:     "application/vnd.apache.parquet",
+		},
+	}
+	entry.Extended = make(map[string][]byte)
+	minTsBytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(minTsBytes, uint64(startTsNs))
+	entry.Extended[mq.ExtendedAttrTimestampMin] = minTsBytes
+	maxTsBytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(maxTsBytes, uint64(stopTsNs))
+	entry.Extended[mq.ExtendedAttrTimestampMax] = maxTsBytes
+
+	// Add offset range metadata for Kafka integration (same as regular log files)
+	if hasOffsets && minOffset > 0 && maxOffset >= minOffset {
+		minOffsetBytes := make([]byte, 8)
+		binary.BigEndian.PutUint64(minOffsetBytes, uint64(minOffset))
+		entry.Extended[mq.ExtendedAttrOffsetMin] = minOffsetBytes
+
+		maxOffsetBytes := make([]byte, 8)
+		binary.BigEndian.PutUint64(maxOffsetBytes, uint64(maxOffset))
+		entry.Extended[mq.ExtendedAttrOffsetMax] = maxOffsetBytes
+	}
+
+	// Store source log files for deduplication (JSON-encoded list)
+	if len(sourceLogFiles) > 0 {
+		sourceLogFilesJson, _ := json.Marshal(sourceLogFiles)
+		entry.Extended[mq.ExtendedAttrSources] = sourceLogFilesJson
+	}
+
+	// Store earliest buffer_start for precise broker deduplication
+	if earliestBufferStart > 0 {
+		bufferStartBytes := make([]byte, 8)
+		binary.BigEndian.PutUint64(bufferStartBytes, uint64(earliestBufferStart))
+		entry.Extended[mq.ExtendedAttrBufferStart] = bufferStartBytes
+	}
+
+	for i := int64(0); i < chunkCount; i++ {
+		fileId, uploadResult, err, _ := uploader.UploadWithRetry(
+			filerClient,
+			&filer_pb.AssignVolumeRequest{
+				Count:       1,
+				Replication: preference.Replication,
+				Collection:  preference.Collection,
+				TtlSec:      0, // TODO set ttl
+				DiskType:    preference.DiskType,
+				Path:        partitionDir + "/" + parquetFileName,
+			},
+			&operation.UploadOption{
+				Filename:          parquetFileName,
+				Cipher:            false,
+				IsInputCompressed: false,
+				MimeType:          "application/vnd.apache.parquet",
+				PairMap:           nil,
+			},
+			io.NewSectionReader(sourceFile, i*chunkSize, chunkSize),
+		)
+		if err != nil {
+			return fmt.Errorf("upload chunk %d: %v", i, err)
+		}
+		if uploadResult.Error != "" {
+			return fmt.Errorf("upload result: %v", uploadResult.Error)
+		}
+		entry.Chunks = append(entry.Chunks, uploadResult.ToPbFileChunk(fileId, i*chunkSize, time.Now().UnixNano()))
+	}
+
+	// write the entry to partitionDir
+	if err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		return filer_pb.CreateEntry(context.Background(), client, &filer_pb.CreateEntryRequest{
+			Directory: partitionDir,
+			Entry:     entry,
+		})
+	}); err != nil {
+		return fmt.Errorf("create entry: %w", err)
+	}
+
+	return nil
+}
+
+func iterateLogEntries(filerClient filer_pb.FilerClient, logFile *filer_pb.Entry, eachLogEntryFn func(entry *filer_pb.LogEntry) error) error {
+	lookupFn := filer.LookupFn(filerClient)
+	_, err := eachFile(logFile, lookupFn, func(logEntry *filer_pb.LogEntry) (isDone bool, err error) {
+		if err := eachLogEntryFn(logEntry); err != nil {
+			return true, err
+		}
+		return false, nil
+	})
+	return err
+}
+
+func eachFile(entry *filer_pb.Entry, lookupFileIdFn func(ctx context.Context, fileId string) (targetUrls []string, err error), eachLogEntryFn log_buffer.EachLogEntryFuncType) (processedTsNs int64, err error) {
+	if len(entry.Content) > 0 {
+		// skip .offset files
+		return
+	}
+	var urlStrings []string
+	for _, chunk := range entry.Chunks {
+		if chunk.Size == 0 {
+			continue
+		}
+		if chunk.IsChunkManifest {
+			return
+		}
+		urlStrings, err = lookupFileIdFn(context.Background(), chunk.FileId)
+		if err != nil {
+			err = fmt.Errorf("lookup %s: %v", chunk.FileId, err)
+			return
+		}
+		if len(urlStrings) == 0 {
+			err = fmt.Errorf("no url found for %s", chunk.FileId)
+			return
+		}
+
+		// try one of the urlString until util.Get(urlString) succeeds
+		var processed bool
+		for _, urlString := range urlStrings {
+			var data []byte
+			if data, _, err = util_http.Get(urlString); err == nil {
+				processed = true
+				if processedTsNs, err = eachChunk(data, eachLogEntryFn); err != nil {
+					return
+				}
+				break
+			}
+		}
+		if !processed {
+			err = fmt.Errorf("no data processed for %s %s", entry.Name, chunk.FileId)
+			return
+		}
+
+	}
+	return
+}
+
+func eachChunk(buf []byte, eachLogEntryFn log_buffer.EachLogEntryFuncType) (processedTsNs int64, err error) {
+	for pos := 0; pos+4 < len(buf); {
+
+		size := util.BytesToUint32(buf[pos : pos+4])
+		if pos+4+int(size) > len(buf) {
+			err = fmt.Errorf("reach each log chunk: read [%d,%d) from [0,%d)", pos, pos+int(size)+4, len(buf))
+			return
+		}
+		entryData := buf[pos+4 : pos+4+int(size)]
+
+		logEntry := &filer_pb.LogEntry{}
+		if err = proto.Unmarshal(entryData, logEntry); err != nil {
+			pos += 4 + int(size)
+			err = fmt.Errorf("unexpected unmarshal mq_pb.Message: %w", err)
+			return
+		}
+
+		if _, err = eachLogEntryFn(logEntry); err != nil {
+			err = fmt.Errorf("process log entry %v: %w", logEntry, err)
+			return
+		}
+
+		processedTsNs = logEntry.TsNs
+
+		pos += 4 + int(size)
+
+	}
+
+	return
+}
+
+// getBufferStartFromLogFile extracts the buffer_start index from log file extended metadata
+func getBufferStartFromLogFile(logFile *filer_pb.Entry) int64 {
+	if logFile.Extended == nil {
+		return 0
+	}
+
+	// Parse buffer_start binary format
+	if startData, exists := logFile.Extended["buffer_start"]; exists {
+		if len(startData) == 8 {
+			startIndex := int64(binary.BigEndian.Uint64(startData))
+			if startIndex > 0 {
+				return startIndex
+			}
+		}
+	}
+
+	return 0
+}

weed/mq/logstore/merged_read.go

@@ -0,0 +1,50 @@
+package logstore
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+)
+
+func GenMergedReadFunc(filerClient filer_pb.FilerClient, t topic.Topic, p topic.Partition) log_buffer.LogReadFromDiskFuncType {
+	fromParquetFn := GenParquetReadFunc(filerClient, t, p)
+	readLogDirectFn := GenLogOnDiskReadFunc(filerClient, t, p)
+	// Reversed order: live logs first (recent), then Parquet files (historical)
+	// This provides better performance for real-time analytics queries
+	return mergeReadFuncs(readLogDirectFn, fromParquetFn)
+}
+
+func mergeReadFuncs(readLogDirectFn, fromParquetFn log_buffer.LogReadFromDiskFuncType) log_buffer.LogReadFromDiskFuncType {
+	// CRITICAL FIX: Removed stateful closure variables (exhaustedLiveLogs, lastProcessedPosition)
+	// These caused the function to skip disk reads on subsequent calls, leading to
+	// Schema Registry timeout when data was flushed after the first read attempt.
+	// The function must be stateless and check for data on EVERY call.
+	return func(startPosition log_buffer.MessagePosition, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (lastReadPosition log_buffer.MessagePosition, isDone bool, err error) {
+		// Always try reading from live logs first (recent data)
+		lastReadPosition, isDone, err = readLogDirectFn(startPosition, stopTsNs, eachLogEntryFn)
+		if isDone {
+			// For very early timestamps (like timestamp=1 for RESET_TO_EARLIEST),
+			// we want to continue to read from in-memory data
+			isDone = false
+		}
+		if err != nil {
+			return
+		}
+
+		// If live logs returned data, update startPosition for parquet read
+		if lastReadPosition.Offset > startPosition.Offset || lastReadPosition.Time.After(startPosition.Time) {
+			startPosition = lastReadPosition
+		}
+
+		// Then try reading from Parquet files (historical data)
+		lastReadPosition, isDone, err = fromParquetFn(startPosition, stopTsNs, eachLogEntryFn)
+
+		if isDone {
+			// For very early timestamps (like timestamp=1 for RESET_TO_EARLIEST),
+			// parquet files won't exist, but we want to continue to in-memory data reading
+			isDone = false
+		}
+
+		return
+	}
+}

weed/mq/logstore/read_log_from_disk.go

@@ -0,0 +1,340 @@
+package logstore
+
+import (
+	"context"
+	"encoding/binary"
+	"fmt"
+	"math"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+	"google.golang.org/protobuf/proto"
+)
+
+func GenLogOnDiskReadFunc(filerClient filer_pb.FilerClient, t topic.Topic, p topic.Partition) log_buffer.LogReadFromDiskFuncType {
+	partitionDir := topic.PartitionDir(t, p)
+
+	// Create a small cache for recently-read file chunks (3 files, 60s TTL)
+	// This significantly reduces Filer load when multiple consumers are catching up
+	fileCache := log_buffer.NewDiskBufferCache(3, 60*time.Second)
+
+	lookupFileIdFn := filer.LookupFn(filerClient)
+
+	eachChunkFn := func(buf []byte, eachLogEntryFn log_buffer.EachLogEntryFuncType, starTsNs, stopTsNs int64, startOffset int64, isOffsetBased bool) (processedTsNs int64, err error) {
+		entriesSkipped := 0
+		entriesProcessed := 0
+		for pos := 0; pos+4 < len(buf); {
+
+			size := util.BytesToUint32(buf[pos : pos+4])
+			if pos+4+int(size) > len(buf) {
+				err = fmt.Errorf("GenLogOnDiskReadFunc: read [%d,%d) from [0,%d)", pos, pos+int(size)+4, len(buf))
+				return
+			}
+			entryData := buf[pos+4 : pos+4+int(size)]
+
+			logEntry := &filer_pb.LogEntry{}
+			if err = proto.Unmarshal(entryData, logEntry); err != nil {
+				pos += 4 + int(size)
+				err = fmt.Errorf("unexpected unmarshal mq_pb.Message: %w", err)
+				return
+			}
+
+			// Filter by offset if this is an offset-based subscription
+			if isOffsetBased {
+				if logEntry.Offset < startOffset {
+					entriesSkipped++
+					pos += 4 + int(size)
+					continue
+				}
+			} else {
+				// Filter by timestamp for timestamp-based subscriptions
+				if logEntry.TsNs <= starTsNs {
+					pos += 4 + int(size)
+					continue
+				}
+				if stopTsNs != 0 && logEntry.TsNs > stopTsNs {
+					println("stopTsNs", stopTsNs, "logEntry.TsNs", logEntry.TsNs)
+					return
+				}
+			}
+
+			// fmt.Printf(" read logEntry: %v, ts %v\n", string(logEntry.Key), time.Unix(0, logEntry.TsNs).UTC())
+			if _, err = eachLogEntryFn(logEntry); err != nil {
+				err = fmt.Errorf("process log entry %v: %w", logEntry, err)
+				return
+			}
+
+			processedTsNs = logEntry.TsNs
+			entriesProcessed++
+
+			pos += 4 + int(size)
+
+		}
+
+		return
+	}
+
+	eachFileFn := func(entry *filer_pb.Entry, eachLogEntryFn log_buffer.EachLogEntryFuncType, starTsNs, stopTsNs int64, startOffset int64, isOffsetBased bool) (processedTsNs int64, err error) {
+		if len(entry.Content) > 0 {
+			// skip .offset files
+			return
+		}
+		var urlStrings []string
+		for _, chunk := range entry.Chunks {
+			if chunk.Size == 0 {
+				continue
+			}
+			if chunk.IsChunkManifest {
+				glog.Warningf("this should not happen. unexpected chunk manifest in %s/%s", partitionDir, entry.Name)
+				return
+			}
+			urlStrings, err = lookupFileIdFn(context.Background(), chunk.FileId)
+			if err != nil {
+				glog.V(1).Infof("lookup %s failed: %v", chunk.FileId, err)
+				err = fmt.Errorf("lookup %s: %v", chunk.FileId, err)
+				return
+			}
+			if len(urlStrings) == 0 {
+				glog.V(1).Infof("no url found for %s", chunk.FileId)
+				err = fmt.Errorf("no url found for %s", chunk.FileId)
+				return
+			}
+			glog.V(2).Infof("lookup %s returned %d URLs", chunk.FileId, len(urlStrings))
+
+			// Try to get data from cache first
+			cacheKey := fmt.Sprintf("%s/%s/%d/%s", t.Name, p.String(), p.RangeStart, chunk.FileId)
+			if cachedData, _, found := fileCache.Get(cacheKey); found {
+				if cachedData == nil {
+					// Negative cache hit - data doesn't exist
+					continue
+				}
+				// Positive cache hit - data exists
+				if processedTsNs, err = eachChunkFn(cachedData, eachLogEntryFn, starTsNs, stopTsNs, startOffset, isOffsetBased); err != nil {
+					glog.V(1).Infof("eachChunkFn failed on cached data: %v", err)
+					return
+				}
+				continue
+			}
+
+			// Cache miss - try one of the urlString until util.Get(urlString) succeeds
+			var processed bool
+			for _, urlString := range urlStrings {
+				// TODO optimization opportunity: reuse the buffer
+				var data []byte
+				glog.V(2).Infof("trying to fetch data from %s", urlString)
+				if data, _, err = util_http.Get(urlString); err == nil {
+					glog.V(2).Infof("successfully fetched %d bytes from %s", len(data), urlString)
+					processed = true
+
+					// Store in cache for future reads
+					fileCache.Put(cacheKey, data, startOffset)
+
+					if processedTsNs, err = eachChunkFn(data, eachLogEntryFn, starTsNs, stopTsNs, startOffset, isOffsetBased); err != nil {
+						glog.V(1).Infof("eachChunkFn failed: %v", err)
+						return
+					}
+					break
+				} else {
+					glog.V(2).Infof("failed to fetch from %s: %v", urlString, err)
+				}
+			}
+			if !processed {
+				// Store negative cache entry - data doesn't exist or all URLs failed
+				fileCache.Put(cacheKey, nil, startOffset)
+				glog.V(1).Infof("no data processed for %s %s - all URLs failed", entry.Name, chunk.FileId)
+				err = fmt.Errorf("no data processed for %s %s", entry.Name, chunk.FileId)
+				return
+			}
+
+		}
+		return
+	}
+
+	return func(startPosition log_buffer.MessagePosition, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (lastReadPosition log_buffer.MessagePosition, isDone bool, err error) {
+		startFileName := startPosition.Time.UTC().Format(topic.TIME_FORMAT)
+		startTsNs := startPosition.Time.UnixNano()
+		stopTime := time.Unix(0, stopTsNs)
+		var processedTsNs int64
+
+		// Check if this is an offset-based subscription
+		isOffsetBased := startPosition.IsOffsetBased
+		var startOffset int64
+		if isOffsetBased {
+			startOffset = startPosition.Offset
+			// CRITICAL FIX: For offset-based reads, ignore startFileName (which is based on Time)
+			// and list all files from the beginning to find the right offset
+			startFileName = ""
+			glog.V(1).Infof("disk read start: topic=%s partition=%s startOffset=%d",
+				t.Name, p, startOffset)
+		}
+
+		// OPTIMIZATION: For offset-based reads, collect all files with their offset ranges first
+		// Then use binary search to find the right file, and skip files that don't contain the offset
+		var candidateFiles []*filer_pb.Entry
+		var foundStartFile bool
+
+		err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+			// First pass: collect all relevant files with their metadata
+			glog.V(2).Infof("listing directory %s for offset %d startFileName=%q", partitionDir, startOffset, startFileName)
+			return filer_pb.SeaweedList(context.Background(), client, partitionDir, "", func(entry *filer_pb.Entry, isLast bool) error {
+
+				if entry.IsDirectory {
+					return nil
+				}
+				if strings.HasSuffix(entry.Name, ".parquet") {
+					return nil
+				}
+				if strings.HasSuffix(entry.Name, ".offset") {
+					return nil
+				}
+				if stopTsNs != 0 && entry.Name > stopTime.UTC().Format(topic.TIME_FORMAT) {
+					return nil
+				}
+
+				// OPTIMIZATION: For offset-based reads, check if this file contains the requested offset
+				if isOffsetBased {
+					glog.V(3).Infof("found file %s", entry.Name)
+					// Check if file has offset range metadata
+					if minOffsetBytes, hasMin := entry.Extended["offset_min"]; hasMin && len(minOffsetBytes) == 8 {
+						if maxOffsetBytes, hasMax := entry.Extended["offset_max"]; hasMax && len(maxOffsetBytes) == 8 {
+							fileMinOffset := int64(binary.BigEndian.Uint64(minOffsetBytes))
+							fileMaxOffset := int64(binary.BigEndian.Uint64(maxOffsetBytes))
+
+							// Skip files that don't contain our offset range
+							if startOffset > fileMaxOffset {
+								return nil
+							}
+
+							// If we haven't found the start file yet, check if this file contains it
+							if !foundStartFile && startOffset >= fileMinOffset && startOffset <= fileMaxOffset {
+								foundStartFile = true
+							}
+						}
+					}
+					// If file doesn't have offset metadata, include it (might be old format)
+				} else {
+					// Timestamp-based filtering
+					topicName := t.Name
+					if dotIndex := strings.LastIndex(topicName, "."); dotIndex != -1 {
+						topicName = topicName[dotIndex+1:]
+					}
+					isSystemTopic := strings.HasPrefix(topicName, "_")
+					if !isSystemTopic && startPosition.Time.Unix() > 86400 && entry.Name < startPosition.Time.UTC().Format(topic.TIME_FORMAT) {
+						return nil
+					}
+				}
+
+				// Add file to candidates for processing
+				candidateFiles = append(candidateFiles, entry)
+				glog.V(3).Infof("added candidate file %s (total=%d)", entry.Name, len(candidateFiles))
+				return nil
+
+			}, startFileName, true, math.MaxInt32)
+		})
+
+		if err != nil {
+			glog.Errorf("failed to list directory %s: %v", partitionDir, err)
+			return
+		}
+
+		glog.V(2).Infof("found %d candidate files for topic=%s partition=%s offset=%d",
+			len(candidateFiles), t.Name, p, startOffset)
+
+		if len(candidateFiles) == 0 {
+			glog.V(2).Infof("no files found in %s", partitionDir)
+			return startPosition, isDone, nil
+		}
+
+		// OPTIMIZATION: For offset-based reads with many files, use binary search to find start file
+		if isOffsetBased && len(candidateFiles) > 10 {
+			// Binary search to find the first file that might contain our offset
+			left, right := 0, len(candidateFiles)-1
+			startIdx := 0
+
+			for left <= right {
+				mid := (left + right) / 2
+				entry := candidateFiles[mid]
+
+				if minOffsetBytes, hasMin := entry.Extended["offset_min"]; hasMin && len(minOffsetBytes) == 8 {
+					if maxOffsetBytes, hasMax := entry.Extended["offset_max"]; hasMax && len(maxOffsetBytes) == 8 {
+						fileMinOffset := int64(binary.BigEndian.Uint64(minOffsetBytes))
+						fileMaxOffset := int64(binary.BigEndian.Uint64(maxOffsetBytes))
+
+						if startOffset < fileMinOffset {
+							// Our offset is before this file, search left
+							right = mid - 1
+						} else if startOffset > fileMaxOffset {
+							// Our offset is after this file, search right
+							left = mid + 1
+							startIdx = left
+						} else {
+							// Found the file containing our offset
+							startIdx = mid
+							break
+						}
+					} else {
+						break
+					}
+				} else {
+					break
+				}
+			}
+
+			// Process files starting from the found index
+			candidateFiles = candidateFiles[startIdx:]
+		}
+
+		// Second pass: process the filtered files
+		// CRITICAL: For offset-based reads, process ALL candidate files in one call
+		// This prevents multiple ReadFromDiskFn calls with 1.127s overhead each
+		var filesProcessed int
+		var lastProcessedOffset int64
+		for _, entry := range candidateFiles {
+			var fileTsNs int64
+			if fileTsNs, err = eachFileFn(entry, eachLogEntryFn, startTsNs, stopTsNs, startOffset, isOffsetBased); err != nil {
+				return lastReadPosition, isDone, err
+			}
+			if fileTsNs > 0 {
+				processedTsNs = fileTsNs
+				filesProcessed++
+			}
+
+			// For offset-based reads, track the last processed offset
+			// We need to continue reading ALL files to avoid multiple disk read calls
+			if isOffsetBased {
+				// Extract the last offset from the file's extended attributes
+				if maxOffsetBytes, hasMax := entry.Extended["offset_max"]; hasMax && len(maxOffsetBytes) == 8 {
+					fileMaxOffset := int64(binary.BigEndian.Uint64(maxOffsetBytes))
+					if fileMaxOffset > lastProcessedOffset {
+						lastProcessedOffset = fileMaxOffset
+					}
+				}
+			}
+		}
+
+		if isOffsetBased && filesProcessed > 0 {
+			// Return a position that indicates we've read all disk data up to lastProcessedOffset
+			// This prevents the subscription from calling ReadFromDiskFn again for these offsets
+			lastReadPosition = log_buffer.NewMessagePositionFromOffset(lastProcessedOffset + 1)
+		} else {
+			// CRITICAL FIX: If no files were processed (e.g., all data already consumed),
+			// return the requested offset to prevent busy loop
+			if isOffsetBased {
+				// For offset-based reads with no data, return the requested offset
+				// This signals "I've checked, there's no data at this offset, move forward"
+				lastReadPosition = log_buffer.NewMessagePositionFromOffset(startOffset)
+			} else {
+				// For timestamp-based reads, return error (-2)
+				lastReadPosition = log_buffer.NewMessagePosition(processedTsNs, -2)
+			}
+		}
+		return
+	}
+}

weed/mq/logstore/read_parquet_to_log.go

@@ -0,0 +1,222 @@
+package logstore
+
+import (
+	"context"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"math"
+	"strings"
+
+	"github.com/parquet-go/parquet-go"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/mq"
+	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+	"google.golang.org/protobuf/proto"
+)
+
+var (
+	chunkCache = chunk_cache.NewChunkCacheInMemory(256) // 256 entries, 8MB max per entry
+)
+
+// isControlEntry checks if a log entry is a control entry without actual data
+// Based on MQ system analysis, control entries are:
+// 1. DataMessages with populated Ctrl field (publisher close signals)
+// 2. Entries with empty keys (as filtered by subscriber)
+// 3. Entries with no data
+func isControlEntry(logEntry *filer_pb.LogEntry) bool {
+	// Skip entries with no data
+	if len(logEntry.Data) == 0 {
+		return true
+	}
+
+	// Skip entries with empty keys (same logic as subscriber)
+	if len(logEntry.Key) == 0 {
+		return true
+	}
+
+	// Check if this is a DataMessage with control field populated
+	dataMessage := &mq_pb.DataMessage{}
+	if err := proto.Unmarshal(logEntry.Data, dataMessage); err == nil {
+		// If it has a control field, it's a control message
+		if dataMessage.Ctrl != nil {
+			return true
+		}
+	}
+
+	return false
+}
+
+func GenParquetReadFunc(filerClient filer_pb.FilerClient, t topic.Topic, p topic.Partition) log_buffer.LogReadFromDiskFuncType {
+	partitionDir := topic.PartitionDir(t, p)
+
+	lookupFileIdFn := filer.LookupFn(filerClient)
+
+	// read topic conf from filer
+	var topicConf *mq_pb.ConfigureTopicResponse
+	var err error
+	if err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		topicConf, err = t.ReadConfFile(client)
+		return err
+	}); err != nil {
+		// Return a no-op function for test environments or when topic config can't be read
+		return func(startPosition log_buffer.MessagePosition, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (log_buffer.MessagePosition, bool, error) {
+			return startPosition, true, nil
+		}
+	}
+	// Get schema - prefer flat schema if available
+	var recordType *schema_pb.RecordType
+	if topicConf.GetMessageRecordType() != nil {
+		// New flat schema format - use directly
+		recordType = topicConf.GetMessageRecordType()
+	}
+
+	if recordType == nil || len(recordType.Fields) == 0 {
+		// Return a no-op function if no schema is available
+		return func(startPosition log_buffer.MessagePosition, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (log_buffer.MessagePosition, bool, error) {
+			return startPosition, true, nil
+		}
+	}
+	recordType = schema.NewRecordTypeBuilder(recordType).
+		WithField(SW_COLUMN_NAME_TS, schema.TypeInt64).
+		WithField(SW_COLUMN_NAME_KEY, schema.TypeBytes).
+		WithField(SW_COLUMN_NAME_OFFSET, schema.TypeInt64).
+		RecordTypeEnd()
+
+	parquetLevels, err := schema.ToParquetLevels(recordType)
+	if err != nil {
+		return nil
+	}
+
+	// eachFileFn reads a parquet file and calls eachLogEntryFn for each log entry
+	eachFileFn := func(entry *filer_pb.Entry, eachLogEntryFn log_buffer.EachLogEntryFuncType, starTsNs, stopTsNs int64) (processedTsNs int64, err error) {
+		// create readerAt for the parquet file
+		fileSize := filer.FileSize(entry)
+		visibleIntervals, _ := filer.NonOverlappingVisibleIntervals(context.Background(), lookupFileIdFn, entry.Chunks, 0, int64(fileSize))
+		chunkViews := filer.ViewFromVisibleIntervals(visibleIntervals, 0, int64(fileSize))
+		readerCache := filer.NewReaderCache(32, chunkCache, lookupFileIdFn, nil)
+		readerAt := filer.NewChunkReaderAtFromClient(context.Background(), readerCache, chunkViews, int64(fileSize), filer.DefaultPrefetchCount)
+
+		// create parquet reader
+		parquetReader := parquet.NewReader(readerAt)
+		rows := make([]parquet.Row, 128)
+		for {
+			rowCount, readErr := parquetReader.ReadRows(rows)
+
+			// Process the rows first, even if EOF is returned
+			for i := 0; i < rowCount; i++ {
+				row := rows[i]
+				// convert parquet row to schema_pb.RecordValue
+				recordValue, err := schema.ToRecordValue(recordType, parquetLevels, row)
+				if err != nil {
+					return processedTsNs, fmt.Errorf("ToRecordValue failed: %w", err)
+				}
+				processedTsNs = recordValue.Fields[SW_COLUMN_NAME_TS].GetInt64Value()
+				if processedTsNs <= starTsNs {
+					continue
+				}
+				if stopTsNs != 0 && processedTsNs >= stopTsNs {
+					return processedTsNs, nil
+				}
+
+				data, marshalErr := proto.Marshal(recordValue)
+				if marshalErr != nil {
+					return processedTsNs, fmt.Errorf("marshal record value: %w", marshalErr)
+				}
+
+				// Get offset from parquet, default to 0 if not present (backward compatibility)
+				var offset int64 = 0
+				if offsetValue, exists := recordValue.Fields[SW_COLUMN_NAME_OFFSET]; exists {
+					offset = offsetValue.GetInt64Value()
+				}
+
+				logEntry := &filer_pb.LogEntry{
+					Key:    recordValue.Fields[SW_COLUMN_NAME_KEY].GetBytesValue(),
+					TsNs:   processedTsNs,
+					Data:   data,
+					Offset: offset,
+				}
+
+				// Skip control entries without actual data
+				if isControlEntry(logEntry) {
+					continue
+				}
+
+				// fmt.Printf(" parquet entry %s ts %v\n", string(logEntry.Key), time.Unix(0, logEntry.TsNs).UTC())
+
+				if _, err = eachLogEntryFn(logEntry); err != nil {
+					return processedTsNs, fmt.Errorf("process log entry %v: %w", logEntry, err)
+				}
+			}
+
+			// Check for end conditions after processing rows
+			if readErr != nil {
+				if readErr == io.EOF {
+					return processedTsNs, nil
+				}
+				return processedTsNs, readErr
+			}
+			if rowCount == 0 {
+				return processedTsNs, nil
+			}
+		}
+	}
+
+	return func(startPosition log_buffer.MessagePosition, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (lastReadPosition log_buffer.MessagePosition, isDone bool, err error) {
+		startFileName := startPosition.Time.UTC().Format(topic.TIME_FORMAT)
+		startTsNs := startPosition.Time.UnixNano()
+		var processedTsNs int64
+
+		err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+			return filer_pb.SeaweedList(context.Background(), client, partitionDir, "", func(entry *filer_pb.Entry, isLast bool) error {
+				if entry.IsDirectory {
+					return nil
+				}
+				if !strings.HasSuffix(entry.Name, ".parquet") {
+					return nil
+				}
+				if len(entry.Extended) == 0 {
+					return nil
+				}
+
+				// read minTs from the parquet file
+				minTsBytes := entry.Extended[mq.ExtendedAttrTimestampMin]
+				if len(minTsBytes) != 8 {
+					return nil
+				}
+				minTsNs := int64(binary.BigEndian.Uint64(minTsBytes))
+
+				// read max ts
+				maxTsBytes := entry.Extended[mq.ExtendedAttrTimestampMax]
+				if len(maxTsBytes) != 8 {
+					return nil
+				}
+				maxTsNs := int64(binary.BigEndian.Uint64(maxTsBytes))
+
+				if stopTsNs != 0 && stopTsNs <= minTsNs {
+					isDone = true
+					return nil
+				}
+
+				if maxTsNs < startTsNs {
+					return nil
+				}
+
+				if processedTsNs, err = eachFileFn(entry, eachLogEntryFn, startTsNs, stopTsNs); err != nil {
+					return err
+				}
+				return nil
+
+			}, startFileName, true, math.MaxInt32)
+		})
+		lastReadPosition = log_buffer.NewMessagePosition(processedTsNs, -2)
+		return
+	}
+}

weed/mq/logstore/write_rows_no_panic_test.go

@@ -0,0 +1,118 @@
+package logstore
+
+import (
+	"os"
+	"testing"
+
+	parquet "github.com/parquet-go/parquet-go"
+	"github.com/parquet-go/parquet-go/compress/zstd"
+	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+// TestWriteRowsNoPanic builds a representative schema and rows and ensures WriteRows completes without panic.
+func TestWriteRowsNoPanic(t *testing.T) {
+	// Build schema similar to ecommerce.user_events
+	recordType := schema.RecordTypeBegin().
+		WithField("id", schema.TypeInt64).
+		WithField("user_id", schema.TypeInt64).
+		WithField("user_type", schema.TypeString).
+		WithField("action", schema.TypeString).
+		WithField("status", schema.TypeString).
+		WithField("amount", schema.TypeDouble).
+		WithField("timestamp", schema.TypeString).
+		WithField("metadata", schema.TypeString).
+		RecordTypeEnd()
+
+	// Add log columns
+	recordType = schema.NewRecordTypeBuilder(recordType).
+		WithField(SW_COLUMN_NAME_TS, schema.TypeInt64).
+		WithField(SW_COLUMN_NAME_KEY, schema.TypeBytes).
+		RecordTypeEnd()
+
+	ps, err := schema.ToParquetSchema("synthetic", recordType)
+	if err != nil {
+		t.Fatalf("schema: %v", err)
+	}
+	levels, err := schema.ToParquetLevels(recordType)
+	if err != nil {
+		t.Fatalf("levels: %v", err)
+	}
+
+	tmp, err := os.CreateTemp(".", "synthetic*.parquet")
+	if err != nil {
+		t.Fatalf("tmp: %v", err)
+	}
+	defer func() {
+		tmp.Close()
+		os.Remove(tmp.Name())
+	}()
+
+	w := parquet.NewWriter(tmp, ps,
+		parquet.Compression(&zstd.Codec{Level: zstd.DefaultLevel}),
+		parquet.DataPageStatistics(true),
+	)
+	defer w.Close()
+
+	rb := parquet.NewRowBuilder(ps)
+	var rows []parquet.Row
+
+	// Build a few hundred rows with various optional/missing values and nil/empty keys
+	for i := 0; i < 200; i++ {
+		rb.Reset()
+
+		rec := &schema_pb.RecordValue{Fields: map[string]*schema_pb.Value{}}
+		// Required-like fields present
+		rec.Fields["id"] = &schema_pb.Value{Kind: &schema_pb.Value_Int64Value{Int64Value: int64(1000 + i)}}
+		rec.Fields["user_id"] = &schema_pb.Value{Kind: &schema_pb.Value_Int64Value{Int64Value: int64(i)}}
+		rec.Fields["user_type"] = &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: "standard"}}
+		rec.Fields["action"] = &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: "click"}}
+		rec.Fields["status"] = &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: "active"}}
+
+		// Optional fields vary: sometimes omitted, sometimes empty
+		if i%3 == 0 {
+			rec.Fields["amount"] = &schema_pb.Value{Kind: &schema_pb.Value_DoubleValue{DoubleValue: float64(i)}}
+		}
+		if i%4 == 0 {
+			rec.Fields["metadata"] = &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: ""}}
+		}
+		if i%5 == 0 {
+			rec.Fields["timestamp"] = &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: "2025-09-03T15:36:29Z"}}
+		}
+
+		// Log columns
+		rec.Fields[SW_COLUMN_NAME_TS] = &schema_pb.Value{Kind: &schema_pb.Value_Int64Value{Int64Value: 1756913789000000000 + int64(i)}}
+		var keyBytes []byte
+		if i%7 == 0 {
+			keyBytes = nil // ensure nil-keys are handled
+		} else if i%7 == 1 {
+			keyBytes = []byte{} // empty
+		} else {
+			keyBytes = []byte("key-")
+		}
+		rec.Fields[SW_COLUMN_NAME_KEY] = &schema_pb.Value{Kind: &schema_pb.Value_BytesValue{BytesValue: keyBytes}}
+
+		if err := schema.AddRecordValue(rb, recordType, levels, rec); err != nil {
+			t.Fatalf("add record: %v", err)
+		}
+		rows = append(rows, rb.Row())
+	}
+
+	deferredPanicked := false
+	defer func() {
+		if r := recover(); r != nil {
+			deferredPanicked = true
+			t.Fatalf("unexpected panic: %v", r)
+		}
+	}()
+
+	if _, err := w.WriteRows(rows); err != nil {
+		t.Fatalf("WriteRows: %v", err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatalf("Close: %v", err)
+	}
+	if deferredPanicked {
+		t.Fatal("panicked")
+	}
+}