snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/mq/schema/flat_schema_utils.go

@@ -0,0 +1,206 @@
+package schema
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+// SplitFlatSchemaToKeyValue takes a flat RecordType and key column names,
+// returns separate key and value RecordTypes
+func SplitFlatSchemaToKeyValue(flatSchema *schema_pb.RecordType, keyColumns []string) (*schema_pb.RecordType, *schema_pb.RecordType, error) {
+	if flatSchema == nil {
+		return nil, nil, nil
+	}
+
+	// Create maps for fast lookup
+	keyColumnSet := make(map[string]bool)
+	for _, col := range keyColumns {
+		keyColumnSet[col] = true
+	}
+
+	var keyFields []*schema_pb.Field
+	var valueFields []*schema_pb.Field
+
+	// Split fields based on key columns
+	for _, field := range flatSchema.Fields {
+		if keyColumnSet[field.Name] {
+			// Create key field with reindexed field index
+			keyField := &schema_pb.Field{
+				Name:       field.Name,
+				FieldIndex: int32(len(keyFields)),
+				Type:       field.Type,
+				IsRepeated: field.IsRepeated,
+				IsRequired: field.IsRequired,
+			}
+			keyFields = append(keyFields, keyField)
+		} else {
+			// Create value field with reindexed field index
+			valueField := &schema_pb.Field{
+				Name:       field.Name,
+				FieldIndex: int32(len(valueFields)),
+				Type:       field.Type,
+				IsRepeated: field.IsRepeated,
+				IsRequired: field.IsRequired,
+			}
+			valueFields = append(valueFields, valueField)
+		}
+	}
+
+	// Validate that all key columns were found
+	if len(keyFields) != len(keyColumns) {
+		missingCols := []string{}
+		for _, col := range keyColumns {
+			found := false
+			for _, field := range keyFields {
+				if field.Name == col {
+					found = true
+					break
+				}
+			}
+			if !found {
+				missingCols = append(missingCols, col)
+			}
+		}
+		if len(missingCols) > 0 {
+			return nil, nil, fmt.Errorf("key columns not found in schema: %v", missingCols)
+		}
+	}
+
+	var keyRecordType *schema_pb.RecordType
+	if len(keyFields) > 0 {
+		keyRecordType = &schema_pb.RecordType{Fields: keyFields}
+	}
+
+	var valueRecordType *schema_pb.RecordType
+	if len(valueFields) > 0 {
+		valueRecordType = &schema_pb.RecordType{Fields: valueFields}
+	}
+
+	return keyRecordType, valueRecordType, nil
+}
+
+// CombineFlatSchemaFromKeyValue creates a flat RecordType by combining key and value schemas
+// Key fields are placed first, then value fields
+func CombineFlatSchemaFromKeyValue(keySchema *schema_pb.RecordType, valueSchema *schema_pb.RecordType) (*schema_pb.RecordType, []string) {
+	var combinedFields []*schema_pb.Field
+	var keyColumns []string
+
+	// Add key fields first
+	if keySchema != nil {
+		for _, field := range keySchema.Fields {
+			combinedField := &schema_pb.Field{
+				Name:       field.Name,
+				FieldIndex: int32(len(combinedFields)),
+				Type:       field.Type,
+				IsRepeated: field.IsRepeated,
+				IsRequired: field.IsRequired,
+			}
+			combinedFields = append(combinedFields, combinedField)
+			keyColumns = append(keyColumns, field.Name)
+		}
+	}
+
+	// Add value fields
+	if valueSchema != nil {
+		for _, field := range valueSchema.Fields {
+			// Check for name conflicts
+			fieldName := field.Name
+			for _, keyCol := range keyColumns {
+				if fieldName == keyCol {
+					// This shouldn't happen in well-formed schemas, but handle gracefully
+					fieldName = "value_" + fieldName
+					break
+				}
+			}
+
+			combinedField := &schema_pb.Field{
+				Name:       fieldName,
+				FieldIndex: int32(len(combinedFields)),
+				Type:       field.Type,
+				IsRepeated: field.IsRepeated,
+				IsRequired: field.IsRequired,
+			}
+			combinedFields = append(combinedFields, combinedField)
+		}
+	}
+
+	if len(combinedFields) == 0 {
+		return nil, keyColumns
+	}
+
+	return &schema_pb.RecordType{Fields: combinedFields}, keyColumns
+}
+
+// ExtractKeyColumnsFromCombinedSchema tries to infer key columns from a combined schema
+// that was created using CreateCombinedRecordType (with key_ prefixes)
+func ExtractKeyColumnsFromCombinedSchema(combinedSchema *schema_pb.RecordType) (flatSchema *schema_pb.RecordType, keyColumns []string) {
+	if combinedSchema == nil {
+		return nil, nil
+	}
+
+	var flatFields []*schema_pb.Field
+	var keyColumns_ []string
+
+	for _, field := range combinedSchema.Fields {
+		if strings.HasPrefix(field.Name, "key_") {
+			// This is a key field - remove the prefix
+			originalName := strings.TrimPrefix(field.Name, "key_")
+			flatField := &schema_pb.Field{
+				Name:       originalName,
+				FieldIndex: int32(len(flatFields)),
+				Type:       field.Type,
+				IsRepeated: field.IsRepeated,
+				IsRequired: field.IsRequired,
+			}
+			flatFields = append(flatFields, flatField)
+			keyColumns_ = append(keyColumns_, originalName)
+		} else {
+			// This is a value field
+			flatField := &schema_pb.Field{
+				Name:       field.Name,
+				FieldIndex: int32(len(flatFields)),
+				Type:       field.Type,
+				IsRepeated: field.IsRepeated,
+				IsRequired: field.IsRequired,
+			}
+			flatFields = append(flatFields, flatField)
+		}
+	}
+
+	// Sort key columns to ensure deterministic order
+	sort.Strings(keyColumns_)
+
+	if len(flatFields) == 0 {
+		return nil, keyColumns_
+	}
+
+	return &schema_pb.RecordType{Fields: flatFields}, keyColumns_
+}
+
+// ValidateKeyColumns checks that all key columns exist in the schema
+func ValidateKeyColumns(schema *schema_pb.RecordType, keyColumns []string) error {
+	if schema == nil || len(keyColumns) == 0 {
+		return nil
+	}
+
+	fieldNames := make(map[string]bool)
+	for _, field := range schema.Fields {
+		fieldNames[field.Name] = true
+	}
+
+	var missingColumns []string
+	for _, keyCol := range keyColumns {
+		if !fieldNames[keyCol] {
+			missingColumns = append(missingColumns, keyCol)
+		}
+	}
+
+	if len(missingColumns) > 0 {
+		return fmt.Errorf("key columns not found in schema: %v", missingColumns)
+	}
+
+	return nil
+}

weed/mq/schema/flat_schema_utils_test.go

@@ -0,0 +1,265 @@
+package schema
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+func TestSplitFlatSchemaToKeyValue(t *testing.T) {
+	// Create a test flat schema
+	flatSchema := &schema_pb.RecordType{
+		Fields: []*schema_pb.Field{
+			{
+				Name:       "user_id",
+				FieldIndex: 0,
+				Type:       &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_INT64}},
+			},
+			{
+				Name:       "session_id",
+				FieldIndex: 1,
+				Type:       &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_STRING}},
+			},
+			{
+				Name:       "event_type",
+				FieldIndex: 2,
+				Type:       &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_STRING}},
+			},
+			{
+				Name:       "timestamp",
+				FieldIndex: 3,
+				Type:       &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_INT64}},
+			},
+		},
+	}
+
+	keyColumns := []string{"user_id", "session_id"}
+
+	keySchema, valueSchema, err := SplitFlatSchemaToKeyValue(flatSchema, keyColumns)
+	if err != nil {
+		t.Fatalf("SplitFlatSchemaToKeyValue failed: %v", err)
+	}
+
+	// Verify key schema
+	if keySchema == nil {
+		t.Fatal("Expected key schema, got nil")
+	}
+	if len(keySchema.Fields) != 2 {
+		t.Errorf("Expected 2 key fields, got %d", len(keySchema.Fields))
+	}
+	if keySchema.Fields[0].Name != "user_id" || keySchema.Fields[1].Name != "session_id" {
+		t.Errorf("Key field names incorrect: %v", []string{keySchema.Fields[0].Name, keySchema.Fields[1].Name})
+	}
+
+	// Verify value schema
+	if valueSchema == nil {
+		t.Fatal("Expected value schema, got nil")
+	}
+	if len(valueSchema.Fields) != 2 {
+		t.Errorf("Expected 2 value fields, got %d", len(valueSchema.Fields))
+	}
+	if valueSchema.Fields[0].Name != "event_type" || valueSchema.Fields[1].Name != "timestamp" {
+		t.Errorf("Value field names incorrect: %v", []string{valueSchema.Fields[0].Name, valueSchema.Fields[1].Name})
+	}
+
+	// Verify field indices are reindexed
+	for i, field := range keySchema.Fields {
+		if field.FieldIndex != int32(i) {
+			t.Errorf("Key field %s has incorrect index %d, expected %d", field.Name, field.FieldIndex, i)
+		}
+	}
+	for i, field := range valueSchema.Fields {
+		if field.FieldIndex != int32(i) {
+			t.Errorf("Value field %s has incorrect index %d, expected %d", field.Name, field.FieldIndex, i)
+		}
+	}
+}
+
+func TestSplitFlatSchemaToKeyValueMissingColumns(t *testing.T) {
+	flatSchema := &schema_pb.RecordType{
+		Fields: []*schema_pb.Field{
+			{Name: "field1", Type: &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_STRING}}},
+		},
+	}
+
+	keyColumns := []string{"field1", "missing_field"}
+
+	_, _, err := SplitFlatSchemaToKeyValue(flatSchema, keyColumns)
+	if err == nil {
+		t.Error("Expected error for missing key column, got nil")
+	}
+	if !contains(err.Error(), "missing_field") {
+		t.Errorf("Error should mention missing_field: %v", err)
+	}
+}
+
+func TestCombineFlatSchemaFromKeyValue(t *testing.T) {
+	keySchema := &schema_pb.RecordType{
+		Fields: []*schema_pb.Field{
+			{
+				Name:       "user_id",
+				FieldIndex: 0,
+				Type:       &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_INT64}},
+			},
+			{
+				Name:       "session_id",
+				FieldIndex: 1,
+				Type:       &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_STRING}},
+			},
+		},
+	}
+
+	valueSchema := &schema_pb.RecordType{
+		Fields: []*schema_pb.Field{
+			{
+				Name:       "event_type",
+				FieldIndex: 0,
+				Type:       &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_STRING}},
+			},
+			{
+				Name:       "timestamp",
+				FieldIndex: 1,
+				Type:       &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_INT64}},
+			},
+		},
+	}
+
+	flatSchema, keyColumns := CombineFlatSchemaFromKeyValue(keySchema, valueSchema)
+
+	// Verify combined schema
+	if flatSchema == nil {
+		t.Fatal("Expected flat schema, got nil")
+	}
+	if len(flatSchema.Fields) != 4 {
+		t.Errorf("Expected 4 fields, got %d", len(flatSchema.Fields))
+	}
+
+	// Verify key columns
+	expectedKeyColumns := []string{"user_id", "session_id"}
+	if !reflect.DeepEqual(keyColumns, expectedKeyColumns) {
+		t.Errorf("Expected key columns %v, got %v", expectedKeyColumns, keyColumns)
+	}
+
+	// Verify field order (key fields first)
+	expectedNames := []string{"user_id", "session_id", "event_type", "timestamp"}
+	actualNames := make([]string, len(flatSchema.Fields))
+	for i, field := range flatSchema.Fields {
+		actualNames[i] = field.Name
+	}
+	if !reflect.DeepEqual(actualNames, expectedNames) {
+		t.Errorf("Expected field names %v, got %v", expectedNames, actualNames)
+	}
+
+	// Verify field indices are sequential
+	for i, field := range flatSchema.Fields {
+		if field.FieldIndex != int32(i) {
+			t.Errorf("Field %s has incorrect index %d, expected %d", field.Name, field.FieldIndex, i)
+		}
+	}
+}
+
+func TestExtractKeyColumnsFromCombinedSchema(t *testing.T) {
+	// Create a combined schema with key_ prefixes (as created by CreateCombinedRecordType)
+	combinedSchema := &schema_pb.RecordType{
+		Fields: []*schema_pb.Field{
+			{
+				Name:       "key_user_id",
+				FieldIndex: 0,
+				Type:       &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_INT64}},
+			},
+			{
+				Name:       "key_session_id",
+				FieldIndex: 1,
+				Type:       &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_STRING}},
+			},
+			{
+				Name:       "event_type",
+				FieldIndex: 2,
+				Type:       &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_STRING}},
+			},
+			{
+				Name:       "timestamp",
+				FieldIndex: 3,
+				Type:       &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_INT64}},
+			},
+		},
+	}
+
+	flatSchema, keyColumns := ExtractKeyColumnsFromCombinedSchema(combinedSchema)
+
+	// Verify flat schema
+	if flatSchema == nil {
+		t.Fatal("Expected flat schema, got nil")
+	}
+	if len(flatSchema.Fields) != 4 {
+		t.Errorf("Expected 4 fields, got %d", len(flatSchema.Fields))
+	}
+
+	// Verify key columns (should be sorted)
+	expectedKeyColumns := []string{"session_id", "user_id"}
+	if !reflect.DeepEqual(keyColumns, expectedKeyColumns) {
+		t.Errorf("Expected key columns %v, got %v", expectedKeyColumns, keyColumns)
+	}
+
+	// Verify field names (key_ prefixes removed)
+	expectedNames := []string{"user_id", "session_id", "event_type", "timestamp"}
+	actualNames := make([]string, len(flatSchema.Fields))
+	for i, field := range flatSchema.Fields {
+		actualNames[i] = field.Name
+	}
+	if !reflect.DeepEqual(actualNames, expectedNames) {
+		t.Errorf("Expected field names %v, got %v", expectedNames, actualNames)
+	}
+}
+
+func TestValidateKeyColumns(t *testing.T) {
+	schema := &schema_pb.RecordType{
+		Fields: []*schema_pb.Field{
+			{Name: "field1", Type: &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_STRING}}},
+			{Name: "field2", Type: &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_INT64}}},
+		},
+	}
+
+	// Valid key columns
+	err := ValidateKeyColumns(schema, []string{"field1"})
+	if err != nil {
+		t.Errorf("Expected no error for valid key columns, got: %v", err)
+	}
+
+	// Invalid key columns
+	err = ValidateKeyColumns(schema, []string{"field1", "missing_field"})
+	if err == nil {
+		t.Error("Expected error for invalid key columns, got nil")
+	}
+
+	// Nil schema should not error
+	err = ValidateKeyColumns(nil, []string{"any_field"})
+	if err != nil {
+		t.Errorf("Expected no error for nil schema, got: %v", err)
+	}
+
+	// Empty key columns should not error
+	err = ValidateKeyColumns(schema, []string{})
+	if err != nil {
+		t.Errorf("Expected no error for empty key columns, got: %v", err)
+	}
+}
+
+// Helper function to check if string contains substring
+func contains(str, substr string) bool {
+	return len(str) >= len(substr) &&
+		(len(substr) == 0 || str[len(str)-len(substr):] == substr ||
+			str[:len(substr)] == substr ||
+			len(str) > len(substr) && (str[len(str)-len(substr)-1:len(str)-len(substr)] == " " || str[len(str)-len(substr)-1] == ' ') && str[len(str)-len(substr):] == substr ||
+			findInString(str, substr))
+}
+
+func findInString(str, substr string) bool {
+	for i := 0; i <= len(str)-len(substr); i++ {
+		if str[i:i+len(substr)] == substr {
+			return true
+		}
+	}
+	return false
+}

weed/mq/schema/schema.go

@@ -0,0 +1,58 @@
+package schema
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+type Schema struct {
+	Namespace  string
+	Name       string
+	RevisionId uint32
+	RecordType *schema_pb.RecordType
+	fieldMap   map[string]*schema_pb.Field
+}
+
+func NewSchema(namespace string, name string, recordType *schema_pb.RecordType) *Schema {
+	fieldMap := make(map[string]*schema_pb.Field)
+	for _, field := range recordType.Fields {
+		fieldMap[field.Name] = field
+	}
+	return &Schema{
+		Namespace:  namespace,
+		Name:       name,
+		RecordType: recordType,
+		fieldMap:   fieldMap,
+	}
+}
+
+func (s *Schema) GetField(name string) (*schema_pb.Field, bool) {
+	field, ok := s.fieldMap[name]
+	return field, ok
+}
+
+func TypeToString(t *schema_pb.Type) string {
+	switch t.Kind.(type) {
+	case *schema_pb.Type_ScalarType:
+		switch t.GetScalarType() {
+		case schema_pb.ScalarType_BOOL:
+			return "bool"
+		case schema_pb.ScalarType_INT32:
+			return "int32"
+		case schema_pb.ScalarType_INT64:
+			return "int64"
+		case schema_pb.ScalarType_FLOAT:
+			return "float"
+		case schema_pb.ScalarType_DOUBLE:
+			return "double"
+		case schema_pb.ScalarType_BYTES:
+			return "bytes"
+		case schema_pb.ScalarType_STRING:
+			return "string"
+		}
+	case *schema_pb.Type_ListType:
+		return "list"
+	case *schema_pb.Type_RecordType:
+		return "record"
+	}
+	return "unknown"
+}

weed/mq/schema/schema_builder.go

@@ -0,0 +1,67 @@
+package schema
+
+import (
+	"sort"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+var (
+	// Basic scalar types
+	TypeBoolean = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_BOOL}}
+	TypeInt32   = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_INT32}}
+	TypeInt64   = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_INT64}}
+	TypeFloat   = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_FLOAT}}
+	TypeDouble  = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_DOUBLE}}
+	TypeBytes   = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_BYTES}}
+	TypeString  = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_STRING}}
+
+	// Parquet logical types
+	TypeTimestamp = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_TIMESTAMP}}
+	TypeDate      = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_DATE}}
+	TypeDecimal   = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_DECIMAL}}
+	TypeTime      = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_TIME}}
+)
+
+type RecordTypeBuilder struct {
+	recordType *schema_pb.RecordType
+}
+
+// RecordTypeBegin creates a new RecordTypeBuilder, it should be followed by a series of WithField methods and RecordTypeEnd
+func RecordTypeBegin() *RecordTypeBuilder {
+	return &RecordTypeBuilder{recordType: &schema_pb.RecordType{}}
+}
+
+// RecordTypeEnd finishes the building of a RecordValue
+func (rtb *RecordTypeBuilder) RecordTypeEnd() *schema_pb.RecordType {
+	// be consistent with parquet.node.go `func (g Group) Fields() []Field`
+	sort.Slice(rtb.recordType.Fields, func(i, j int) bool {
+		return rtb.recordType.Fields[i].Name < rtb.recordType.Fields[j].Name
+	})
+	return rtb.recordType
+}
+
+// NewRecordTypeBuilder creates a new RecordTypeBuilder from an existing RecordType, it should be followed by a series of WithField methods and RecordTypeEnd
+func NewRecordTypeBuilder(recordType *schema_pb.RecordType) (rtb *RecordTypeBuilder) {
+	return &RecordTypeBuilder{recordType: recordType}
+}
+
+func (rtb *RecordTypeBuilder) WithField(name string, scalarType *schema_pb.Type) *RecordTypeBuilder {
+	rtb.recordType.Fields = append(rtb.recordType.Fields, &schema_pb.Field{
+		Name: name,
+		Type: scalarType,
+	})
+	return rtb
+}
+
+func (rtb *RecordTypeBuilder) WithRecordField(name string, recordType *schema_pb.RecordType) *RecordTypeBuilder {
+	rtb.recordType.Fields = append(rtb.recordType.Fields, &schema_pb.Field{
+		Name: name,
+		Type: &schema_pb.Type{Kind: &schema_pb.Type_RecordType{RecordType: recordType}},
+	})
+	return rtb
+}
+
+func ListOf(elementType *schema_pb.Type) *schema_pb.Type {
+	return &schema_pb.Type{Kind: &schema_pb.Type_ListType{ListType: &schema_pb.ListType{ElementType: elementType}}}
+}

weed/mq/schema/schema_test.go

@@ -0,0 +1,86 @@
+package schema
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/golang/protobuf/proto"
+	. "github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEnumScalarType(t *testing.T) {
+	tests := []struct {
+		name     string
+		enum     ScalarType
+		expected int32
+	}{
+		{"Boolean", ScalarType_BOOL, 0},
+		{"Integer", ScalarType_INT32, 1},
+		{"Long", ScalarType_INT64, 3},
+		{"Float", ScalarType_FLOAT, 4},
+		{"Double", ScalarType_DOUBLE, 5},
+		{"Bytes", ScalarType_BYTES, 6},
+		{"String", ScalarType_STRING, 7},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.expected, int32(tt.enum))
+		})
+	}
+}
+
+func TestField(t *testing.T) {
+	field := &Field{
+		Name:       "field_name",
+		Type:       &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}},
+		FieldIndex: 1,
+		IsRepeated: false,
+	}
+	assert.NotNil(t, field)
+}
+
+func TestRecordType(t *testing.T) {
+	subRecord := &RecordType{
+		Fields: []*Field{
+			{
+				Name:       "field_1",
+				Type:       &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}},
+				FieldIndex: 1,
+				IsRepeated: false,
+			},
+			{
+				Name:       "field_2",
+				Type:       &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_STRING}},
+				FieldIndex: 2,
+				IsRepeated: false,
+			},
+		},
+	}
+	record := &RecordType{
+		Fields: []*Field{
+			{
+				Name:       "field_key",
+				Type:       &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}},
+				FieldIndex: 1,
+				IsRepeated: false,
+			},
+			{
+				Name:       "field_record",
+				Type:       &Type{Kind: &Type_RecordType{RecordType: subRecord}},
+				FieldIndex: 2,
+				IsRepeated: false,
+			},
+		},
+	}
+
+	// serialize record to protobuf text marshalling
+	text := proto.MarshalTextString(record)
+	println(text)
+
+	bytes, _ := json.Marshal(record)
+	println(string(bytes))
+
+	assert.NotNil(t, record)
+}

weed/mq/schema/struct_to_schema.go

@@ -0,0 +1,105 @@
+package schema
+
+import (
+	"reflect"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+func StructToSchema(instance any) *schema_pb.RecordType {
+	myType := reflect.TypeOf(instance)
+	if myType.Kind() != reflect.Struct {
+		return nil
+	}
+	st := reflectTypeToSchemaType(myType)
+	return st.GetRecordType()
+}
+
+// CreateCombinedRecordType creates a combined RecordType that includes fields from both key and value schemas
+// Key fields are prefixed with "key_" to distinguish them from value fields
+func CreateCombinedRecordType(keyRecordType *schema_pb.RecordType, valueRecordType *schema_pb.RecordType) *schema_pb.RecordType {
+	var combinedFields []*schema_pb.Field
+
+	// Add key fields with "key_" prefix
+	if keyRecordType != nil {
+		for _, field := range keyRecordType.Fields {
+			keyField := &schema_pb.Field{
+				Name:       "key_" + field.Name,
+				FieldIndex: field.FieldIndex, // Will be reindexed later
+				Type:       field.Type,
+				IsRepeated: field.IsRepeated,
+				IsRequired: field.IsRequired,
+			}
+			combinedFields = append(combinedFields, keyField)
+		}
+	}
+
+	// Add value fields (no prefix)
+	if valueRecordType != nil {
+		for _, field := range valueRecordType.Fields {
+			combinedFields = append(combinedFields, field)
+		}
+	}
+
+	// Reindex all fields to have sequential indices
+	for i, field := range combinedFields {
+		field.FieldIndex = int32(i)
+	}
+
+	return &schema_pb.RecordType{
+		Fields: combinedFields,
+	}
+}
+
+func reflectTypeToSchemaType(t reflect.Type) *schema_pb.Type {
+	switch t.Kind() {
+	case reflect.Bool:
+		return TypeBoolean
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32:
+		return TypeInt32
+	case reflect.Int64:
+		return TypeInt64
+	case reflect.Float32:
+		return TypeFloat
+	case reflect.Float64:
+		return TypeDouble
+	case reflect.String:
+		return TypeString
+	case reflect.Slice:
+		switch t.Elem().Kind() {
+		case reflect.Uint8:
+			return TypeBytes
+		default:
+			if st := reflectTypeToSchemaType(t.Elem()); st != nil {
+				return &schema_pb.Type{
+					Kind: &schema_pb.Type_ListType{
+						ListType: &schema_pb.ListType{
+							ElementType: st,
+						},
+					},
+				}
+			}
+		}
+	case reflect.Struct:
+		recordType := &schema_pb.RecordType{}
+		for i := 0; i < t.NumField(); i++ {
+			field := t.Field(i)
+			fieldType := field.Type
+			fieldName := field.Name
+			schemaField := reflectTypeToSchemaType(fieldType)
+			if schemaField == nil {
+				return nil
+			}
+			recordType.Fields = append(recordType.Fields, &schema_pb.Field{
+				Name: fieldName,
+				Type: schemaField,
+			})
+		}
+		return &schema_pb.Type{
+			Kind: &schema_pb.Type_RecordType{
+				RecordType: recordType,
+			},
+		}
+	}
+	return nil
+}

weed/mq/schema/struct_to_schema_test.go

@@ -0,0 +1,120 @@
+package schema
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStructToSchema(t *testing.T) {
+	type args struct {
+		instance any
+	}
+	tests := []struct {
+		name string
+		args args
+		want *schema_pb.RecordType
+	}{
+		{
+			name: "scalar type",
+			args: args{
+				instance: 1,
+			},
+			want: nil,
+		},
+		{
+			name: "simple struct type",
+			args: args{
+				instance: struct {
+					Field1 int
+					Field2 string
+				}{},
+			},
+			want: RecordTypeBegin().
+				WithField("Field1", TypeInt32).
+				WithField("Field2", TypeString).
+				RecordTypeEnd(),
+		},
+		{
+			name: "simple list",
+			args: args{
+				instance: struct {
+					Field1 []int
+					Field2 string
+				}{},
+			},
+			want: RecordTypeBegin().
+				WithField("Field1", ListOf(TypeInt32)).
+				WithField("Field2", TypeString).
+				RecordTypeEnd(),
+		},
+		{
+			name: "simple []byte",
+			args: args{
+				instance: struct {
+					Field2 []byte
+				}{},
+			},
+			want: RecordTypeBegin().
+				WithField("Field2", TypeBytes).
+				RecordTypeEnd(),
+		},
+		{
+			name: "nested simple structs",
+			args: args{
+				instance: struct {
+					Field1 int
+					Field2 struct {
+						Field3 string
+						Field4 int
+					}
+				}{},
+			},
+			want: RecordTypeBegin().
+				WithField("Field1", TypeInt32).
+				WithRecordField("Field2",
+					RecordTypeBegin().
+						WithField("Field3", TypeString).
+						WithField("Field4", TypeInt32).
+						RecordTypeEnd(),
+				).
+				RecordTypeEnd(),
+		},
+		{
+			name: "nested struct type",
+			args: args{
+				instance: struct {
+					Field1 int
+					Field2 struct {
+						Field3 string
+						Field4 []int
+						Field5 struct {
+							Field6 string
+							Field7 []byte
+						}
+					}
+				}{},
+			},
+			want: RecordTypeBegin().
+				WithField("Field1", TypeInt32).
+				WithRecordField("Field2", RecordTypeBegin().
+					WithField("Field3", TypeString).
+					WithField("Field4", ListOf(TypeInt32)).
+					WithRecordField("Field5",
+						RecordTypeBegin().
+							WithField("Field6", TypeString).
+							WithField("Field7", TypeBytes).
+							RecordTypeEnd(),
+					).RecordTypeEnd(),
+				).
+				RecordTypeEnd(),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equalf(t, tt.want, StructToSchema(tt.args.instance), "StructToSchema(%v)", tt.args.instance)
+		})
+	}
+}

weed/mq/schema/to_parquet_levels.go

@@ -0,0 +1,59 @@
+package schema
+
+import (
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+type ParquetLevels struct {
+	startColumnIndex int
+	endColumnIndex   int
+	definitionDepth  int
+	levels           map[string]*ParquetLevels
+}
+
+func ToParquetLevels(recordType *schema_pb.RecordType) (*ParquetLevels, error) {
+	return toRecordTypeLevels(recordType, 0, 0)
+}
+
+func toFieldTypeLevels(fieldType *schema_pb.Type, startColumnIndex, definitionDepth int) (*ParquetLevels, error) {
+	switch fieldType.Kind.(type) {
+	case *schema_pb.Type_ScalarType:
+		return toFieldTypeScalarLevels(fieldType.GetScalarType(), startColumnIndex, definitionDepth)
+	case *schema_pb.Type_RecordType:
+		return toRecordTypeLevels(fieldType.GetRecordType(), startColumnIndex, definitionDepth)
+	case *schema_pb.Type_ListType:
+		return toFieldTypeListLevels(fieldType.GetListType(), startColumnIndex, definitionDepth)
+	}
+	return nil, fmt.Errorf("unknown field type: %T", fieldType.Kind)
+}
+
+func toFieldTypeListLevels(listType *schema_pb.ListType, startColumnIndex, definitionDepth int) (*ParquetLevels, error) {
+	return toFieldTypeLevels(listType.ElementType, startColumnIndex, definitionDepth)
+}
+
+func toFieldTypeScalarLevels(scalarType schema_pb.ScalarType, startColumnIndex, definitionDepth int) (*ParquetLevels, error) {
+	return &ParquetLevels{
+		startColumnIndex: startColumnIndex,
+		endColumnIndex:   startColumnIndex + 1,
+		definitionDepth:  definitionDepth,
+	}, nil
+}
+func toRecordTypeLevels(recordType *schema_pb.RecordType, startColumnIndex, definitionDepth int) (*ParquetLevels, error) {
+	recordTypeLevels := &ParquetLevels{
+		startColumnIndex: startColumnIndex,
+		definitionDepth:  definitionDepth,
+		levels:           make(map[string]*ParquetLevels),
+	}
+	for _, field := range recordType.Fields {
+		fieldTypeLevels, err := toFieldTypeLevels(field.Type, startColumnIndex, definitionDepth+1)
+		if err != nil {
+			return nil, err
+		}
+		recordTypeLevels.levels[field.Name] = fieldTypeLevels
+		startColumnIndex = fieldTypeLevels.endColumnIndex
+	}
+	recordTypeLevels.endColumnIndex = startColumnIndex
+	return recordTypeLevels, nil
+}

weed/mq/schema/to_parquet_levels_test.go

@@ -0,0 +1,103 @@
+package schema
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestToParquetLevels(t *testing.T) {
+	type args struct {
+		recordType *schema_pb.RecordType
+	}
+	tests := []struct {
+		name string
+		args args
+		want *ParquetLevels
+	}{
+		{
+			name: "nested type",
+			args: args{
+				RecordTypeBegin().
+					WithField("ID", TypeInt64).
+					WithField("CreatedAt", TypeInt64).
+					WithRecordField("Person",
+						RecordTypeBegin().
+							WithField("zName", TypeString).
+							WithField("emails", ListOf(TypeString)).
+							RecordTypeEnd()).
+					WithField("Company", TypeString).
+					WithRecordField("Address",
+						RecordTypeBegin().
+							WithField("Street", TypeString).
+							WithField("City", TypeString).
+							RecordTypeEnd()).
+					RecordTypeEnd(),
+			},
+			want: &ParquetLevels{
+				startColumnIndex: 0,
+				endColumnIndex:   7,
+				definitionDepth:  0,
+				levels: map[string]*ParquetLevels{
+					"Address": {
+						startColumnIndex: 0,
+						endColumnIndex:   2,
+						definitionDepth:  1,
+						levels: map[string]*ParquetLevels{
+							"City": {
+								startColumnIndex: 0,
+								endColumnIndex:   1,
+								definitionDepth:  2,
+							},
+							"Street": {
+								startColumnIndex: 1,
+								endColumnIndex:   2,
+								definitionDepth:  2,
+							},
+						},
+					},
+					"Company": {
+						startColumnIndex: 2,
+						endColumnIndex:   3,
+						definitionDepth:  1,
+					},
+					"CreatedAt": {
+						startColumnIndex: 3,
+						endColumnIndex:   4,
+						definitionDepth:  1,
+					},
+					"ID": {
+						startColumnIndex: 4,
+						endColumnIndex:   5,
+						definitionDepth:  1,
+					},
+					"Person": {
+						startColumnIndex: 5,
+						endColumnIndex:   7,
+						definitionDepth:  1,
+						levels: map[string]*ParquetLevels{
+							"emails": {
+								startColumnIndex: 5,
+								endColumnIndex:   6,
+								definitionDepth:  2,
+							},
+							"zName": {
+								startColumnIndex: 6,
+								endColumnIndex:   7,
+								definitionDepth:  2,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ToParquetLevels(tt.args.recordType)
+			assert.Nil(t, err)
+			assert.Equalf(t, tt.want, got, "ToParquetLevels(%v)", tt.args.recordType)
+		})
+	}
+}

weed/mq/schema/to_parquet_schema.go

@@ -0,0 +1,117 @@
+package schema
+
+import (
+	"fmt"
+
+	parquet "github.com/parquet-go/parquet-go"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+func ToParquetSchema(topicName string, recordType *schema_pb.RecordType) (*parquet.Schema, error) {
+	rootNode, err := toParquetFieldTypeRecord(recordType)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert record type to parquet schema: %w", err)
+	}
+
+	// Fields are sorted by name, so the value should be sorted also
+	// the sorting is inside parquet.`func (g Group) Fields() []Field`
+	return parquet.NewSchema(topicName, rootNode), nil
+}
+
+func toParquetFieldType(fieldType *schema_pb.Type) (dataType parquet.Node, err error) {
+	// This is the old function - now defaults to Optional for backward compatibility
+	return toParquetFieldTypeWithRequirement(fieldType, false)
+}
+
+func toParquetFieldTypeList(listType *schema_pb.ListType) (parquet.Node, error) {
+	elementType, err := toParquetFieldType(listType.ElementType)
+	if err != nil {
+		return nil, err
+	}
+	return parquet.Repeated(elementType), nil
+}
+
+func toParquetFieldTypeScalar(scalarType schema_pb.ScalarType) (parquet.Node, error) {
+	switch scalarType {
+	case schema_pb.ScalarType_BOOL:
+		return parquet.Leaf(parquet.BooleanType), nil
+	case schema_pb.ScalarType_INT32:
+		return parquet.Leaf(parquet.Int32Type), nil
+	case schema_pb.ScalarType_INT64:
+		return parquet.Leaf(parquet.Int64Type), nil
+	case schema_pb.ScalarType_FLOAT:
+		return parquet.Leaf(parquet.FloatType), nil
+	case schema_pb.ScalarType_DOUBLE:
+		return parquet.Leaf(parquet.DoubleType), nil
+	case schema_pb.ScalarType_BYTES:
+		return parquet.Leaf(parquet.ByteArrayType), nil
+	case schema_pb.ScalarType_STRING:
+		return parquet.Leaf(parquet.ByteArrayType), nil
+	// Parquet logical types - map to their physical storage types
+	case schema_pb.ScalarType_TIMESTAMP:
+		// Stored as INT64 (microseconds since Unix epoch)
+		return parquet.Leaf(parquet.Int64Type), nil
+	case schema_pb.ScalarType_DATE:
+		// Stored as INT32 (days since Unix epoch)
+		return parquet.Leaf(parquet.Int32Type), nil
+	case schema_pb.ScalarType_DECIMAL:
+		// Use maximum precision/scale to accommodate any decimal value
+		// Per Parquet spec: precision ≤9→INT32, ≤18→INT64, >18→FixedLenByteArray
+		// Using precision=38 (max for most systems), scale=18 for flexibility
+		// Individual values can have smaller precision/scale, but schema supports maximum
+		return parquet.Decimal(18, 38, parquet.FixedLenByteArrayType(16)), nil
+	case schema_pb.ScalarType_TIME:
+		// Stored as INT64 (microseconds since midnight)
+		return parquet.Leaf(parquet.Int64Type), nil
+	default:
+		return nil, fmt.Errorf("unknown scalar type: %v", scalarType)
+	}
+}
+func toParquetFieldTypeRecord(recordType *schema_pb.RecordType) (parquet.Node, error) {
+	recordNode := parquet.Group{}
+	for _, field := range recordType.Fields {
+		parquetFieldType, err := toParquetFieldTypeWithRequirement(field.Type, field.IsRequired)
+		if err != nil {
+			return nil, err
+		}
+		recordNode[field.Name] = parquetFieldType
+	}
+	return recordNode, nil
+}
+
+// toParquetFieldTypeWithRequirement creates parquet field type respecting required/optional constraints
+func toParquetFieldTypeWithRequirement(fieldType *schema_pb.Type, isRequired bool) (dataType parquet.Node, err error) {
+	switch fieldType.Kind.(type) {
+	case *schema_pb.Type_ScalarType:
+		dataType, err = toParquetFieldTypeScalar(fieldType.GetScalarType())
+		if err != nil {
+			return nil, err
+		}
+		if isRequired {
+			// Required fields are NOT wrapped in Optional
+			return dataType, nil
+		} else {
+			// Optional fields are wrapped in Optional
+			return parquet.Optional(dataType), nil
+		}
+	case *schema_pb.Type_RecordType:
+		dataType, err = toParquetFieldTypeRecord(fieldType.GetRecordType())
+		if err != nil {
+			return nil, err
+		}
+		if isRequired {
+			return dataType, nil
+		} else {
+			return parquet.Optional(dataType), nil
+		}
+	case *schema_pb.Type_ListType:
+		dataType, err = toParquetFieldTypeList(fieldType.GetListType())
+		if err != nil {
+			return nil, err
+		}
+		// Lists are typically optional by nature
+		return dataType, nil
+	default:
+		return nil, fmt.Errorf("unknown field type: %T", fieldType.Kind)
+	}
+}

weed/mq/schema/to_parquet_value.go

@@ -0,0 +1,346 @@
+package schema
+
+import (
+	"fmt"
+	"strconv"
+
+	parquet "github.com/parquet-go/parquet-go"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+func rowBuilderVisit(rowBuilder *parquet.RowBuilder, fieldType *schema_pb.Type, levels *ParquetLevels, fieldValue *schema_pb.Value) (err error) {
+	switch fieldType.Kind.(type) {
+	case *schema_pb.Type_ScalarType:
+		// If value is missing, write NULL at the correct column to keep rows aligned
+		if fieldValue == nil || fieldValue.Kind == nil {
+			rowBuilder.Add(levels.startColumnIndex, parquet.NullValue())
+			return nil
+		}
+		var parquetValue parquet.Value
+		parquetValue, err = toParquetValueForType(fieldType, fieldValue)
+		if err != nil {
+			return
+		}
+
+		// Safety check: prevent nil byte arrays from reaching parquet library
+		if parquetValue.Kind() == parquet.ByteArray {
+			byteData := parquetValue.ByteArray()
+			if byteData == nil {
+				parquetValue = parquet.ByteArrayValue([]byte{})
+			}
+		}
+
+		rowBuilder.Add(levels.startColumnIndex, parquetValue)
+	case *schema_pb.Type_ListType:
+		// Advance to list position even if value is missing
+		rowBuilder.Next(levels.startColumnIndex)
+		if fieldValue == nil || fieldValue.GetListValue() == nil {
+			return nil
+		}
+
+		elementType := fieldType.GetListType().ElementType
+		for _, value := range fieldValue.GetListValue().Values {
+			if err = rowBuilderVisit(rowBuilder, elementType, levels, value); err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+func AddRecordValue(rowBuilder *parquet.RowBuilder, recordType *schema_pb.RecordType, parquetLevels *ParquetLevels, recordValue *schema_pb.RecordValue) error {
+	visitor := func(fieldType *schema_pb.Type, levels *ParquetLevels, fieldValue *schema_pb.Value) (err error) {
+		return rowBuilderVisit(rowBuilder, fieldType, levels, fieldValue)
+	}
+	fieldType := &schema_pb.Type{Kind: &schema_pb.Type_RecordType{RecordType: recordType}}
+	fieldValue := &schema_pb.Value{Kind: &schema_pb.Value_RecordValue{RecordValue: recordValue}}
+	return doVisitValue(fieldType, parquetLevels, fieldValue, visitor)
+}
+
+// typeValueVisitor is a function that is called for each value in a schema_pb.Value
+// Find the column index.
+// intended to be used in RowBuilder.Add(columnIndex, value)
+type typeValueVisitor func(fieldType *schema_pb.Type, levels *ParquetLevels, fieldValue *schema_pb.Value) (err error)
+
+// endIndex is exclusive
+// same logic as RowBuilder.configure in row_builder.go
+func doVisitValue(fieldType *schema_pb.Type, levels *ParquetLevels, fieldValue *schema_pb.Value, visitor typeValueVisitor) (err error) {
+	switch fieldType.Kind.(type) {
+	case *schema_pb.Type_ScalarType:
+		return visitor(fieldType, levels, fieldValue)
+	case *schema_pb.Type_ListType:
+		return visitor(fieldType, levels, fieldValue)
+	case *schema_pb.Type_RecordType:
+		for _, field := range fieldType.GetRecordType().Fields {
+			var fv *schema_pb.Value
+			if fieldValue != nil && fieldValue.GetRecordValue() != nil {
+				var found bool
+				fv, found = fieldValue.GetRecordValue().Fields[field.Name]
+				if !found {
+					// pass nil so visitor can emit NULL for alignment
+					fv = nil
+				}
+			}
+			fieldLevels := levels.levels[field.Name]
+			err = doVisitValue(field.Type, fieldLevels, fv, visitor)
+			if err != nil {
+				return
+			}
+		}
+		return
+	}
+	return
+}
+
+func toParquetValue(value *schema_pb.Value) (parquet.Value, error) {
+	// Safety check for nil value
+	if value == nil || value.Kind == nil {
+		return parquet.NullValue(), fmt.Errorf("nil value or nil value kind")
+	}
+
+	switch value.Kind.(type) {
+	case *schema_pb.Value_BoolValue:
+		return parquet.BooleanValue(value.GetBoolValue()), nil
+	case *schema_pb.Value_Int32Value:
+		return parquet.Int32Value(value.GetInt32Value()), nil
+	case *schema_pb.Value_Int64Value:
+		return parquet.Int64Value(value.GetInt64Value()), nil
+	case *schema_pb.Value_FloatValue:
+		return parquet.FloatValue(value.GetFloatValue()), nil
+	case *schema_pb.Value_DoubleValue:
+		return parquet.DoubleValue(value.GetDoubleValue()), nil
+	case *schema_pb.Value_BytesValue:
+		// Handle nil byte slices to prevent growslice panic in parquet-go
+		byteData := value.GetBytesValue()
+		if byteData == nil {
+			byteData = []byte{} // Use empty slice instead of nil
+		}
+		return parquet.ByteArrayValue(byteData), nil
+	case *schema_pb.Value_StringValue:
+		// Convert string to bytes, ensuring we never pass nil
+		stringData := value.GetStringValue()
+		return parquet.ByteArrayValue([]byte(stringData)), nil
+	// Parquet logical types with safe conversion (preventing commit 7a4aeec60 panic)
+	case *schema_pb.Value_TimestampValue:
+		timestampValue := value.GetTimestampValue()
+		if timestampValue == nil {
+			return parquet.NullValue(), nil
+		}
+		return parquet.Int64Value(timestampValue.TimestampMicros), nil
+	case *schema_pb.Value_DateValue:
+		dateValue := value.GetDateValue()
+		if dateValue == nil {
+			return parquet.NullValue(), nil
+		}
+		return parquet.Int32Value(dateValue.DaysSinceEpoch), nil
+	case *schema_pb.Value_DecimalValue:
+		decimalValue := value.GetDecimalValue()
+		if decimalValue == nil || decimalValue.Value == nil || len(decimalValue.Value) == 0 {
+			return parquet.NullValue(), nil
+		}
+
+		// Validate input data - reject unreasonably large values instead of corrupting data
+		if len(decimalValue.Value) > 64 {
+			// Reject extremely large decimal values (>512 bits) as likely corrupted data
+			// Better to fail fast than silently corrupt financial/scientific data
+			return parquet.NullValue(), fmt.Errorf("decimal value too large: %d bytes (max 64)", len(decimalValue.Value))
+		}
+
+		// Convert to FixedLenByteArray to match schema (DECIMAL with FixedLenByteArray physical type)
+		// This accommodates any precision up to 38 digits (16 bytes = 128 bits)
+
+		// Pad or truncate to exactly 16 bytes for FixedLenByteArray
+		fixedBytes := make([]byte, 16)
+		if len(decimalValue.Value) <= 16 {
+			// Right-align the value (big-endian)
+			copy(fixedBytes[16-len(decimalValue.Value):], decimalValue.Value)
+		} else {
+			// Truncate if too large, taking the least significant bytes
+			copy(fixedBytes, decimalValue.Value[len(decimalValue.Value)-16:])
+		}
+
+		return parquet.FixedLenByteArrayValue(fixedBytes), nil
+	case *schema_pb.Value_TimeValue:
+		timeValue := value.GetTimeValue()
+		if timeValue == nil {
+			return parquet.NullValue(), nil
+		}
+		return parquet.Int64Value(timeValue.TimeMicros), nil
+	default:
+		return parquet.NullValue(), fmt.Errorf("unknown value type: %T", value.Kind)
+	}
+}
+
+// toParquetValueForType coerces a schema_pb.Value into a parquet.Value that matches the declared field type.
+func toParquetValueForType(fieldType *schema_pb.Type, value *schema_pb.Value) (parquet.Value, error) {
+	switch t := fieldType.Kind.(type) {
+	case *schema_pb.Type_ScalarType:
+		switch t.ScalarType {
+		case schema_pb.ScalarType_BOOL:
+			switch v := value.Kind.(type) {
+			case *schema_pb.Value_BoolValue:
+				return parquet.BooleanValue(v.BoolValue), nil
+			case *schema_pb.Value_StringValue:
+				if b, err := strconv.ParseBool(v.StringValue); err == nil {
+					return parquet.BooleanValue(b), nil
+				}
+				return parquet.BooleanValue(false), nil
+			default:
+				return parquet.BooleanValue(false), nil
+			}
+
+		case schema_pb.ScalarType_INT32:
+			switch v := value.Kind.(type) {
+			case *schema_pb.Value_Int32Value:
+				return parquet.Int32Value(v.Int32Value), nil
+			case *schema_pb.Value_Int64Value:
+				return parquet.Int32Value(int32(v.Int64Value)), nil
+			case *schema_pb.Value_DoubleValue:
+				return parquet.Int32Value(int32(v.DoubleValue)), nil
+			case *schema_pb.Value_StringValue:
+				if i, err := strconv.ParseInt(v.StringValue, 10, 32); err == nil {
+					return parquet.Int32Value(int32(i)), nil
+				}
+				return parquet.Int32Value(0), nil
+			default:
+				return parquet.Int32Value(0), nil
+			}
+
+		case schema_pb.ScalarType_INT64:
+			switch v := value.Kind.(type) {
+			case *schema_pb.Value_Int64Value:
+				return parquet.Int64Value(v.Int64Value), nil
+			case *schema_pb.Value_Int32Value:
+				return parquet.Int64Value(int64(v.Int32Value)), nil
+			case *schema_pb.Value_DoubleValue:
+				return parquet.Int64Value(int64(v.DoubleValue)), nil
+			case *schema_pb.Value_StringValue:
+				if i, err := strconv.ParseInt(v.StringValue, 10, 64); err == nil {
+					return parquet.Int64Value(i), nil
+				}
+				return parquet.Int64Value(0), nil
+			default:
+				return parquet.Int64Value(0), nil
+			}
+
+		case schema_pb.ScalarType_FLOAT:
+			switch v := value.Kind.(type) {
+			case *schema_pb.Value_FloatValue:
+				return parquet.FloatValue(v.FloatValue), nil
+			case *schema_pb.Value_DoubleValue:
+				return parquet.FloatValue(float32(v.DoubleValue)), nil
+			case *schema_pb.Value_Int64Value:
+				return parquet.FloatValue(float32(v.Int64Value)), nil
+			case *schema_pb.Value_StringValue:
+				if f, err := strconv.ParseFloat(v.StringValue, 32); err == nil {
+					return parquet.FloatValue(float32(f)), nil
+				}
+				return parquet.FloatValue(0), nil
+			default:
+				return parquet.FloatValue(0), nil
+			}
+
+		case schema_pb.ScalarType_DOUBLE:
+			switch v := value.Kind.(type) {
+			case *schema_pb.Value_DoubleValue:
+				return parquet.DoubleValue(v.DoubleValue), nil
+			case *schema_pb.Value_Int64Value:
+				return parquet.DoubleValue(float64(v.Int64Value)), nil
+			case *schema_pb.Value_Int32Value:
+				return parquet.DoubleValue(float64(v.Int32Value)), nil
+			case *schema_pb.Value_StringValue:
+				if f, err := strconv.ParseFloat(v.StringValue, 64); err == nil {
+					return parquet.DoubleValue(f), nil
+				}
+				return parquet.DoubleValue(0), nil
+			default:
+				return parquet.DoubleValue(0), nil
+			}
+
+		case schema_pb.ScalarType_BYTES:
+			switch v := value.Kind.(type) {
+			case *schema_pb.Value_BytesValue:
+				b := v.BytesValue
+				if b == nil {
+					b = []byte{}
+				}
+				return parquet.ByteArrayValue(b), nil
+			case *schema_pb.Value_StringValue:
+				return parquet.ByteArrayValue([]byte(v.StringValue)), nil
+			case *schema_pb.Value_Int64Value:
+				return parquet.ByteArrayValue([]byte(strconv.FormatInt(v.Int64Value, 10))), nil
+			case *schema_pb.Value_Int32Value:
+				return parquet.ByteArrayValue([]byte(strconv.FormatInt(int64(v.Int32Value), 10))), nil
+			case *schema_pb.Value_DoubleValue:
+				return parquet.ByteArrayValue([]byte(strconv.FormatFloat(v.DoubleValue, 'f', -1, 64))), nil
+			case *schema_pb.Value_FloatValue:
+				return parquet.ByteArrayValue([]byte(strconv.FormatFloat(float64(v.FloatValue), 'f', -1, 32))), nil
+			case *schema_pb.Value_BoolValue:
+				if v.BoolValue {
+					return parquet.ByteArrayValue([]byte("true")), nil
+				}
+				return parquet.ByteArrayValue([]byte("false")), nil
+			default:
+				return parquet.ByteArrayValue([]byte{}), nil
+			}
+
+		case schema_pb.ScalarType_STRING:
+			// Same as bytes but semantically string
+			switch v := value.Kind.(type) {
+			case *schema_pb.Value_StringValue:
+				return parquet.ByteArrayValue([]byte(v.StringValue)), nil
+			default:
+				// Fallback through bytes coercion
+				b, _ := toParquetValueForType(&schema_pb.Type{Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_BYTES}}, value)
+				return b, nil
+			}
+
+		case schema_pb.ScalarType_TIMESTAMP:
+			switch v := value.Kind.(type) {
+			case *schema_pb.Value_Int64Value:
+				return parquet.Int64Value(v.Int64Value), nil
+			case *schema_pb.Value_StringValue:
+				if i, err := strconv.ParseInt(v.StringValue, 10, 64); err == nil {
+					return parquet.Int64Value(i), nil
+				}
+				return parquet.Int64Value(0), nil
+			default:
+				return parquet.Int64Value(0), nil
+			}
+
+		case schema_pb.ScalarType_DATE:
+			switch v := value.Kind.(type) {
+			case *schema_pb.Value_Int32Value:
+				return parquet.Int32Value(v.Int32Value), nil
+			case *schema_pb.Value_Int64Value:
+				return parquet.Int32Value(int32(v.Int64Value)), nil
+			case *schema_pb.Value_StringValue:
+				if i, err := strconv.ParseInt(v.StringValue, 10, 32); err == nil {
+					return parquet.Int32Value(int32(i)), nil
+				}
+				return parquet.Int32Value(0), nil
+			default:
+				return parquet.Int32Value(0), nil
+			}
+
+		case schema_pb.ScalarType_DECIMAL:
+			// Reuse existing conversion path (FixedLenByteArray 16)
+			return toParquetValue(value)
+
+		case schema_pb.ScalarType_TIME:
+			switch v := value.Kind.(type) {
+			case *schema_pb.Value_Int64Value:
+				return parquet.Int64Value(v.Int64Value), nil
+			case *schema_pb.Value_StringValue:
+				if i, err := strconv.ParseInt(v.StringValue, 10, 64); err == nil {
+					return parquet.Int64Value(i), nil
+				}
+				return parquet.Int64Value(0), nil
+			default:
+				return parquet.Int64Value(0), nil
+			}
+		}
+	}
+	// Fallback to generic conversion
+	return toParquetValue(value)
+}

weed/mq/schema/to_parquet_value_test.go

@@ -0,0 +1,666 @@
+package schema
+
+import (
+	"math/big"
+	"testing"
+	"time"
+
+	"github.com/parquet-go/parquet-go"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+func TestToParquetValue_BasicTypes(t *testing.T) {
+	tests := []struct {
+		name     string
+		value    *schema_pb.Value
+		expected parquet.Value
+		wantErr  bool
+	}{
+		{
+			name: "BoolValue true",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_BoolValue{BoolValue: true},
+			},
+			expected: parquet.BooleanValue(true),
+		},
+		{
+			name: "Int32Value",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_Int32Value{Int32Value: 42},
+			},
+			expected: parquet.Int32Value(42),
+		},
+		{
+			name: "Int64Value",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_Int64Value{Int64Value: 12345678901234},
+			},
+			expected: parquet.Int64Value(12345678901234),
+		},
+		{
+			name: "FloatValue",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_FloatValue{FloatValue: 3.14159},
+			},
+			expected: parquet.FloatValue(3.14159),
+		},
+		{
+			name: "DoubleValue",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_DoubleValue{DoubleValue: 2.718281828},
+			},
+			expected: parquet.DoubleValue(2.718281828),
+		},
+		{
+			name: "BytesValue",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_BytesValue{BytesValue: []byte("hello world")},
+			},
+			expected: parquet.ByteArrayValue([]byte("hello world")),
+		},
+		{
+			name: "BytesValue empty",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_BytesValue{BytesValue: []byte{}},
+			},
+			expected: parquet.ByteArrayValue([]byte{}),
+		},
+		{
+			name: "StringValue",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_StringValue{StringValue: "test string"},
+			},
+			expected: parquet.ByteArrayValue([]byte("test string")),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := toParquetValue(tt.value)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("toParquetValue() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !parquetValuesEqual(result, tt.expected) {
+				t.Errorf("toParquetValue() = %v, want %v", result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestToParquetValue_TimestampValue(t *testing.T) {
+	tests := []struct {
+		name     string
+		value    *schema_pb.Value
+		expected parquet.Value
+		wantErr  bool
+	}{
+		{
+			name: "Valid TimestampValue UTC",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_TimestampValue{
+					TimestampValue: &schema_pb.TimestampValue{
+						TimestampMicros: 1704067200000000, // 2024-01-01 00:00:00 UTC in microseconds
+						IsUtc:           true,
+					},
+				},
+			},
+			expected: parquet.Int64Value(1704067200000000),
+		},
+		{
+			name: "Valid TimestampValue local",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_TimestampValue{
+					TimestampValue: &schema_pb.TimestampValue{
+						TimestampMicros: 1704067200000000,
+						IsUtc:           false,
+					},
+				},
+			},
+			expected: parquet.Int64Value(1704067200000000),
+		},
+		{
+			name: "TimestampValue zero",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_TimestampValue{
+					TimestampValue: &schema_pb.TimestampValue{
+						TimestampMicros: 0,
+						IsUtc:           true,
+					},
+				},
+			},
+			expected: parquet.Int64Value(0),
+		},
+		{
+			name: "TimestampValue negative (before epoch)",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_TimestampValue{
+					TimestampValue: &schema_pb.TimestampValue{
+						TimestampMicros: -1000000, // 1 second before epoch
+						IsUtc:           true,
+					},
+				},
+			},
+			expected: parquet.Int64Value(-1000000),
+		},
+		{
+			name: "TimestampValue nil pointer",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_TimestampValue{
+					TimestampValue: nil,
+				},
+			},
+			expected: parquet.NullValue(),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := toParquetValue(tt.value)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("toParquetValue() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !parquetValuesEqual(result, tt.expected) {
+				t.Errorf("toParquetValue() = %v, want %v", result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestToParquetValue_DateValue(t *testing.T) {
+	tests := []struct {
+		name     string
+		value    *schema_pb.Value
+		expected parquet.Value
+		wantErr  bool
+	}{
+		{
+			name: "Valid DateValue (2024-01-01)",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_DateValue{
+					DateValue: &schema_pb.DateValue{
+						DaysSinceEpoch: 19723, // 2024-01-01 = 19723 days since epoch
+					},
+				},
+			},
+			expected: parquet.Int32Value(19723),
+		},
+		{
+			name: "DateValue epoch (1970-01-01)",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_DateValue{
+					DateValue: &schema_pb.DateValue{
+						DaysSinceEpoch: 0,
+					},
+				},
+			},
+			expected: parquet.Int32Value(0),
+		},
+		{
+			name: "DateValue before epoch",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_DateValue{
+					DateValue: &schema_pb.DateValue{
+						DaysSinceEpoch: -365, // 1969-01-01
+					},
+				},
+			},
+			expected: parquet.Int32Value(-365),
+		},
+		{
+			name: "DateValue nil pointer",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_DateValue{
+					DateValue: nil,
+				},
+			},
+			expected: parquet.NullValue(),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := toParquetValue(tt.value)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("toParquetValue() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !parquetValuesEqual(result, tt.expected) {
+				t.Errorf("toParquetValue() = %v, want %v", result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestToParquetValue_DecimalValue(t *testing.T) {
+	tests := []struct {
+		name     string
+		value    *schema_pb.Value
+		expected parquet.Value
+		wantErr  bool
+	}{
+		{
+			name: "Small Decimal (precision <= 9) - positive",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_DecimalValue{
+					DecimalValue: &schema_pb.DecimalValue{
+						Value:     encodeBigIntToBytes(big.NewInt(12345)), // 123.45 with scale 2
+						Precision: 5,
+						Scale:     2,
+					},
+				},
+			},
+			expected: createFixedLenByteArray(encodeBigIntToBytes(big.NewInt(12345))), // FixedLenByteArray conversion
+		},
+		{
+			name: "Small Decimal (precision <= 9) - negative",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_DecimalValue{
+					DecimalValue: &schema_pb.DecimalValue{
+						Value:     encodeBigIntToBytes(big.NewInt(-12345)),
+						Precision: 5,
+						Scale:     2,
+					},
+				},
+			},
+			expected: createFixedLenByteArray(encodeBigIntToBytes(big.NewInt(-12345))), // FixedLenByteArray conversion
+		},
+		{
+			name: "Medium Decimal (9 < precision <= 18)",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_DecimalValue{
+					DecimalValue: &schema_pb.DecimalValue{
+						Value:     encodeBigIntToBytes(big.NewInt(123456789012345)),
+						Precision: 15,
+						Scale:     2,
+					},
+				},
+			},
+			expected: createFixedLenByteArray(encodeBigIntToBytes(big.NewInt(123456789012345))), // FixedLenByteArray conversion
+		},
+		{
+			name: "Large Decimal (precision > 18)",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_DecimalValue{
+					DecimalValue: &schema_pb.DecimalValue{
+						Value:     []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, // Large number as bytes
+						Precision: 25,
+						Scale:     5,
+					},
+				},
+			},
+			expected: createFixedLenByteArray([]byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}), // FixedLenByteArray conversion
+		},
+		{
+			name: "Decimal with zero precision",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_DecimalValue{
+					DecimalValue: &schema_pb.DecimalValue{
+						Value:     encodeBigIntToBytes(big.NewInt(0)),
+						Precision: 0,
+						Scale:     0,
+					},
+				},
+			},
+			expected: createFixedLenByteArray(encodeBigIntToBytes(big.NewInt(0))), // Zero as FixedLenByteArray
+		},
+		{
+			name: "Decimal nil pointer",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_DecimalValue{
+					DecimalValue: nil,
+				},
+			},
+			expected: parquet.NullValue(),
+		},
+		{
+			name: "Decimal with nil Value bytes",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_DecimalValue{
+					DecimalValue: &schema_pb.DecimalValue{
+						Value:     nil, // This was the original panic cause
+						Precision: 5,
+						Scale:     2,
+					},
+				},
+			},
+			expected: parquet.NullValue(),
+		},
+		{
+			name: "Decimal with empty Value bytes",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_DecimalValue{
+					DecimalValue: &schema_pb.DecimalValue{
+						Value:     []byte{}, // Empty slice
+						Precision: 5,
+						Scale:     2,
+					},
+				},
+			},
+			expected: parquet.NullValue(), // Returns null for empty bytes
+		},
+		{
+			name: "Decimal out of int32 range (stored as binary)",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_DecimalValue{
+					DecimalValue: &schema_pb.DecimalValue{
+						Value:     encodeBigIntToBytes(big.NewInt(999999999999)), // Too large for int32
+						Precision: 5,                                             // But precision says int32
+						Scale:     0,
+					},
+				},
+			},
+			expected: createFixedLenByteArray(encodeBigIntToBytes(big.NewInt(999999999999))), // FixedLenByteArray
+		},
+		{
+			name: "Decimal out of int64 range (stored as binary)",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_DecimalValue{
+					DecimalValue: &schema_pb.DecimalValue{
+						Value: func() []byte {
+							// Create a number larger than int64 max
+							bigNum := new(big.Int)
+							bigNum.SetString("99999999999999999999999999999", 10)
+							return encodeBigIntToBytes(bigNum)
+						}(),
+						Precision: 15, // Says int64 but value is too large
+						Scale:     0,
+					},
+				},
+			},
+			expected: createFixedLenByteArray(func() []byte {
+				bigNum := new(big.Int)
+				bigNum.SetString("99999999999999999999999999999", 10)
+				return encodeBigIntToBytes(bigNum)
+			}()), // Large number as FixedLenByteArray (truncated to 16 bytes)
+		},
+		{
+			name: "Decimal extremely large value (should be rejected)",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_DecimalValue{
+					DecimalValue: &schema_pb.DecimalValue{
+						Value:     make([]byte, 100), // 100 bytes > 64 byte limit
+						Precision: 100,
+						Scale:     0,
+					},
+				},
+			},
+			expected: parquet.NullValue(),
+			wantErr:  true, // Should return error instead of corrupting data
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := toParquetValue(tt.value)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("toParquetValue() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !parquetValuesEqual(result, tt.expected) {
+				t.Errorf("toParquetValue() = %v, want %v", result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestToParquetValue_TimeValue(t *testing.T) {
+	tests := []struct {
+		name     string
+		value    *schema_pb.Value
+		expected parquet.Value
+		wantErr  bool
+	}{
+		{
+			name: "Valid TimeValue (12:34:56.789)",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_TimeValue{
+					TimeValue: &schema_pb.TimeValue{
+						TimeMicros: 45296789000, // 12:34:56.789 in microseconds since midnight
+					},
+				},
+			},
+			expected: parquet.Int64Value(45296789000),
+		},
+		{
+			name: "TimeValue midnight",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_TimeValue{
+					TimeValue: &schema_pb.TimeValue{
+						TimeMicros: 0,
+					},
+				},
+			},
+			expected: parquet.Int64Value(0),
+		},
+		{
+			name: "TimeValue end of day (23:59:59.999999)",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_TimeValue{
+					TimeValue: &schema_pb.TimeValue{
+						TimeMicros: 86399999999, // 23:59:59.999999
+					},
+				},
+			},
+			expected: parquet.Int64Value(86399999999),
+		},
+		{
+			name: "TimeValue nil pointer",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_TimeValue{
+					TimeValue: nil,
+				},
+			},
+			expected: parquet.NullValue(),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := toParquetValue(tt.value)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("toParquetValue() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !parquetValuesEqual(result, tt.expected) {
+				t.Errorf("toParquetValue() = %v, want %v", result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestToParquetValue_EdgeCases(t *testing.T) {
+	tests := []struct {
+		name     string
+		value    *schema_pb.Value
+		expected parquet.Value
+		wantErr  bool
+	}{
+		{
+			name: "Nil value",
+			value: &schema_pb.Value{
+				Kind: nil,
+			},
+			wantErr: true,
+		},
+		{
+			name:    "Completely nil value",
+			value:   nil,
+			wantErr: true,
+		},
+		{
+			name: "BytesValue with nil slice",
+			value: &schema_pb.Value{
+				Kind: &schema_pb.Value_BytesValue{BytesValue: nil},
+			},
+			expected: parquet.ByteArrayValue([]byte{}), // Should convert nil to empty slice
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := toParquetValue(tt.value)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("toParquetValue() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !tt.wantErr && !parquetValuesEqual(result, tt.expected) {
+				t.Errorf("toParquetValue() = %v, want %v", result, tt.expected)
+			}
+		})
+	}
+}
+
+// Helper function to encode a big.Int to bytes using two's complement representation
+func encodeBigIntToBytes(n *big.Int) []byte {
+	if n.Sign() == 0 {
+		return []byte{0}
+	}
+
+	// For positive numbers, just use Bytes()
+	if n.Sign() > 0 {
+		return n.Bytes()
+	}
+
+	// For negative numbers, we need two's complement representation
+	bitLen := n.BitLen()
+	if bitLen%8 != 0 {
+		bitLen += 8 - (bitLen % 8) // Round up to byte boundary
+	}
+	byteLen := bitLen / 8
+	if byteLen == 0 {
+		byteLen = 1
+	}
+
+	// Calculate 2^(byteLen*8)
+	modulus := new(big.Int).Lsh(big.NewInt(1), uint(byteLen*8))
+
+	// Convert negative to positive representation: n + 2^(byteLen*8)
+	positive := new(big.Int).Add(n, modulus)
+
+	bytes := positive.Bytes()
+
+	// Pad with leading zeros if needed
+	if len(bytes) < byteLen {
+		padded := make([]byte, byteLen)
+		copy(padded[byteLen-len(bytes):], bytes)
+		return padded
+	}
+
+	return bytes
+}
+
+// Helper function to create a FixedLenByteArray(16) matching our conversion logic
+func createFixedLenByteArray(inputBytes []byte) parquet.Value {
+	fixedBytes := make([]byte, 16)
+	if len(inputBytes) <= 16 {
+		// Right-align the value (big-endian) - same as our conversion logic
+		copy(fixedBytes[16-len(inputBytes):], inputBytes)
+	} else {
+		// Truncate if too large, taking the least significant bytes
+		copy(fixedBytes, inputBytes[len(inputBytes)-16:])
+	}
+	return parquet.FixedLenByteArrayValue(fixedBytes)
+}
+
+// Helper function to compare parquet values
+func parquetValuesEqual(a, b parquet.Value) bool {
+	// Handle both being null
+	if a.IsNull() && b.IsNull() {
+		return true
+	}
+	if a.IsNull() != b.IsNull() {
+		return false
+	}
+
+	// Compare kind first
+	if a.Kind() != b.Kind() {
+		return false
+	}
+
+	// Compare based on type
+	switch a.Kind() {
+	case parquet.Boolean:
+		return a.Boolean() == b.Boolean()
+	case parquet.Int32:
+		return a.Int32() == b.Int32()
+	case parquet.Int64:
+		return a.Int64() == b.Int64()
+	case parquet.Float:
+		return a.Float() == b.Float()
+	case parquet.Double:
+		return a.Double() == b.Double()
+	case parquet.ByteArray:
+		aBytes := a.ByteArray()
+		bBytes := b.ByteArray()
+		if len(aBytes) != len(bBytes) {
+			return false
+		}
+		for i, v := range aBytes {
+			if v != bBytes[i] {
+				return false
+			}
+		}
+		return true
+	case parquet.FixedLenByteArray:
+		aBytes := a.ByteArray() // FixedLenByteArray also uses ByteArray() method
+		bBytes := b.ByteArray()
+		if len(aBytes) != len(bBytes) {
+			return false
+		}
+		for i, v := range aBytes {
+			if v != bBytes[i] {
+				return false
+			}
+		}
+		return true
+	default:
+		return false
+	}
+}
+
+// Benchmark tests
+func BenchmarkToParquetValue_BasicTypes(b *testing.B) {
+	value := &schema_pb.Value{
+		Kind: &schema_pb.Value_Int64Value{Int64Value: 12345678901234},
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, _ = toParquetValue(value)
+	}
+}
+
+func BenchmarkToParquetValue_TimestampValue(b *testing.B) {
+	value := &schema_pb.Value{
+		Kind: &schema_pb.Value_TimestampValue{
+			TimestampValue: &schema_pb.TimestampValue{
+				TimestampMicros: time.Now().UnixMicro(),
+				IsUtc:           true,
+			},
+		},
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, _ = toParquetValue(value)
+	}
+}
+
+func BenchmarkToParquetValue_DecimalValue(b *testing.B) {
+	value := &schema_pb.Value{
+		Kind: &schema_pb.Value_DecimalValue{
+			DecimalValue: &schema_pb.DecimalValue{
+				Value:     encodeBigIntToBytes(big.NewInt(123456789012345)),
+				Precision: 15,
+				Scale:     2,
+			},
+		},
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, _ = toParquetValue(value)
+	}
+}

weed/mq/schema/to_schema_value.go

@@ -0,0 +1,146 @@
+package schema
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/parquet-go/parquet-go"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+// ToRecordValue converts a parquet.Row to a schema_pb.RecordValue
+// This does not work or did not test with nested structures.
+// Using this may fail to convert the parquet.Row to schema_pb.RecordValue
+func ToRecordValue(recordType *schema_pb.RecordType, parquetLevels *ParquetLevels, row parquet.Row) (*schema_pb.RecordValue, error) {
+	values := []parquet.Value(row)
+	recordValue, _, err := toRecordValue(recordType, parquetLevels, values, 0)
+	if err != nil {
+		return nil, err
+	}
+	return recordValue.GetRecordValue(), nil
+}
+
+func ToValue(t *schema_pb.Type, levels *ParquetLevels, values []parquet.Value, valueIndex int) (value *schema_pb.Value, endValueIndex int, err error) {
+	switch t.Kind.(type) {
+	case *schema_pb.Type_ScalarType:
+		return toScalarValue(t.GetScalarType(), levels, values, valueIndex)
+	case *schema_pb.Type_ListType:
+		return toListValue(t.GetListType(), levels, values, valueIndex)
+	case *schema_pb.Type_RecordType:
+		return toRecordValue(t.GetRecordType(), levels, values, valueIndex)
+	}
+	return nil, valueIndex, fmt.Errorf("unsupported type: %v", t)
+}
+
+func toRecordValue(recordType *schema_pb.RecordType, levels *ParquetLevels, values []parquet.Value, valueIndex int) (*schema_pb.Value, int, error) {
+	recordValue := schema_pb.RecordValue{Fields: make(map[string]*schema_pb.Value)}
+	for _, field := range recordType.Fields {
+		fieldLevels := levels.levels[field.Name]
+		fieldValue, endValueIndex, err := ToValue(field.Type, fieldLevels, values, valueIndex)
+		if err != nil {
+			return nil, 0, err
+		}
+		valueIndex = endValueIndex
+		recordValue.Fields[field.Name] = fieldValue
+	}
+	return &schema_pb.Value{Kind: &schema_pb.Value_RecordValue{RecordValue: &recordValue}}, valueIndex, nil
+}
+
+func toListValue(listType *schema_pb.ListType, levels *ParquetLevels, values []parquet.Value, valueIndex int) (listValue *schema_pb.Value, endValueIndex int, err error) {
+	listValues := make([]*schema_pb.Value, 0)
+	var value *schema_pb.Value
+	for valueIndex < len(values) {
+		if values[valueIndex].Column() != levels.startColumnIndex {
+			break
+		}
+		value, valueIndex, err = ToValue(listType.ElementType, levels, values, valueIndex)
+		if err != nil {
+			return nil, valueIndex, err
+		}
+		listValues = append(listValues, value)
+	}
+	return &schema_pb.Value{Kind: &schema_pb.Value_ListValue{ListValue: &schema_pb.ListValue{Values: listValues}}}, valueIndex, nil
+}
+
+func toScalarValue(scalarType schema_pb.ScalarType, levels *ParquetLevels, values []parquet.Value, valueIndex int) (*schema_pb.Value, int, error) {
+	value := values[valueIndex]
+	if value.Column() != levels.startColumnIndex {
+		return nil, valueIndex, nil
+	}
+	switch scalarType {
+	case schema_pb.ScalarType_BOOL:
+		return &schema_pb.Value{Kind: &schema_pb.Value_BoolValue{BoolValue: value.Boolean()}}, valueIndex + 1, nil
+	case schema_pb.ScalarType_INT32:
+		return &schema_pb.Value{Kind: &schema_pb.Value_Int32Value{Int32Value: value.Int32()}}, valueIndex + 1, nil
+	case schema_pb.ScalarType_INT64:
+		return &schema_pb.Value{Kind: &schema_pb.Value_Int64Value{Int64Value: value.Int64()}}, valueIndex + 1, nil
+	case schema_pb.ScalarType_FLOAT:
+		return &schema_pb.Value{Kind: &schema_pb.Value_FloatValue{FloatValue: value.Float()}}, valueIndex + 1, nil
+	case schema_pb.ScalarType_DOUBLE:
+		return &schema_pb.Value{Kind: &schema_pb.Value_DoubleValue{DoubleValue: value.Double()}}, valueIndex + 1, nil
+	case schema_pb.ScalarType_BYTES:
+		// Handle nil byte arrays from parquet to prevent growslice panic
+		byteData := value.ByteArray()
+		if byteData == nil {
+			byteData = []byte{} // Use empty slice instead of nil
+		}
+		return &schema_pb.Value{Kind: &schema_pb.Value_BytesValue{BytesValue: byteData}}, valueIndex + 1, nil
+	case schema_pb.ScalarType_STRING:
+		// Handle nil byte arrays from parquet to prevent string conversion issues
+		byteData := value.ByteArray()
+		if byteData == nil {
+			byteData = []byte{} // Use empty slice instead of nil
+		}
+		return &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: string(byteData)}}, valueIndex + 1, nil
+	// Parquet logical types - convert from their physical storage back to logical values
+	case schema_pb.ScalarType_TIMESTAMP:
+		// Stored as INT64, convert back to TimestampValue
+		return &schema_pb.Value{
+			Kind: &schema_pb.Value_TimestampValue{
+				TimestampValue: &schema_pb.TimestampValue{
+					TimestampMicros: value.Int64(),
+					IsUtc:           true, // Default to UTC for compatibility
+				},
+			},
+		}, valueIndex + 1, nil
+	case schema_pb.ScalarType_DATE:
+		// Stored as INT32, convert back to DateValue
+		return &schema_pb.Value{
+			Kind: &schema_pb.Value_DateValue{
+				DateValue: &schema_pb.DateValue{
+					DaysSinceEpoch: value.Int32(),
+				},
+			},
+		}, valueIndex + 1, nil
+	case schema_pb.ScalarType_DECIMAL:
+		// Stored as FixedLenByteArray, convert back to DecimalValue
+		fixedBytes := value.ByteArray() // FixedLenByteArray also uses ByteArray() method
+		if fixedBytes == nil {
+			fixedBytes = []byte{} // Use empty slice instead of nil
+		}
+		// Remove leading zeros to get the minimal representation
+		trimmedBytes := bytes.TrimLeft(fixedBytes, "\x00")
+		if len(trimmedBytes) == 0 {
+			trimmedBytes = []byte{0} // Ensure we have at least one byte for zero
+		}
+		return &schema_pb.Value{
+			Kind: &schema_pb.Value_DecimalValue{
+				DecimalValue: &schema_pb.DecimalValue{
+					Value:     trimmedBytes,
+					Precision: 38, // Maximum precision supported by schema
+					Scale:     18, // Maximum scale supported by schema
+				},
+			},
+		}, valueIndex + 1, nil
+	case schema_pb.ScalarType_TIME:
+		// Stored as INT64, convert back to TimeValue
+		return &schema_pb.Value{
+			Kind: &schema_pb.Value_TimeValue{
+				TimeValue: &schema_pb.TimeValue{
+					TimeMicros: value.Int64(),
+				},
+			},
+		}, valueIndex + 1, nil
+	}
+	return nil, valueIndex, fmt.Errorf("unsupported scalar type: %v", scalarType)
+}

weed/mq/schema/value_builder.go

@@ -0,0 +1,113 @@
+package schema
+
+import "github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+
+// RecordValueBuilder helps in constructing RecordValue protobuf messages
+type RecordValueBuilder struct {
+	recordValue *schema_pb.RecordValue
+}
+
+// RecordBegin creates a new RecordValueBuilder instance
+func RecordBegin() *RecordValueBuilder {
+	return &RecordValueBuilder{recordValue: &schema_pb.RecordValue{Fields: make(map[string]*schema_pb.Value)}}
+}
+
+// RecordEnd returns the constructed RecordValue message
+func (rvb *RecordValueBuilder) RecordEnd() *schema_pb.RecordValue {
+	return rvb.recordValue
+}
+
+func (rvb *RecordValueBuilder) SetBool(key string, value bool) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_BoolValue{BoolValue: value}}
+	return rvb
+}
+func (rvb *RecordValueBuilder) SetInt32(key string, value int32) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_Int32Value{Int32Value: value}}
+	return rvb
+}
+func (rvb *RecordValueBuilder) SetInt64(key string, value int64) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_Int64Value{Int64Value: value}}
+	return rvb
+}
+func (rvb *RecordValueBuilder) SetFloat(key string, value float32) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_FloatValue{FloatValue: value}}
+	return rvb
+}
+func (rvb *RecordValueBuilder) SetDouble(key string, value float64) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_DoubleValue{DoubleValue: value}}
+	return rvb
+}
+func (rvb *RecordValueBuilder) SetBytes(key string, value []byte) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_BytesValue{BytesValue: value}}
+	return rvb
+}
+func (rvb *RecordValueBuilder) SetString(key string, value string) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: value}}
+	return rvb
+}
+func (rvb *RecordValueBuilder) SetRecord(key string, value *schema_pb.RecordValue) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_RecordValue{RecordValue: value}}
+	return rvb
+}
+
+func (rvb *RecordValueBuilder) addListValue(key string, values []*schema_pb.Value) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_ListValue{ListValue: &schema_pb.ListValue{Values: values}}}
+	return rvb
+}
+
+func (rvb *RecordValueBuilder) SetBoolList(key string, values ...bool) *RecordValueBuilder {
+	var listValues []*schema_pb.Value
+	for _, v := range values {
+		listValues = append(listValues, &schema_pb.Value{Kind: &schema_pb.Value_BoolValue{BoolValue: v}})
+	}
+	return rvb.addListValue(key, listValues)
+}
+func (rvb *RecordValueBuilder) SetInt32List(key string, values ...int32) *RecordValueBuilder {
+	var listValues []*schema_pb.Value
+	for _, v := range values {
+		listValues = append(listValues, &schema_pb.Value{Kind: &schema_pb.Value_Int32Value{Int32Value: v}})
+	}
+	return rvb.addListValue(key, listValues)
+}
+func (rvb *RecordValueBuilder) SetInt64List(key string, values ...int64) *RecordValueBuilder {
+	var listValues []*schema_pb.Value
+	for _, v := range values {
+		listValues = append(listValues, &schema_pb.Value{Kind: &schema_pb.Value_Int64Value{Int64Value: v}})
+	}
+	return rvb.addListValue(key, listValues)
+}
+func (rvb *RecordValueBuilder) SetFloatList(key string, values ...float32) *RecordValueBuilder {
+	var listValues []*schema_pb.Value
+	for _, v := range values {
+		listValues = append(listValues, &schema_pb.Value{Kind: &schema_pb.Value_FloatValue{FloatValue: v}})
+	}
+	return rvb.addListValue(key, listValues)
+}
+func (rvb *RecordValueBuilder) SetDoubleList(key string, values ...float64) *RecordValueBuilder {
+	var listValues []*schema_pb.Value
+	for _, v := range values {
+		listValues = append(listValues, &schema_pb.Value{Kind: &schema_pb.Value_DoubleValue{DoubleValue: v}})
+	}
+	return rvb.addListValue(key, listValues)
+}
+func (rvb *RecordValueBuilder) SetBytesList(key string, values ...[]byte) *RecordValueBuilder {
+	var listValues []*schema_pb.Value
+	for _, v := range values {
+		listValues = append(listValues, &schema_pb.Value{Kind: &schema_pb.Value_BytesValue{BytesValue: v}})
+	}
+	return rvb.addListValue(key, listValues)
+}
+func (rvb *RecordValueBuilder) SetStringList(key string, values ...string) *RecordValueBuilder {
+	var listValues []*schema_pb.Value
+	for _, v := range values {
+		listValues = append(listValues, &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: v}})
+	}
+	return rvb.addListValue(key, listValues)
+}
+func (rvb *RecordValueBuilder) SetRecordList(key string, values ...*schema_pb.RecordValue) *RecordValueBuilder {
+	var listValues []*schema_pb.Value
+	for _, v := range values {
+		listValues = append(listValues, &schema_pb.Value{Kind: &schema_pb.Value_RecordValue{RecordValue: v}})
+	}
+	return rvb.addListValue(key, listValues)
+}

weed/mq/schema/write_parquet_test.go

@@ -0,0 +1,174 @@
+package schema
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/parquet-go/parquet-go"
+	"github.com/parquet-go/parquet-go/compress/zstd"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+func TestWriteReadParquet(t *testing.T) {
+	// create a schema_pb.RecordType
+	recordType := RecordTypeBegin().
+		WithField("ID", TypeInt64).
+		WithField("CreatedAt", TypeInt64).
+		WithRecordField("Person",
+			RecordTypeBegin().
+				WithField("zName", TypeString).
+				WithField("emails", ListOf(TypeString)).
+				RecordTypeEnd()).
+		WithField("Company", TypeString).
+		WithRecordField("Address",
+			RecordTypeBegin().
+				WithField("Street", TypeString).
+				WithField("City", TypeString).
+				RecordTypeEnd()).
+		RecordTypeEnd()
+	fmt.Printf("RecordType: %v\n", recordType)
+
+	// create a parquet schema
+	parquetSchema, err := ToParquetSchema("example", recordType)
+	if err != nil {
+		t.Fatalf("ToParquetSchema failed: %v", err)
+	}
+	fmt.Printf("ParquetSchema: %v\n", parquetSchema)
+
+	fmt.Printf("Go Type: %+v\n", parquetSchema.GoType())
+
+	filename := "example.parquet"
+
+	count := 3
+
+	testWritingParquetFile(t, count, filename, parquetSchema, recordType)
+
+	total := testReadingParquetFile(t, filename, parquetSchema, recordType)
+
+	if total != count {
+		t.Fatalf("total != 128*1024: %v", total)
+	}
+
+	if err = os.Remove(filename); err != nil {
+		t.Fatalf("os.Remove failed: %v", err)
+	}
+
+}
+
+func testWritingParquetFile(t *testing.T, count int, filename string, parquetSchema *parquet.Schema, recordType *schema_pb.RecordType) {
+	parquetLevels, err := ToParquetLevels(recordType)
+	if err != nil {
+		t.Fatalf("ToParquetLevels failed: %v", err)
+	}
+
+	// create a parquet file
+	file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0664)
+	if err != nil {
+		t.Fatalf("os.Open failed: %v", err)
+	}
+	defer file.Close()
+	writer := parquet.NewWriter(file, parquetSchema, parquet.Compression(&zstd.Codec{Level: zstd.DefaultLevel}))
+	rowBuilder := parquet.NewRowBuilder(parquetSchema)
+	for i := 0; i < count; i++ {
+		rowBuilder.Reset()
+		// generate random data
+		recordValue := RecordBegin().
+			SetInt64("ID", 1+int64(i)).
+			SetInt64("CreatedAt", 2+2*int64(i)).
+			SetRecord("Person",
+				RecordBegin().
+					SetString("zName", fmt.Sprintf("john_%d", i)).
+					SetStringList("emails",
+						fmt.Sprintf("john_%d@a.com", i),
+						fmt.Sprintf("john_%d@b.com", i),
+						fmt.Sprintf("john_%d@c.com", i),
+						fmt.Sprintf("john_%d@d.com", i),
+						fmt.Sprintf("john_%d@e.com", i)).
+					RecordEnd()).
+			SetString("Company", fmt.Sprintf("company_%d", i)).
+			RecordEnd()
+		AddRecordValue(rowBuilder, recordType, parquetLevels, recordValue)
+
+		if count < 10 {
+			fmt.Printf("Write RecordValue: %v\n", recordValue)
+		}
+
+		row := rowBuilder.Row()
+
+		if count < 10 {
+			fmt.Printf("Build Row: %+v\n", row)
+		}
+
+		if err != nil {
+			t.Fatalf("rowBuilder.Build failed: %v", err)
+		}
+
+		if _, err = writer.WriteRows([]parquet.Row{row}); err != nil {
+			t.Fatalf("writer.Write failed: %v", err)
+		}
+	}
+	if err = writer.Close(); err != nil {
+		t.Fatalf("writer.WriteStop failed: %v", err)
+	}
+}
+
+func testReadingParquetFile(t *testing.T, filename string, parquetSchema *parquet.Schema, recordType *schema_pb.RecordType) (total int) {
+	parquetLevels, err := ToParquetLevels(recordType)
+	if err != nil {
+		t.Fatalf("ToParquetLevels failed: %v", err)
+	}
+
+	// read the parquet file
+	file, err := os.Open(filename)
+	if err != nil {
+		t.Fatalf("os.Open failed: %v", err)
+	}
+	defer file.Close()
+
+	// Get file info to determine size
+	fileInfo, err := file.Stat()
+	if err != nil {
+		t.Fatalf("file.Stat failed: %v", err)
+	}
+
+	// Create a parquet file from the opened file
+	parquetFile, err := parquet.OpenFile(file, fileInfo.Size())
+	if err != nil {
+		t.Fatalf("parquet.OpenFile failed: %v", err)
+	}
+
+	reader := parquet.NewReader(parquetFile)
+	rows := make([]parquet.Row, 128)
+	for {
+		rowCount, err := reader.ReadRows(rows)
+
+		// Process the rows first, even if EOF is returned
+		for i := 0; i < rowCount; i++ {
+			row := rows[i]
+			// convert parquet row to schema_pb.RecordValue
+			recordValue, err := ToRecordValue(recordType, parquetLevels, row)
+			if err != nil {
+				t.Fatalf("ToRecordValue failed: %v", err)
+			}
+			if rowCount < 10 {
+				fmt.Printf("Read RecordValue: %v\n", recordValue)
+			}
+		}
+		total += rowCount
+
+		// Check for end conditions after processing rows
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			t.Fatalf("reader.Read failed: %v", err)
+		}
+		if rowCount == 0 {
+			break
+		}
+	}
+	fmt.Printf("total: %v\n", total)
+	return
+}