snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/mq/kafka/protocol/batch_crc_compat_test.go

@@ -0,0 +1,368 @@
+package protocol
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"hash/crc32"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/integration"
+)
+
+// TestBatchConstruction tests that our batch construction produces valid CRC
+func TestBatchConstruction(t *testing.T) {
+	// Create test data
+	key := []byte("test-key")
+	value := []byte("test-value")
+	timestamp := time.Now()
+
+	// Build batch using our implementation
+	batch := constructTestBatch(0, timestamp, key, value)
+
+	t.Logf("Batch size: %d bytes", len(batch))
+	t.Logf("Batch hex:\n%s", hexDumpTest(batch))
+
+	// Extract and verify CRC
+	if len(batch) < 21 {
+		t.Fatalf("Batch too short: %d bytes", len(batch))
+	}
+
+	storedCRC := binary.BigEndian.Uint32(batch[17:21])
+	t.Logf("Stored CRC: 0x%08x", storedCRC)
+
+	// Recalculate CRC from the data
+	crcData := batch[21:]
+	calculatedCRC := crc32.Checksum(crcData, crc32.MakeTable(crc32.Castagnoli))
+	t.Logf("Calculated CRC: 0x%08x (over %d bytes)", calculatedCRC, len(crcData))
+
+	if storedCRC != calculatedCRC {
+		t.Errorf("CRC mismatch: stored=0x%08x calculated=0x%08x", storedCRC, calculatedCRC)
+
+		// Debug: show what bytes the CRC is calculated over
+		t.Logf("CRC data (first 100 bytes):")
+		dumpSize := 100
+		if len(crcData) < dumpSize {
+			dumpSize = len(crcData)
+		}
+		for i := 0; i < dumpSize; i += 16 {
+			end := i + 16
+			if end > dumpSize {
+				end = dumpSize
+			}
+			t.Logf("  %04d: %x", i, crcData[i:end])
+		}
+	} else {
+		t.Log("CRC verification PASSED")
+	}
+
+	// Verify batch structure
+	t.Log("\n=== Batch Structure ===")
+	verifyField(t, "Base Offset", batch[0:8], binary.BigEndian.Uint64(batch[0:8]))
+	verifyField(t, "Batch Length", batch[8:12], binary.BigEndian.Uint32(batch[8:12]))
+	verifyField(t, "Leader Epoch", batch[12:16], int32(binary.BigEndian.Uint32(batch[12:16])))
+	verifyField(t, "Magic", batch[16:17], batch[16])
+	verifyField(t, "CRC", batch[17:21], binary.BigEndian.Uint32(batch[17:21]))
+	verifyField(t, "Attributes", batch[21:23], binary.BigEndian.Uint16(batch[21:23]))
+	verifyField(t, "Last Offset Delta", batch[23:27], binary.BigEndian.Uint32(batch[23:27]))
+	verifyField(t, "Base Timestamp", batch[27:35], binary.BigEndian.Uint64(batch[27:35]))
+	verifyField(t, "Max Timestamp", batch[35:43], binary.BigEndian.Uint64(batch[35:43]))
+	verifyField(t, "Record Count", batch[57:61], binary.BigEndian.Uint32(batch[57:61]))
+
+	// Verify the batch length field is correct
+	expectedBatchLength := uint32(len(batch) - 12)
+	actualBatchLength := binary.BigEndian.Uint32(batch[8:12])
+	if expectedBatchLength != actualBatchLength {
+		t.Errorf("Batch length mismatch: expected=%d actual=%d", expectedBatchLength, actualBatchLength)
+	} else {
+		t.Logf("Batch length correct: %d", actualBatchLength)
+	}
+}
+
+// TestMultipleRecordsBatch tests batch construction with multiple records
+func TestMultipleRecordsBatch(t *testing.T) {
+	timestamp := time.Now()
+
+	// We can't easily test multiple records without the full implementation
+	// So let's test that our single record batch matches expected structure
+
+	batch1 := constructTestBatch(0, timestamp, []byte("key1"), []byte("value1"))
+	batch2 := constructTestBatch(1, timestamp, []byte("key2"), []byte("value2"))
+
+	t.Logf("Batch 1 size: %d, CRC: 0x%08x", len(batch1), binary.BigEndian.Uint32(batch1[17:21]))
+	t.Logf("Batch 2 size: %d, CRC: 0x%08x", len(batch2), binary.BigEndian.Uint32(batch2[17:21]))
+
+	// Verify both batches have valid CRCs
+	for i, batch := range [][]byte{batch1, batch2} {
+		storedCRC := binary.BigEndian.Uint32(batch[17:21])
+		calculatedCRC := crc32.Checksum(batch[21:], crc32.MakeTable(crc32.Castagnoli))
+
+		if storedCRC != calculatedCRC {
+			t.Errorf("Batch %d CRC mismatch: stored=0x%08x calculated=0x%08x", i+1, storedCRC, calculatedCRC)
+		} else {
+			t.Logf("Batch %d CRC valid", i+1)
+		}
+	}
+}
+
+// TestVarintEncoding tests our varint encoding implementation
+func TestVarintEncoding(t *testing.T) {
+	testCases := []struct {
+		value    int64
+		expected []byte
+	}{
+		{0, []byte{0x00}},
+		{1, []byte{0x02}},
+		{-1, []byte{0x01}},
+		{5, []byte{0x0a}},
+		{-5, []byte{0x09}},
+		{127, []byte{0xfe, 0x01}},
+		{128, []byte{0x80, 0x02}},
+		{-127, []byte{0xfd, 0x01}},
+		{-128, []byte{0xff, 0x01}},
+	}
+
+	for _, tc := range testCases {
+		result := encodeVarint(tc.value)
+		if !bytes.Equal(result, tc.expected) {
+			t.Errorf("encodeVarint(%d) = %x, expected %x", tc.value, result, tc.expected)
+		} else {
+			t.Logf("encodeVarint(%d) = %x", tc.value, result)
+		}
+	}
+}
+
+// constructTestBatch builds a batch using our implementation
+func constructTestBatch(baseOffset int64, timestamp time.Time, key, value []byte) []byte {
+	batch := make([]byte, 0, 256)
+
+	// Base offset (0-7)
+	baseOffsetBytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(baseOffsetBytes, uint64(baseOffset))
+	batch = append(batch, baseOffsetBytes...)
+
+	// Batch length placeholder (8-11)
+	batchLengthPos := len(batch)
+	batch = append(batch, 0, 0, 0, 0)
+
+	// Partition leader epoch (12-15)
+	batch = append(batch, 0xFF, 0xFF, 0xFF, 0xFF)
+
+	// Magic (16)
+	batch = append(batch, 0x02)
+
+	// CRC placeholder (17-20)
+	crcPos := len(batch)
+	batch = append(batch, 0, 0, 0, 0)
+
+	// Attributes (21-22)
+	batch = append(batch, 0, 0)
+
+	// Last offset delta (23-26)
+	batch = append(batch, 0, 0, 0, 0)
+
+	// Base timestamp (27-34)
+	timestampMs := timestamp.UnixMilli()
+	timestampBytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(timestampBytes, uint64(timestampMs))
+	batch = append(batch, timestampBytes...)
+
+	// Max timestamp (35-42)
+	batch = append(batch, timestampBytes...)
+
+	// Producer ID (43-50)
+	batch = append(batch, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF)
+
+	// Producer epoch (51-52)
+	batch = append(batch, 0xFF, 0xFF)
+
+	// Base sequence (53-56)
+	batch = append(batch, 0xFF, 0xFF, 0xFF, 0xFF)
+
+	// Record count (57-60)
+	recordCountBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(recordCountBytes, 1)
+	batch = append(batch, recordCountBytes...)
+
+	// Build record (61+)
+	recordBody := []byte{}
+
+	// Attributes
+	recordBody = append(recordBody, 0)
+
+	// Timestamp delta
+	recordBody = append(recordBody, encodeVarint(0)...)
+
+	// Offset delta
+	recordBody = append(recordBody, encodeVarint(0)...)
+
+	// Key length and key
+	if key == nil {
+		recordBody = append(recordBody, encodeVarint(-1)...)
+	} else {
+		recordBody = append(recordBody, encodeVarint(int64(len(key)))...)
+		recordBody = append(recordBody, key...)
+	}
+
+	// Value length and value
+	if value == nil {
+		recordBody = append(recordBody, encodeVarint(-1)...)
+	} else {
+		recordBody = append(recordBody, encodeVarint(int64(len(value)))...)
+		recordBody = append(recordBody, value...)
+	}
+
+	// Headers count
+	recordBody = append(recordBody, encodeVarint(0)...)
+
+	// Prepend record length
+	recordLength := int64(len(recordBody))
+	batch = append(batch, encodeVarint(recordLength)...)
+	batch = append(batch, recordBody...)
+
+	// Fill in batch length
+	batchLength := uint32(len(batch) - 12)
+	binary.BigEndian.PutUint32(batch[batchLengthPos:], batchLength)
+
+	// Calculate CRC
+	crcData := batch[21:]
+	crc := crc32.Checksum(crcData, crc32.MakeTable(crc32.Castagnoli))
+	binary.BigEndian.PutUint32(batch[crcPos:], crc)
+
+	return batch
+}
+
+// verifyField logs a field's value
+func verifyField(t *testing.T, name string, bytes []byte, value interface{}) {
+	t.Logf("  %s: %x (value: %v)", name, bytes, value)
+}
+
+// hexDump formats bytes as hex dump
+func hexDumpTest(data []byte) string {
+	var buf bytes.Buffer
+	for i := 0; i < len(data); i += 16 {
+		end := i + 16
+		if end > len(data) {
+			end = len(data)
+		}
+		buf.WriteString(fmt.Sprintf("  %04d: %x\n", i, data[i:end]))
+	}
+	return buf.String()
+}
+
+// TestClientSideCRCValidation mimics what a Kafka client does
+func TestClientSideCRCValidation(t *testing.T) {
+	// Build a batch
+	batch := constructTestBatch(0, time.Now(), []byte("test-key"), []byte("test-value"))
+
+	t.Logf("Constructed batch: %d bytes", len(batch))
+
+	// Now pretend we're a Kafka client receiving this batch
+	// Step 1: Read the batch header to get the CRC
+	if len(batch) < 21 {
+		t.Fatalf("Batch too short for client to read CRC")
+	}
+
+	clientReadCRC := binary.BigEndian.Uint32(batch[17:21])
+	t.Logf("Client read CRC from header: 0x%08x", clientReadCRC)
+
+	// Step 2: Calculate CRC over the data (from byte 21 onwards)
+	clientCalculatedCRC := crc32.Checksum(batch[21:], crc32.MakeTable(crc32.Castagnoli))
+	t.Logf("Client calculated CRC: 0x%08x", clientCalculatedCRC)
+
+	// Step 3: Compare
+	if clientReadCRC != clientCalculatedCRC {
+		t.Errorf("CLIENT WOULD REJECT: CRC mismatch: read=0x%08x calculated=0x%08x",
+			clientReadCRC, clientCalculatedCRC)
+		t.Log("This is the error consumers are seeing!")
+	} else {
+		t.Log("CLIENT WOULD ACCEPT: CRC valid")
+	}
+}
+
+// TestConcurrentBatchConstruction tests if there are race conditions
+func TestConcurrentBatchConstruction(t *testing.T) {
+	timestamp := time.Now()
+
+	// Build multiple batches concurrently
+	const numBatches = 10
+	results := make(chan bool, numBatches)
+
+	for i := 0; i < numBatches; i++ {
+		go func(id int) {
+			batch := constructTestBatch(int64(id), timestamp,
+				[]byte(fmt.Sprintf("key-%d", id)),
+				[]byte(fmt.Sprintf("value-%d", id)))
+
+			// Validate CRC
+			storedCRC := binary.BigEndian.Uint32(batch[17:21])
+			calculatedCRC := crc32.Checksum(batch[21:], crc32.MakeTable(crc32.Castagnoli))
+
+			results <- (storedCRC == calculatedCRC)
+		}(i)
+	}
+
+	// Check all results
+	allValid := true
+	for i := 0; i < numBatches; i++ {
+		if !<-results {
+			allValid = false
+			t.Errorf("Batch %d has invalid CRC", i)
+		}
+	}
+
+	if allValid {
+		t.Logf("All %d concurrent batches have valid CRCs", numBatches)
+	}
+}
+
+// TestProductionBatchConstruction tests the actual production code
+func TestProductionBatchConstruction(t *testing.T) {
+	// Create a mock SMQ record
+	mockRecord := &mockSMQRecord{
+		key:       []byte("prod-key"),
+		value:     []byte("prod-value"),
+		timestamp: time.Now().UnixNano(),
+	}
+
+	// Create a mock handler
+	mockHandler := &Handler{}
+
+	// Create fetcher
+	fetcher := NewMultiBatchFetcher(mockHandler)
+
+	// Construct batch using production code
+	batch := fetcher.constructSingleRecordBatch("test-topic", 0, []integration.SMQRecord{mockRecord})
+
+	t.Logf("Production batch size: %d bytes", len(batch))
+
+	// Validate CRC
+	if len(batch) < 21 {
+		t.Fatalf("Production batch too short: %d bytes", len(batch))
+	}
+
+	storedCRC := binary.BigEndian.Uint32(batch[17:21])
+	calculatedCRC := crc32.Checksum(batch[21:], crc32.MakeTable(crc32.Castagnoli))
+
+	t.Logf("Production batch CRC: stored=0x%08x calculated=0x%08x", storedCRC, calculatedCRC)
+
+	if storedCRC != calculatedCRC {
+		t.Errorf("PRODUCTION CODE CRC INVALID: stored=0x%08x calculated=0x%08x", storedCRC, calculatedCRC)
+		t.Log("This means the production constructSingleRecordBatch has a bug!")
+	} else {
+		t.Log("PRODUCTION CODE CRC VALID")
+	}
+}
+
+// mockSMQRecord implements the SMQRecord interface for testing
+type mockSMQRecord struct {
+	key       []byte
+	value     []byte
+	timestamp int64
+}
+
+func (m *mockSMQRecord) GetKey() []byte      { return m.key }
+func (m *mockSMQRecord) GetValue() []byte    { return m.value }
+func (m *mockSMQRecord) GetTimestamp() int64 { return m.timestamp }
+func (m *mockSMQRecord) GetOffset() int64    { return 0 }

weed/mq/kafka/protocol/consumer_coordination.go

@@ -0,0 +1,730 @@
+package protocol
+
+import (
+	"encoding/binary"
+	"fmt"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/consumer"
+)
+
+// Heartbeat API (key 12) - Consumer group heartbeat
+// Consumers send periodic heartbeats to stay in the group and receive rebalancing signals
+
+// HeartbeatRequest represents a Heartbeat request from a Kafka client
+type HeartbeatRequest struct {
+	GroupID         string
+	GenerationID    int32
+	MemberID        string
+	GroupInstanceID string // Optional static membership ID
+}
+
+// HeartbeatResponse represents a Heartbeat response to a Kafka client
+type HeartbeatResponse struct {
+	CorrelationID uint32
+	ErrorCode     int16
+}
+
+// LeaveGroup API (key 13) - Consumer graceful departure
+// Consumers call this when shutting down to trigger immediate rebalancing
+
+// LeaveGroupRequest represents a LeaveGroup request from a Kafka client
+type LeaveGroupRequest struct {
+	GroupID         string
+	MemberID        string
+	GroupInstanceID string             // Optional static membership ID
+	Members         []LeaveGroupMember // For newer versions, can leave multiple members
+}
+
+// LeaveGroupMember represents a member leaving the group (for batch departures)
+type LeaveGroupMember struct {
+	MemberID        string
+	GroupInstanceID string
+	Reason          string // Optional reason for leaving
+}
+
+// LeaveGroupResponse represents a LeaveGroup response to a Kafka client
+type LeaveGroupResponse struct {
+	CorrelationID uint32
+	ErrorCode     int16
+	Members       []LeaveGroupMemberResponse // Per-member responses for newer versions
+}
+
+// LeaveGroupMemberResponse represents per-member leave group response
+type LeaveGroupMemberResponse struct {
+	MemberID        string
+	GroupInstanceID string
+	ErrorCode       int16
+}
+
+// Error codes specific to consumer coordination are imported from errors.go
+
+func (h *Handler) handleHeartbeat(correlationID uint32, apiVersion uint16, requestBody []byte) ([]byte, error) {
+	// Parse Heartbeat request
+	request, err := h.parseHeartbeatRequest(requestBody, apiVersion)
+	if err != nil {
+		return h.buildHeartbeatErrorResponseV(correlationID, ErrorCodeInvalidGroupID, apiVersion), nil
+	}
+
+	// Validate request
+	if request.GroupID == "" || request.MemberID == "" {
+		return h.buildHeartbeatErrorResponseV(correlationID, ErrorCodeInvalidGroupID, apiVersion), nil
+	}
+
+	// Get consumer group
+	group := h.groupCoordinator.GetGroup(request.GroupID)
+	if group == nil {
+		return h.buildHeartbeatErrorResponseV(correlationID, ErrorCodeInvalidGroupID, apiVersion), nil
+	}
+
+	group.Mu.Lock()
+	defer group.Mu.Unlock()
+
+	// Update group's last activity
+	group.LastActivity = time.Now()
+
+	// Validate member exists
+	member, exists := group.Members[request.MemberID]
+	if !exists {
+		return h.buildHeartbeatErrorResponseV(correlationID, ErrorCodeUnknownMemberID, apiVersion), nil
+	}
+
+	// Validate generation. If the group is rebalancing we want Sarama to rejoin
+	// (via ErrRebalanceInProgress -> heartbeatLoop cancels session -> newSession)
+	// rather than tear down on ErrIllegalGeneration, which would exit the caller's
+	// Consume() entirely since Consume does not internally re-enter on that error.
+	if request.GenerationID != group.Generation {
+		switch group.State {
+		case consumer.GroupStatePreparingRebalance, consumer.GroupStateCompletingRebalance:
+			return h.buildHeartbeatErrorResponseV(correlationID, ErrorCodeRebalanceInProgress, apiVersion), nil
+		default:
+			return h.buildHeartbeatErrorResponseV(correlationID, ErrorCodeIllegalGeneration, apiVersion), nil
+		}
+	}
+
+	// Update member's last heartbeat
+	member.LastHeartbeat = time.Now()
+
+	// Check if rebalancing is in progress
+	var errorCode int16 = ErrorCodeNone
+	switch group.State {
+	case consumer.GroupStatePreparingRebalance, consumer.GroupStateCompletingRebalance:
+		// Signal the consumer that rebalancing is happening
+		errorCode = ErrorCodeRebalanceInProgress
+	case consumer.GroupStateDead:
+		errorCode = ErrorCodeInvalidGroupID
+	case consumer.GroupStateEmpty:
+		// This shouldn't happen if member exists, but handle gracefully
+		errorCode = ErrorCodeUnknownMemberID
+	case consumer.GroupStateStable:
+		// Normal case - heartbeat accepted
+		errorCode = ErrorCodeNone
+	}
+
+	// Build successful response
+	response := HeartbeatResponse{
+		CorrelationID: correlationID,
+		ErrorCode:     errorCode,
+	}
+
+	return h.buildHeartbeatResponseV(response, apiVersion), nil
+}
+
+func (h *Handler) handleLeaveGroup(correlationID uint32, apiVersion uint16, requestBody []byte) ([]byte, error) {
+	// Parse LeaveGroup request
+	request, err := h.parseLeaveGroupRequest(requestBody, apiVersion)
+	if err != nil {
+		return h.buildLeaveGroupErrorResponse(correlationID, ErrorCodeInvalidGroupID, apiVersion), nil
+	}
+
+	// Validate request - v3+ carries member IDs inside the Members array
+	if request.GroupID == "" {
+		return h.buildLeaveGroupErrorResponse(correlationID, ErrorCodeInvalidGroupID, apiVersion), nil
+	}
+	if request.MemberID == "" && len(request.Members) == 0 {
+		return h.buildLeaveGroupErrorResponse(correlationID, ErrorCodeInvalidGroupID, apiVersion), nil
+	}
+
+	// Get consumer group
+	group := h.groupCoordinator.GetGroup(request.GroupID)
+	if group == nil {
+		return h.buildLeaveGroupErrorResponse(correlationID, ErrorCodeInvalidGroupID, apiVersion), nil
+	}
+
+	group.Mu.Lock()
+	defer group.Mu.Unlock()
+
+	// Update group's last activity
+	group.LastActivity = time.Now()
+
+	// Normalize to a list of leaving members. v0-v2 carry a single MemberID at
+	// the top level; v3+ carry an array. parseLeaveGroupRequest already copies
+	// Members[0] to MemberID for back-compat, but the full list must be honored
+	// so batch departures (KIP-345) don't leave stale phantoms in the group.
+	leavingMembers := request.Members
+	if len(leavingMembers) == 0 {
+		leavingMembers = []LeaveGroupMember{{
+			MemberID:        request.MemberID,
+			GroupInstanceID: request.GroupInstanceID,
+		}}
+	}
+
+	memberResponses := make([]LeaveGroupMemberResponse, 0, len(leavingMembers))
+	removedAny := false
+	leaderLeft := false
+
+	for _, leaving := range leavingMembers {
+		member, exists := group.Members[leaving.MemberID]
+		if !exists {
+			memberResponses = append(memberResponses, LeaveGroupMemberResponse{
+				MemberID:        leaving.MemberID,
+				GroupInstanceID: leaving.GroupInstanceID,
+				ErrorCode:       ErrorCodeUnknownMemberID,
+			})
+			continue
+		}
+
+		if h.groupCoordinator.IsStaticMember(member) {
+			if leaving.GroupInstanceID != "" && *member.GroupInstanceID != leaving.GroupInstanceID {
+				memberResponses = append(memberResponses, LeaveGroupMemberResponse{
+					MemberID:        leaving.MemberID,
+					GroupInstanceID: leaving.GroupInstanceID,
+					ErrorCode:       ErrorCodeFencedInstanceID,
+				})
+				continue
+			}
+			h.groupCoordinator.UnregisterStaticMemberLocked(group, *member.GroupInstanceID)
+		}
+
+		if group.Leader == leaving.MemberID {
+			leaderLeft = true
+		}
+		delete(group.Members, leaving.MemberID)
+		removedAny = true
+		memberResponses = append(memberResponses, LeaveGroupMemberResponse{
+			MemberID:        leaving.MemberID,
+			GroupInstanceID: leaving.GroupInstanceID,
+			ErrorCode:       ErrorCodeNone,
+		})
+	}
+
+	if removedAny {
+		// One group-state transition applied after processing all departures.
+		if len(group.Members) == 0 {
+			group.State = consumer.GroupStateEmpty
+			group.Generation++
+			group.Leader = ""
+		} else {
+			group.State = consumer.GroupStatePreparingRebalance
+			group.Generation++
+			if leaderLeft {
+				for memberID := range group.Members {
+					group.Leader = memberID
+					break
+				}
+			}
+			for _, m := range group.Members {
+				m.State = consumer.MemberStatePending
+			}
+		}
+		h.updateGroupSubscriptionFromMembers(group)
+	}
+
+	response := LeaveGroupResponse{
+		CorrelationID: correlationID,
+		ErrorCode:     ErrorCodeNone,
+		Members:       memberResponses,
+	}
+	return h.buildLeaveGroupResponse(response, apiVersion), nil
+}
+
+func (h *Handler) parseHeartbeatRequest(data []byte, apiVersion uint16) (*HeartbeatRequest, error) {
+	if len(data) < 8 {
+		return nil, fmt.Errorf("request too short")
+	}
+
+	offset := 0
+	isFlexible := IsFlexibleVersion(12, apiVersion) // Heartbeat API key = 12
+
+	// ADMINCLIENT COMPATIBILITY FIX: Parse top-level tagged fields at the beginning for flexible versions
+	if isFlexible {
+		_, consumed, err := DecodeTaggedFields(data[offset:])
+		if err == nil {
+			offset += consumed
+		}
+	}
+
+	// Parse GroupID
+	var groupID string
+	if isFlexible {
+		// FLEXIBLE V4+ FIX: GroupID is a compact string
+		groupIDBytes, consumed := parseCompactString(data[offset:])
+		if consumed == 0 {
+			return nil, fmt.Errorf("invalid group ID compact string")
+		}
+		if groupIDBytes != nil {
+			groupID = string(groupIDBytes)
+		}
+		offset += consumed
+	} else {
+		// Non-flexible parsing (v0-v3)
+		groupIDLength := int(binary.BigEndian.Uint16(data[offset:]))
+		offset += 2
+		if offset+groupIDLength > len(data) {
+			return nil, fmt.Errorf("invalid group ID length")
+		}
+		groupID = string(data[offset : offset+groupIDLength])
+		offset += groupIDLength
+	}
+
+	// Generation ID (4 bytes) - always fixed-length
+	if offset+4 > len(data) {
+		return nil, fmt.Errorf("missing generation ID")
+	}
+	generationID := int32(binary.BigEndian.Uint32(data[offset:]))
+	offset += 4
+
+	// Parse MemberID
+	var memberID string
+	if isFlexible {
+		// FLEXIBLE V4+ FIX: MemberID is a compact string
+		memberIDBytes, consumed := parseCompactString(data[offset:])
+		if consumed == 0 {
+			return nil, fmt.Errorf("invalid member ID compact string")
+		}
+		if memberIDBytes != nil {
+			memberID = string(memberIDBytes)
+		}
+		offset += consumed
+	} else {
+		// Non-flexible parsing (v0-v3)
+		if offset+2 > len(data) {
+			return nil, fmt.Errorf("missing member ID length")
+		}
+		memberIDLength := int(binary.BigEndian.Uint16(data[offset:]))
+		offset += 2
+		if offset+memberIDLength > len(data) {
+			return nil, fmt.Errorf("invalid member ID length")
+		}
+		memberID = string(data[offset : offset+memberIDLength])
+		offset += memberIDLength
+	}
+
+	// Parse GroupInstanceID (nullable string) - for Heartbeat v1+
+	var groupInstanceID string
+	if apiVersion >= 1 {
+		if isFlexible {
+			// FLEXIBLE V4+ FIX: GroupInstanceID is a compact nullable string
+			groupInstanceIDBytes, consumed := parseCompactString(data[offset:])
+			if consumed == 0 && len(data) > offset && data[offset] == 0x00 {
+				groupInstanceID = "" // null
+				offset += 1
+			} else {
+				if groupInstanceIDBytes != nil {
+					groupInstanceID = string(groupInstanceIDBytes)
+				}
+				offset += consumed
+			}
+		} else {
+			// Non-flexible v1-v3: regular nullable string
+			if offset+2 <= len(data) {
+				instanceIDLength := int16(binary.BigEndian.Uint16(data[offset:]))
+				offset += 2
+				if instanceIDLength == -1 {
+					groupInstanceID = "" // null string
+				} else if instanceIDLength >= 0 && offset+int(instanceIDLength) <= len(data) {
+					groupInstanceID = string(data[offset : offset+int(instanceIDLength)])
+					offset += int(instanceIDLength)
+				}
+			}
+		}
+	}
+
+	// Parse request-level tagged fields (v4+)
+	if isFlexible {
+		if offset < len(data) {
+			_, consumed, err := DecodeTaggedFields(data[offset:])
+			if err == nil {
+				offset += consumed
+			}
+		}
+	}
+
+	return &HeartbeatRequest{
+		GroupID:         groupID,
+		GenerationID:    generationID,
+		MemberID:        memberID,
+		GroupInstanceID: groupInstanceID,
+	}, nil
+}
+
+func (h *Handler) parseLeaveGroupRequest(data []byte, apiVersion uint16) (*LeaveGroupRequest, error) {
+	if len(data) < 4 {
+		return nil, fmt.Errorf("request too short")
+	}
+
+	offset := 0
+	isFlexible := IsFlexibleVersion(uint16(APIKeyLeaveGroup), apiVersion)
+
+	if isFlexible {
+		// Skip top-level tagged fields
+		_, consumed, err := DecodeTaggedFields(data[offset:])
+		if err != nil {
+			return nil, fmt.Errorf("LeaveGroup v%d: top-level tagged fields: %w", apiVersion, err)
+		}
+		offset += consumed
+	}
+
+	// GroupID
+	var groupID string
+	if isFlexible {
+		bytes, consumed := parseCompactString(data[offset:])
+		if consumed == 0 {
+			return nil, fmt.Errorf("LeaveGroup v%d: invalid group ID compact string", apiVersion)
+		}
+		if bytes != nil {
+			groupID = string(bytes)
+		}
+		offset += consumed
+	} else {
+		if offset+2 > len(data) {
+			return nil, fmt.Errorf("missing group ID length")
+		}
+		groupIDLength := int(binary.BigEndian.Uint16(data[offset:]))
+		offset += 2
+		if offset+groupIDLength > len(data) {
+			return nil, fmt.Errorf("invalid group ID length")
+		}
+		groupID = string(data[offset : offset+groupIDLength])
+		offset += groupIDLength
+	}
+
+	req := &LeaveGroupRequest{
+		GroupID: groupID,
+		Members: []LeaveGroupMember{},
+	}
+
+	// v0-v2: top-level MemberID (string). v3+: Members array.
+	if apiVersion <= 2 {
+		if offset+2 > len(data) {
+			return nil, fmt.Errorf("missing member ID length")
+		}
+		memberIDLength := int(binary.BigEndian.Uint16(data[offset:]))
+		offset += 2
+		if offset+memberIDLength > len(data) {
+			return nil, fmt.Errorf("invalid member ID length")
+		}
+		req.MemberID = string(data[offset : offset+memberIDLength])
+		offset += memberIDLength
+		return req, nil
+	}
+
+	// v3+: Members array [{MemberID, GroupInstanceID[, Reason]}]
+	var membersCount int
+	if isFlexible {
+		compactLen, consumed, err := DecodeCompactArrayLength(data[offset:])
+		if err != nil {
+			return nil, fmt.Errorf("LeaveGroup v%d: invalid members compact array: %w", apiVersion, err)
+		}
+		membersCount = int(compactLen)
+		offset += consumed
+	} else {
+		if offset+4 > len(data) {
+			return nil, fmt.Errorf("missing members array length")
+		}
+		membersCount = int(int32(binary.BigEndian.Uint32(data[offset:])))
+		offset += 4
+	}
+
+	if membersCount < 0 || membersCount > 100000 {
+		return nil, fmt.Errorf("unreasonable members count: %d", membersCount)
+	}
+
+	for i := 0; i < membersCount; i++ {
+		var m LeaveGroupMember
+		// MemberID
+		if isFlexible {
+			bytes, consumed := parseCompactString(data[offset:])
+			if consumed == 0 {
+				return nil, fmt.Errorf("LeaveGroup v%d: invalid members[%d].member_id", apiVersion, i)
+			}
+			if bytes != nil {
+				m.MemberID = string(bytes)
+			}
+			offset += consumed
+		} else {
+			if offset+2 > len(data) {
+				return nil, fmt.Errorf("members[%d]: missing member_id length", i)
+			}
+			ml := int(binary.BigEndian.Uint16(data[offset:]))
+			offset += 2
+			if offset+ml > len(data) {
+				return nil, fmt.Errorf("members[%d]: invalid member_id length", i)
+			}
+			m.MemberID = string(data[offset : offset+ml])
+			offset += ml
+		}
+		// GroupInstanceID (nullable)
+		if isFlexible {
+			bytes, consumed := parseCompactString(data[offset:])
+			if consumed == 0 {
+				return nil, fmt.Errorf("LeaveGroup v%d: invalid members[%d].group_instance_id", apiVersion, i)
+			}
+			if bytes != nil {
+				m.GroupInstanceID = string(bytes)
+			}
+			offset += consumed
+		} else {
+			if offset+2 > len(data) {
+				return nil, fmt.Errorf("members[%d]: missing group_instance_id length", i)
+			}
+			gil := int16(binary.BigEndian.Uint16(data[offset:]))
+			offset += 2
+			if gil >= 0 {
+				if offset+int(gil) > len(data) {
+					return nil, fmt.Errorf("members[%d]: invalid group_instance_id length", i)
+				}
+				m.GroupInstanceID = string(data[offset : offset+int(gil)])
+				offset += int(gil)
+			}
+		}
+		// v5+: Reason (compact nullable string, flexible only)
+		if apiVersion >= 5 && isFlexible {
+			bytes, consumed := parseCompactString(data[offset:])
+			if consumed > 0 {
+				if bytes != nil {
+					m.Reason = string(bytes)
+				}
+				offset += consumed
+			}
+		}
+		// Per-member tagged fields
+		if isFlexible {
+			_, consumed, err := DecodeTaggedFields(data[offset:])
+			if err != nil {
+				return nil, fmt.Errorf("LeaveGroup v%d: members[%d] tagged fields: %w", apiVersion, i, err)
+			}
+			offset += consumed
+		}
+		req.Members = append(req.Members, m)
+	}
+
+	// Back-compat convenience: expose the first member as top-level MemberID so
+	// handler code that pre-dates Members support keeps working.
+	if req.MemberID == "" && len(req.Members) > 0 {
+		req.MemberID = req.Members[0].MemberID
+		req.GroupInstanceID = req.Members[0].GroupInstanceID
+	}
+
+	return req, nil
+}
+
+func (h *Handler) buildHeartbeatResponseV(response HeartbeatResponse, apiVersion uint16) []byte {
+	isFlexible := IsFlexibleVersion(12, apiVersion) // Heartbeat API key = 12
+	result := make([]byte, 0, 16)
+
+	// NOTE: Correlation ID is handled by writeResponseWithCorrelationID
+	// Do NOT include it in the response body
+
+	if isFlexible {
+		// FLEXIBLE V4+ FORMAT
+		// NOTE: Response header tagged fields are handled by writeResponseWithHeader
+		// Do NOT include them in the response body
+
+		// Throttle time (4 bytes, 0 = no throttling) - comes first in flexible format
+		result = append(result, 0, 0, 0, 0)
+
+		// Error code (2 bytes)
+		errorCodeBytes := make([]byte, 2)
+		binary.BigEndian.PutUint16(errorCodeBytes, uint16(response.ErrorCode))
+		result = append(result, errorCodeBytes...)
+
+		// Response body tagged fields (varint: 0x00 = empty)
+		result = append(result, 0x00)
+	} else if apiVersion >= 1 {
+		// NON-FLEXIBLE V1-V3 FORMAT: throttle_time_ms BEFORE error_code
+		// CRITICAL FIX: Kafka protocol specifies throttle_time_ms comes FIRST in v1+
+
+		// Throttle time (4 bytes, 0 = no throttling) - comes first in v1-v3
+		result = append(result, 0, 0, 0, 0)
+
+		// Error code (2 bytes)
+		errorCodeBytes := make([]byte, 2)
+		binary.BigEndian.PutUint16(errorCodeBytes, uint16(response.ErrorCode))
+		result = append(result, errorCodeBytes...)
+	} else {
+		// V0 FORMAT: Only error_code, NO throttle_time_ms
+
+		// Error code (2 bytes)
+		errorCodeBytes := make([]byte, 2)
+		binary.BigEndian.PutUint16(errorCodeBytes, uint16(response.ErrorCode))
+		result = append(result, errorCodeBytes...)
+	}
+
+	return result
+}
+
+func (h *Handler) buildLeaveGroupResponse(response LeaveGroupResponse, apiVersion uint16) []byte {
+	// LeaveGroup v0 only includes correlation_id and error_code (no throttle_time_ms, no members)
+	if apiVersion == 0 {
+		return h.buildLeaveGroupV0Response(response)
+	}
+
+	// v4+ uses flexible encoding (compact strings/arrays + tagged fields).
+	if IsFlexibleVersion(uint16(APIKeyLeaveGroup), apiVersion) {
+		return h.buildLeaveGroupFlexibleResponse(response)
+	}
+
+	// v1-v3 use the full non-flexible response format.
+	return h.buildLeaveGroupFullResponse(response)
+}
+
+// buildLeaveGroupFlexibleResponse encodes a LeaveGroup response for v4+ clients,
+// which negotiate flexible versions. Fields:
+//
+//	throttle_time_ms (int32)
+//	error_code       (int16)
+//	members          (compact_array of {member_id: compact_string,
+//	                                    group_instance_id: compact_nullable_string,
+//	                                    error_code: int16, _tagged_fields})
+//	_tagged_fields
+func (h *Handler) buildLeaveGroupFlexibleResponse(response LeaveGroupResponse) []byte {
+	estimatedSize := 16
+	for _, m := range response.Members {
+		estimatedSize += len(m.MemberID) + len(m.GroupInstanceID) + 10
+	}
+	result := make([]byte, 0, estimatedSize)
+
+	// throttle_time_ms
+	result = append(result, 0, 0, 0, 0)
+
+	// error_code
+	errorCodeBytes := make([]byte, 2)
+	binary.BigEndian.PutUint16(errorCodeBytes, uint16(response.ErrorCode))
+	result = append(result, errorCodeBytes...)
+
+	// members: compact array
+	result = append(result, CompactArrayLength(uint32(len(response.Members)))...)
+	for _, m := range response.Members {
+		// member_id: compact string
+		result = append(result, CompactStringLength(len(m.MemberID))...)
+		result = append(result, []byte(m.MemberID)...)
+
+		// group_instance_id: compact nullable string
+		if m.GroupInstanceID == "" {
+			result = append(result, 0) // null
+		} else {
+			result = append(result, CompactStringLength(len(m.GroupInstanceID))...)
+			result = append(result, []byte(m.GroupInstanceID)...)
+		}
+
+		// member error_code
+		memberErrorBytes := make([]byte, 2)
+		binary.BigEndian.PutUint16(memberErrorBytes, uint16(m.ErrorCode))
+		result = append(result, memberErrorBytes...)
+
+		// per-member _tagged_fields (empty)
+		result = append(result, 0)
+	}
+
+	// response-level _tagged_fields (empty)
+	result = append(result, 0)
+
+	return result
+}
+
+func (h *Handler) buildLeaveGroupV0Response(response LeaveGroupResponse) []byte {
+	result := make([]byte, 0, 6)
+
+	// NOTE: Correlation ID is handled by writeResponseWithCorrelationID
+	// Do NOT include it in the response body
+
+	// Error code (2 bytes) - that's it for v0!
+	errorCodeBytes := make([]byte, 2)
+	binary.BigEndian.PutUint16(errorCodeBytes, uint16(response.ErrorCode))
+	result = append(result, errorCodeBytes...)
+
+	return result
+}
+
+func (h *Handler) buildLeaveGroupFullResponse(response LeaveGroupResponse) []byte {
+	estimatedSize := 16
+	for _, member := range response.Members {
+		estimatedSize += len(member.MemberID) + len(member.GroupInstanceID) + 8
+	}
+
+	result := make([]byte, 0, estimatedSize)
+
+	// NOTE: Correlation ID is handled by writeResponseWithCorrelationID
+	// Do NOT include it in the response body
+
+	// For LeaveGroup v1+, throttle_time_ms comes first (4 bytes)
+	result = append(result, 0, 0, 0, 0)
+
+	// Error code (2 bytes)
+	errorCodeBytes := make([]byte, 2)
+	binary.BigEndian.PutUint16(errorCodeBytes, uint16(response.ErrorCode))
+	result = append(result, errorCodeBytes...)
+
+	// Members array length (4 bytes)
+	membersLengthBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(membersLengthBytes, uint32(len(response.Members)))
+	result = append(result, membersLengthBytes...)
+
+	// Members
+	for _, member := range response.Members {
+		// Member ID length (2 bytes)
+		memberIDLength := make([]byte, 2)
+		binary.BigEndian.PutUint16(memberIDLength, uint16(len(member.MemberID)))
+		result = append(result, memberIDLength...)
+
+		// Member ID
+		result = append(result, []byte(member.MemberID)...)
+
+		// Group instance ID length (2 bytes)
+		instanceIDLength := make([]byte, 2)
+		binary.BigEndian.PutUint16(instanceIDLength, uint16(len(member.GroupInstanceID)))
+		result = append(result, instanceIDLength...)
+
+		// Group instance ID
+		if len(member.GroupInstanceID) > 0 {
+			result = append(result, []byte(member.GroupInstanceID)...)
+		}
+
+		// Error code (2 bytes)
+		memberErrorBytes := make([]byte, 2)
+		binary.BigEndian.PutUint16(memberErrorBytes, uint16(member.ErrorCode))
+		result = append(result, memberErrorBytes...)
+	}
+
+	return result
+}
+
+func (h *Handler) buildHeartbeatErrorResponseV(correlationID uint32, errorCode int16, apiVersion uint16) []byte {
+	response := HeartbeatResponse{
+		CorrelationID: correlationID,
+		ErrorCode:     errorCode,
+	}
+
+	return h.buildHeartbeatResponseV(response, apiVersion)
+}
+
+func (h *Handler) buildLeaveGroupErrorResponse(correlationID uint32, errorCode int16, apiVersion uint16) []byte {
+	response := LeaveGroupResponse{
+		CorrelationID: correlationID,
+		ErrorCode:     errorCode,
+		Members:       []LeaveGroupMemberResponse{},
+	}
+
+	return h.buildLeaveGroupResponse(response, apiVersion)
+}
+
+func (h *Handler) updateGroupSubscriptionFromMembers(group *consumer.ConsumerGroup) {
+	// Update group's subscribed topics from remaining members
+	group.SubscribedTopics = make(map[string]bool)
+	for _, member := range group.Members {
+		for _, topic := range member.Subscription {
+			group.SubscribedTopics[topic] = true
+		}
+	}
+}

weed/mq/kafka/protocol/consumer_group_metadata.go

@@ -0,0 +1,278 @@
+package protocol
+
+import (
+	"encoding/binary"
+	"fmt"
+	"net"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/consumer"
+)
+
+// ConsumerProtocolMetadata represents parsed consumer protocol metadata
+type ConsumerProtocolMetadata struct {
+	Version            int16    // Protocol metadata version
+	Topics             []string // Subscribed topic names
+	UserData           []byte   // Optional user data
+	AssignmentStrategy string   // Preferred assignment strategy
+}
+
+// ConnectionContext holds connection-specific information for requests
+type ConnectionContext struct {
+	RemoteAddr    net.Addr // Client's remote address
+	LocalAddr     net.Addr // Server's local address
+	ConnectionID  string   // Connection identifier
+	ClientID      string   // Kafka client ID from request headers
+	ConsumerGroup string   // Consumer group (set by JoinGroup)
+	MemberID      string   // Consumer group member ID (set by JoinGroup)
+	// Per-connection broker client for isolated gRPC streams
+	// Each Kafka connection MUST have its own gRPC streams to avoid interference
+	// when multiple consumers or requests are active on different connections
+	BrokerClient interface{} // Will be set to *integration.BrokerClient
+
+	// Persistent partition readers - one goroutine per topic-partition that maintains position
+	// and streams forward, eliminating repeated offset lookups and reducing broker CPU load
+	partitionReaders sync.Map // map[TopicPartitionKey]*partitionReader
+}
+
+// ExtractClientHost extracts the client hostname/IP from connection context
+func ExtractClientHost(connCtx *ConnectionContext) string {
+	if connCtx == nil || connCtx.RemoteAddr == nil {
+		return "unknown"
+	}
+
+	// Extract host portion from address
+	if tcpAddr, ok := connCtx.RemoteAddr.(*net.TCPAddr); ok {
+		return tcpAddr.IP.String()
+	}
+
+	// Fallback: parse string representation
+	addrStr := connCtx.RemoteAddr.String()
+	if host, _, err := net.SplitHostPort(addrStr); err == nil {
+		return host
+	}
+
+	// Last resort: return full address
+	return addrStr
+}
+
+// ParseConsumerProtocolMetadata parses consumer protocol metadata with enhanced error handling
+func ParseConsumerProtocolMetadata(metadata []byte, strategyName string) (*ConsumerProtocolMetadata, error) {
+	if len(metadata) < 2 {
+		return &ConsumerProtocolMetadata{
+			Version:            0,
+			Topics:             []string{},
+			UserData:           []byte{},
+			AssignmentStrategy: strategyName,
+		}, nil
+	}
+
+	result := &ConsumerProtocolMetadata{
+		AssignmentStrategy: strategyName,
+	}
+
+	offset := 0
+
+	// Parse version (2 bytes)
+	if len(metadata) < offset+2 {
+		return nil, fmt.Errorf("metadata too short for version field")
+	}
+	result.Version = int16(binary.BigEndian.Uint16(metadata[offset : offset+2]))
+	offset += 2
+
+	// Parse topics array
+	if len(metadata) < offset+4 {
+		return nil, fmt.Errorf("metadata too short for topics count")
+	}
+	topicsCount := binary.BigEndian.Uint32(metadata[offset : offset+4])
+	offset += 4
+
+	// Validate topics count (reasonable limit)
+	if topicsCount > 10000 {
+		return nil, fmt.Errorf("unreasonable topics count: %d", topicsCount)
+	}
+
+	result.Topics = make([]string, 0, topicsCount)
+
+	for i := uint32(0); i < topicsCount && offset < len(metadata); i++ {
+		// Parse topic name length
+		if len(metadata) < offset+2 {
+			return nil, fmt.Errorf("metadata too short for topic %d name length", i)
+		}
+		topicNameLength := binary.BigEndian.Uint16(metadata[offset : offset+2])
+		offset += 2
+
+		// Validate topic name length
+		if topicNameLength > 1000 {
+			return nil, fmt.Errorf("unreasonable topic name length: %d", topicNameLength)
+		}
+
+		if len(metadata) < offset+int(topicNameLength) {
+			return nil, fmt.Errorf("metadata too short for topic %d name data", i)
+		}
+
+		topicName := string(metadata[offset : offset+int(topicNameLength)])
+		offset += int(topicNameLength)
+
+		// Validate topic name (basic validation)
+		if len(topicName) == 0 {
+			continue // Skip empty topic names
+		}
+
+		result.Topics = append(result.Topics, topicName)
+	}
+
+	// Parse user data if remaining bytes exist
+	if len(metadata) >= offset+4 {
+		userDataLength := binary.BigEndian.Uint32(metadata[offset : offset+4])
+		offset += 4
+
+		// Handle -1 (0xFFFFFFFF) as null/empty user data (Kafka protocol convention)
+		if userDataLength == 0xFFFFFFFF {
+			result.UserData = []byte{}
+			return result, nil
+		}
+
+		// Validate user data length
+		if userDataLength > 100000 { // 100KB limit
+			return nil, fmt.Errorf("unreasonable user data length: %d", userDataLength)
+		}
+
+		if len(metadata) >= offset+int(userDataLength) {
+			result.UserData = make([]byte, userDataLength)
+			copy(result.UserData, metadata[offset:offset+int(userDataLength)])
+		}
+	}
+
+	return result, nil
+}
+
+// ValidateAssignmentStrategy checks if an assignment strategy is supported
+func ValidateAssignmentStrategy(strategy string) bool {
+	supportedStrategies := map[string]bool{
+		consumer.ProtocolNameRange:             true,
+		consumer.ProtocolNameRoundRobin:        true,
+		consumer.ProtocolNameSticky:            true,
+		consumer.ProtocolNameCooperativeSticky: true, // Incremental cooperative rebalancing (Kafka 2.4+)
+	}
+
+	return supportedStrategies[strategy]
+}
+
+// ExtractTopicsFromMetadata extracts topic list from protocol metadata with fallback
+func ExtractTopicsFromMetadata(protocols []GroupProtocol, fallbackTopics []string) []string {
+	for _, protocol := range protocols {
+		if ValidateAssignmentStrategy(protocol.Name) {
+			parsed, err := ParseConsumerProtocolMetadata(protocol.Metadata, protocol.Name)
+			if err != nil {
+				continue
+			}
+
+			if len(parsed.Topics) > 0 {
+				return parsed.Topics
+			}
+		}
+	}
+
+	// Fallback to provided topics or empty list
+	if len(fallbackTopics) > 0 {
+		return fallbackTopics
+	}
+
+	// Return empty slice if no topics found - consumer may be using pattern subscription
+	return []string{}
+}
+
+// SelectBestProtocol chooses the best assignment protocol from available options
+func SelectBestProtocol(protocols []GroupProtocol, groupProtocols []string) string {
+	// Priority order: sticky > roundrobin > range
+	protocolPriority := []string{consumer.ProtocolNameSticky, consumer.ProtocolNameRoundRobin, consumer.ProtocolNameRange}
+
+	// Find supported protocols in client's list
+	clientProtocols := make(map[string]bool)
+	for _, protocol := range protocols {
+		if ValidateAssignmentStrategy(protocol.Name) {
+			clientProtocols[protocol.Name] = true
+		}
+	}
+
+	// Find supported protocols in group's list
+	groupProtocolSet := make(map[string]bool)
+	for _, protocol := range groupProtocols {
+		groupProtocolSet[protocol] = true
+	}
+
+	// Select highest priority protocol that both client and group support
+	for _, preferred := range protocolPriority {
+		if clientProtocols[preferred] && (len(groupProtocols) == 0 || groupProtocolSet[preferred]) {
+			return preferred
+		}
+	}
+
+	// If group has existing protocols, find a protocol supported by both client and group
+	if len(groupProtocols) > 0 {
+		// Try to find a protocol that both client and group support
+		for _, preferred := range protocolPriority {
+			if clientProtocols[preferred] && groupProtocolSet[preferred] {
+				return preferred
+			}
+		}
+
+		// No common protocol found - handle special fallback case
+		// If client supports nothing we validate, but group supports "range", use "range"
+		if len(clientProtocols) == 0 && groupProtocolSet[consumer.ProtocolNameRange] {
+			return consumer.ProtocolNameRange
+		}
+
+		// Return empty string to indicate no compatible protocol found
+		return ""
+	}
+
+	// Fallback to first supported protocol from client (only when group has no existing protocols)
+	for _, protocol := range protocols {
+		if ValidateAssignmentStrategy(protocol.Name) {
+			return protocol.Name
+		}
+	}
+
+	// Last resort
+	return consumer.ProtocolNameRange
+}
+
+// ProtocolMetadataDebugInfo returns debug information about protocol metadata
+type ProtocolMetadataDebugInfo struct {
+	Strategy     string
+	Version      int16
+	TopicCount   int
+	Topics       []string
+	UserDataSize int
+	ParsedOK     bool
+	ParseError   string
+}
+
+// AnalyzeProtocolMetadata provides detailed debug information about protocol metadata
+func AnalyzeProtocolMetadata(protocols []GroupProtocol) []ProtocolMetadataDebugInfo {
+	result := make([]ProtocolMetadataDebugInfo, 0, len(protocols))
+
+	for _, protocol := range protocols {
+		info := ProtocolMetadataDebugInfo{
+			Strategy: protocol.Name,
+		}
+
+		parsed, err := ParseConsumerProtocolMetadata(protocol.Metadata, protocol.Name)
+		if err != nil {
+			info.ParsedOK = false
+			info.ParseError = err.Error()
+		} else {
+			info.ParsedOK = true
+			info.Version = parsed.Version
+			info.TopicCount = len(parsed.Topics)
+			info.Topics = parsed.Topics
+			info.UserDataSize = len(parsed.UserData)
+		}
+
+		result = append(result, info)
+	}
+
+	return result
+}

weed/mq/kafka/protocol/debug_state.go

@@ -0,0 +1,199 @@
+package protocol
+
+import (
+	"sort"
+	"time"
+)
+
+type DebugGroupSnapshot struct {
+	GroupID          string                      `json:"group_id"`
+	Exists           bool                        `json:"exists"`
+	KnownGroups      []string                    `json:"known_groups,omitempty"`
+	State            string                      `json:"state,omitempty"`
+	Generation       int32                       `json:"generation,omitempty"`
+	Protocol         string                      `json:"protocol,omitempty"`
+	Leader           string                      `json:"leader,omitempty"`
+	CreatedAt        time.Time                   `json:"created_at,omitempty"`
+	LastActivity     time.Time                   `json:"last_activity,omitempty"`
+	SubscribedTopics []string                    `json:"subscribed_topics,omitempty"`
+	Members          []DebugGroupMemberSnapshot  `json:"members,omitempty"`
+	OffsetCommits    []DebugOffsetCommitSnapshot `json:"offset_commits,omitempty"`
+}
+
+type DebugGroupMemberSnapshot struct {
+	MemberID           string    `json:"member_id"`
+	ClientID           string    `json:"client_id,omitempty"`
+	ConnectionID       string    `json:"connection_id,omitempty"`
+	ClientHost         string    `json:"client_host,omitempty"`
+	State              string    `json:"state,omitempty"`
+	SessionTimeoutMs   int32     `json:"session_timeout_ms,omitempty"`
+	RebalanceTimeoutMs int32     `json:"rebalance_timeout_ms,omitempty"`
+	JoinedAt           time.Time `json:"joined_at,omitempty"`
+	LastHeartbeat      time.Time `json:"last_heartbeat,omitempty"`
+	LastHeartbeatAgeMs int64     `json:"last_heartbeat_age_ms,omitempty"`
+	Subscription       []string  `json:"subscription,omitempty"`
+	AssignedPartitions []string  `json:"assigned_partitions,omitempty"`
+	GroupInstanceID    string    `json:"group_instance_id,omitempty"`
+}
+
+type DebugOffsetCommitSnapshot struct {
+	Topic       string    `json:"topic"`
+	Partition   int32     `json:"partition"`
+	Offset      int64     `json:"offset"`
+	Metadata    string    `json:"metadata,omitempty"`
+	Timestamp   time.Time `json:"timestamp,omitempty"`
+	CommitAgeMs int64     `json:"commit_age_ms,omitempty"`
+}
+
+func (h *Handler) DebugGroupSnapshot(groupID string) *DebugGroupSnapshot {
+	snapshot := &DebugGroupSnapshot{
+		GroupID:     groupID,
+		KnownGroups: h.debugKnownGroups(),
+	}
+	if h == nil || h.groupCoordinator == nil {
+		return snapshot
+	}
+
+	group := h.groupCoordinator.GetGroup(groupID)
+	if group == nil {
+		return snapshot
+	}
+
+	now := time.Now()
+
+	group.Mu.RLock()
+	defer group.Mu.RUnlock()
+
+	snapshot.Exists = true
+	snapshot.State = group.State.String()
+	snapshot.Generation = group.Generation
+	snapshot.Protocol = group.Protocol
+	snapshot.Leader = group.Leader
+	snapshot.CreatedAt = group.CreatedAt
+	snapshot.LastActivity = group.LastActivity
+
+	if len(group.SubscribedTopics) > 0 {
+		topics := make([]string, 0, len(group.SubscribedTopics))
+		for topic := range group.SubscribedTopics {
+			topics = append(topics, topic)
+		}
+		sort.Strings(topics)
+		snapshot.SubscribedTopics = topics
+	}
+
+	memberIDs := make([]string, 0, len(group.Members))
+	for memberID := range group.Members {
+		memberIDs = append(memberIDs, memberID)
+	}
+	sort.Strings(memberIDs)
+
+	snapshot.Members = make([]DebugGroupMemberSnapshot, 0, len(memberIDs))
+	for _, memberID := range memberIDs {
+		member := group.Members[memberID]
+		if member == nil {
+			continue
+		}
+
+		subscription := append([]string(nil), member.Subscription...)
+		sort.Strings(subscription)
+
+		assignments := make([]string, 0, len(member.Assignment))
+		for _, assignment := range member.Assignment {
+			assignments = append(assignments, assignment.Topic+":"+itoa32(assignment.Partition))
+		}
+		sort.Strings(assignments)
+
+		groupInstanceID := ""
+		if member.GroupInstanceID != nil {
+			groupInstanceID = *member.GroupInstanceID
+		}
+
+		snapshot.Members = append(snapshot.Members, DebugGroupMemberSnapshot{
+			MemberID:           member.ID,
+			ClientID:           member.ClientID,
+			ConnectionID:       member.ConnectionID,
+			ClientHost:         member.ClientHost,
+			State:              member.State.String(),
+			SessionTimeoutMs:   member.SessionTimeout,
+			RebalanceTimeoutMs: member.RebalanceTimeout,
+			JoinedAt:           member.JoinedAt,
+			LastHeartbeat:      member.LastHeartbeat,
+			LastHeartbeatAgeMs: ageMillis(now, member.LastHeartbeat),
+			Subscription:       subscription,
+			AssignedPartitions: assignments,
+			GroupInstanceID:    groupInstanceID,
+		})
+	}
+
+	if len(group.OffsetCommits) > 0 {
+		topics := make([]string, 0, len(group.OffsetCommits))
+		for topic := range group.OffsetCommits {
+			topics = append(topics, topic)
+		}
+		sort.Strings(topics)
+
+		offsets := make([]DebugOffsetCommitSnapshot, 0)
+		for _, topic := range topics {
+			partitions := make([]int32, 0, len(group.OffsetCommits[topic]))
+			for partition := range group.OffsetCommits[topic] {
+				partitions = append(partitions, partition)
+			}
+			sort.Slice(partitions, func(i, j int) bool { return partitions[i] < partitions[j] })
+			for _, partition := range partitions {
+				offsetCommit := group.OffsetCommits[topic][partition]
+				offsets = append(offsets, DebugOffsetCommitSnapshot{
+					Topic:       topic,
+					Partition:   partition,
+					Offset:      offsetCommit.Offset,
+					Metadata:    offsetCommit.Metadata,
+					Timestamp:   offsetCommit.Timestamp,
+					CommitAgeMs: ageMillis(now, offsetCommit.Timestamp),
+				})
+			}
+		}
+		snapshot.OffsetCommits = offsets
+	}
+
+	return snapshot
+}
+
+func (h *Handler) debugKnownGroups() []string {
+	if h == nil || h.groupCoordinator == nil {
+		return nil
+	}
+
+	groupIDs := h.groupCoordinator.ListGroups()
+	sort.Strings(groupIDs)
+	return groupIDs
+}
+
+func ageMillis(now, then time.Time) int64 {
+	if then.IsZero() {
+		return 0
+	}
+	return now.Sub(then).Milliseconds()
+}
+
+func itoa32(value int32) string {
+	if value == 0 {
+		return "0"
+	}
+
+	negative := value < 0
+	if negative {
+		value = -value
+	}
+
+	var digits [12]byte
+	i := len(digits)
+	for value > 0 {
+		i--
+		digits[i] = byte('0' + (value % 10))
+		value /= 10
+	}
+	if negative {
+		i--
+		digits[i] = '-'
+	}
+	return string(digits[i:])
+}

weed/mq/kafka/protocol/describe_cluster.go

@@ -0,0 +1,112 @@
+package protocol
+
+import (
+	"encoding/binary"
+	"fmt"
+)
+
+// handleDescribeCluster implements the DescribeCluster API (key 60, versions 0-1)
+// This API is used by Java AdminClient for broker discovery (KIP-919)
+// Response format (flexible, all versions):
+//
+//	ThrottleTimeMs(int32) + ErrorCode(int16) + ErrorMessage(compact nullable string) +
+//	[v1+: EndpointType(int8)] + ClusterId(compact string) + ControllerId(int32) +
+//	Brokers(compact array) + ClusterAuthorizedOperations(int32) + TaggedFields
+func (h *Handler) handleDescribeCluster(correlationID uint32, apiVersion uint16, requestBody []byte) ([]byte, error) {
+
+	// Parse request fields (all flexible format)
+	offset := 0
+
+	// IncludeClusterAuthorizedOperations (bool - 1 byte)
+	if offset >= len(requestBody) {
+		return nil, fmt.Errorf("incomplete DescribeCluster request")
+	}
+	includeAuthorizedOps := requestBody[offset] != 0
+	offset++
+
+	// EndpointType (int8, v1+)
+	var endpointType int8 = 1 // Default: brokers
+	if apiVersion >= 1 {
+		if offset >= len(requestBody) {
+			return nil, fmt.Errorf("incomplete DescribeCluster v1+ request")
+		}
+		endpointType = int8(requestBody[offset])
+		offset++
+	}
+
+	// Tagged fields at end of request
+	// (We don't parse them, just skip)
+
+	// Build response
+	response := make([]byte, 0, 256)
+
+	// ThrottleTimeMs (int32)
+	response = append(response, 0, 0, 0, 0)
+
+	// ErrorCode (int16) - no error
+	response = append(response, 0, 0)
+
+	// ErrorMessage (compact nullable string) - null
+	response = append(response, 0x00) // varint 0 = null
+
+	// EndpointType (int8, v1+)
+	if apiVersion >= 1 {
+		response = append(response, byte(endpointType))
+	}
+
+	// ClusterId (compact string)
+	clusterID := "seaweedfs-kafka-gateway"
+	response = append(response, CompactArrayLength(uint32(len(clusterID)))...)
+	response = append(response, []byte(clusterID)...)
+
+	// ControllerId (int32) - use broker ID 1
+	controllerIDBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(controllerIDBytes, uint32(1))
+	response = append(response, controllerIDBytes...)
+
+	// Brokers (compact array)
+	// Get advertised address
+	host, port := h.GetAdvertisedAddress(h.GetGatewayAddress())
+
+	// Broker count (compact array length)
+	response = append(response, CompactArrayLength(1)...) // 1 broker
+
+	// Broker 0: BrokerId(int32) + Host(compact string) + Port(int32) + Rack(compact nullable string) + TaggedFields
+	brokerIDBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(brokerIDBytes, uint32(1))
+	response = append(response, brokerIDBytes...) // BrokerId = 1
+
+	// Host (compact string)
+	response = append(response, CompactArrayLength(uint32(len(host)))...)
+	response = append(response, []byte(host)...)
+
+	// Port (int32) - validate port range
+	if port < 0 || port > 65535 {
+		return nil, fmt.Errorf("invalid port number: %d", port)
+	}
+	portBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(portBytes, uint32(port))
+	response = append(response, portBytes...)
+
+	// Rack (compact nullable string) - null
+	response = append(response, 0x00) // varint 0 = null
+
+	// Per-broker tagged fields
+	response = append(response, 0x00) // Empty tagged fields
+
+	// ClusterAuthorizedOperations (int32) - -2147483648 (INT32_MIN) means not included
+	authOpsBytes := make([]byte, 4)
+	if includeAuthorizedOps {
+		// For now, return 0 (no operations authorized)
+		binary.BigEndian.PutUint32(authOpsBytes, 0)
+	} else {
+		// -2147483648 = INT32_MIN = operations not included
+		binary.BigEndian.PutUint32(authOpsBytes, 0x80000000)
+	}
+	response = append(response, authOpsBytes...)
+
+	// Response-level tagged fields (flexible response)
+	response = append(response, 0x00) // Empty tagged fields
+
+	return response, nil
+}

weed/mq/kafka/protocol/errors.go

@@ -0,0 +1,585 @@
+package protocol
+
+import (
+	"context"
+	"encoding/binary"
+	"net"
+	"time"
+)
+
+// Kafka Protocol Error Codes
+// Based on Apache Kafka protocol specification
+const (
+	// Success
+	ErrorCodeNone int16 = 0
+
+	// General server errors
+	ErrorCodeUnknownServerError           int16 = -1
+	ErrorCodeOffsetOutOfRange             int16 = 1
+	ErrorCodeCorruptMessage               int16 = 3 // Also UNKNOWN_TOPIC_OR_PARTITION
+	ErrorCodeUnknownTopicOrPartition      int16 = 3
+	ErrorCodeInvalidFetchSize             int16 = 4
+	ErrorCodeLeaderNotAvailable           int16 = 5
+	ErrorCodeNotLeaderOrFollower          int16 = 6 // Formerly NOT_LEADER_FOR_PARTITION
+	ErrorCodeRequestTimedOut              int16 = 7
+	ErrorCodeBrokerNotAvailable           int16 = 8
+	ErrorCodeReplicaNotAvailable          int16 = 9
+	ErrorCodeMessageTooLarge              int16 = 10
+	ErrorCodeStaleControllerEpoch         int16 = 11
+	ErrorCodeOffsetMetadataTooLarge       int16 = 12
+	ErrorCodeNetworkException             int16 = 13
+	ErrorCodeOffsetLoadInProgress         int16 = 14
+	ErrorCodeGroupLoadInProgress          int16 = 15
+	ErrorCodeNotCoordinatorForGroup       int16 = 16
+	ErrorCodeNotCoordinatorForTransaction int16 = 17
+
+	// Consumer group coordination errors
+	ErrorCodeIllegalGeneration          int16 = 22
+	ErrorCodeInconsistentGroupProtocol  int16 = 23
+	ErrorCodeInvalidGroupID             int16 = 24
+	ErrorCodeUnknownMemberID            int16 = 25
+	ErrorCodeInvalidSessionTimeout      int16 = 26
+	ErrorCodeRebalanceInProgress        int16 = 27
+	ErrorCodeInvalidCommitOffsetSize    int16 = 28
+	ErrorCodeTopicAuthorizationFailed   int16 = 29
+	ErrorCodeGroupAuthorizationFailed   int16 = 30
+	ErrorCodeClusterAuthorizationFailed int16 = 31
+	ErrorCodeInvalidTimestamp           int16 = 32
+	ErrorCodeUnsupportedSASLMechanism   int16 = 33
+	ErrorCodeIllegalSASLState           int16 = 34
+	ErrorCodeUnsupportedVersion         int16 = 35
+
+	// Topic management errors
+	ErrorCodeTopicAlreadyExists        int16 = 36
+	ErrorCodeInvalidPartitions         int16 = 37
+	ErrorCodeInvalidReplicationFactor  int16 = 38
+	ErrorCodeInvalidReplicaAssignment  int16 = 39
+	ErrorCodeInvalidConfig             int16 = 40
+	ErrorCodeNotController             int16 = 41
+	ErrorCodeInvalidRecord             int16 = 42
+	ErrorCodePolicyViolation           int16 = 43
+	ErrorCodeOutOfOrderSequenceNumber  int16 = 44
+	ErrorCodeDuplicateSequenceNumber   int16 = 45
+	ErrorCodeInvalidProducerEpoch      int16 = 46
+	ErrorCodeInvalidTxnState           int16 = 47
+	ErrorCodeInvalidProducerIDMapping  int16 = 48
+	ErrorCodeInvalidTransactionTimeout int16 = 49
+	ErrorCodeConcurrentTransactions    int16 = 50
+
+	// Connection and timeout errors
+	ErrorCodeConnectionRefused int16 = 60 // Custom for connection issues
+	ErrorCodeConnectionTimeout int16 = 61 // Custom for connection timeouts
+	ErrorCodeReadTimeout       int16 = 62 // Custom for read timeouts
+	ErrorCodeWriteTimeout      int16 = 63 // Custom for write timeouts
+
+	// Consumer group specific errors
+	ErrorCodeMemberIDRequired     int16 = 79
+	ErrorCodeFencedInstanceID     int16 = 82
+	ErrorCodeGroupMaxSizeReached  int16 = 84
+	ErrorCodeUnstableOffsetCommit int16 = 95
+)
+
+// ErrorInfo contains metadata about a Kafka error
+type ErrorInfo struct {
+	Code        int16
+	Name        string
+	Description string
+	Retriable   bool
+}
+
+// KafkaErrors maps error codes to their metadata
+var KafkaErrors = map[int16]ErrorInfo{
+	ErrorCodeNone: {
+		Code: ErrorCodeNone, Name: "NONE", Description: "No error", Retriable: false,
+	},
+	ErrorCodeUnknownServerError: {
+		Code: ErrorCodeUnknownServerError, Name: "UNKNOWN_SERVER_ERROR",
+		Description: "Unknown server error", Retriable: true,
+	},
+	ErrorCodeOffsetOutOfRange: {
+		Code: ErrorCodeOffsetOutOfRange, Name: "OFFSET_OUT_OF_RANGE",
+		Description: "Offset out of range", Retriable: false,
+	},
+	ErrorCodeUnknownTopicOrPartition: {
+		Code: ErrorCodeUnknownTopicOrPartition, Name: "UNKNOWN_TOPIC_OR_PARTITION",
+		Description: "Topic or partition does not exist", Retriable: false,
+	},
+	ErrorCodeInvalidFetchSize: {
+		Code: ErrorCodeInvalidFetchSize, Name: "INVALID_FETCH_SIZE",
+		Description: "Invalid fetch size", Retriable: false,
+	},
+	ErrorCodeLeaderNotAvailable: {
+		Code: ErrorCodeLeaderNotAvailable, Name: "LEADER_NOT_AVAILABLE",
+		Description: "Leader not available", Retriable: true,
+	},
+	ErrorCodeNotLeaderOrFollower: {
+		Code: ErrorCodeNotLeaderOrFollower, Name: "NOT_LEADER_OR_FOLLOWER",
+		Description: "Not leader or follower", Retriable: true,
+	},
+	ErrorCodeRequestTimedOut: {
+		Code: ErrorCodeRequestTimedOut, Name: "REQUEST_TIMED_OUT",
+		Description: "Request timed out", Retriable: true,
+	},
+	ErrorCodeBrokerNotAvailable: {
+		Code: ErrorCodeBrokerNotAvailable, Name: "BROKER_NOT_AVAILABLE",
+		Description: "Broker not available", Retriable: true,
+	},
+	ErrorCodeMessageTooLarge: {
+		Code: ErrorCodeMessageTooLarge, Name: "MESSAGE_TOO_LARGE",
+		Description: "Message size exceeds limit", Retriable: false,
+	},
+	ErrorCodeOffsetMetadataTooLarge: {
+		Code: ErrorCodeOffsetMetadataTooLarge, Name: "OFFSET_METADATA_TOO_LARGE",
+		Description: "Offset metadata too large", Retriable: false,
+	},
+	ErrorCodeNetworkException: {
+		Code: ErrorCodeNetworkException, Name: "NETWORK_EXCEPTION",
+		Description: "Network error", Retriable: true,
+	},
+	ErrorCodeOffsetLoadInProgress: {
+		Code: ErrorCodeOffsetLoadInProgress, Name: "OFFSET_LOAD_IN_PROGRESS",
+		Description: "Offset load in progress", Retriable: true,
+	},
+	ErrorCodeNotCoordinatorForGroup: {
+		Code: ErrorCodeNotCoordinatorForGroup, Name: "NOT_COORDINATOR_FOR_GROUP",
+		Description: "Not coordinator for group", Retriable: true,
+	},
+	ErrorCodeInvalidGroupID: {
+		Code: ErrorCodeInvalidGroupID, Name: "INVALID_GROUP_ID",
+		Description: "Invalid group ID", Retriable: false,
+	},
+	ErrorCodeUnknownMemberID: {
+		Code: ErrorCodeUnknownMemberID, Name: "UNKNOWN_MEMBER_ID",
+		Description: "Unknown member ID", Retriable: false,
+	},
+	ErrorCodeInvalidSessionTimeout: {
+		Code: ErrorCodeInvalidSessionTimeout, Name: "INVALID_SESSION_TIMEOUT",
+		Description: "Invalid session timeout", Retriable: false,
+	},
+	ErrorCodeRebalanceInProgress: {
+		Code: ErrorCodeRebalanceInProgress, Name: "REBALANCE_IN_PROGRESS",
+		Description: "Group rebalance in progress", Retriable: true,
+	},
+	ErrorCodeInvalidCommitOffsetSize: {
+		Code: ErrorCodeInvalidCommitOffsetSize, Name: "INVALID_COMMIT_OFFSET_SIZE",
+		Description: "Invalid commit offset size", Retriable: false,
+	},
+	ErrorCodeTopicAuthorizationFailed: {
+		Code: ErrorCodeTopicAuthorizationFailed, Name: "TOPIC_AUTHORIZATION_FAILED",
+		Description: "Topic authorization failed", Retriable: false,
+	},
+	ErrorCodeGroupAuthorizationFailed: {
+		Code: ErrorCodeGroupAuthorizationFailed, Name: "GROUP_AUTHORIZATION_FAILED",
+		Description: "Group authorization failed", Retriable: false,
+	},
+	ErrorCodeUnsupportedVersion: {
+		Code: ErrorCodeUnsupportedVersion, Name: "UNSUPPORTED_VERSION",
+		Description: "Unsupported version", Retriable: false,
+	},
+	ErrorCodeTopicAlreadyExists: {
+		Code: ErrorCodeTopicAlreadyExists, Name: "TOPIC_ALREADY_EXISTS",
+		Description: "Topic already exists", Retriable: false,
+	},
+	ErrorCodeInvalidPartitions: {
+		Code: ErrorCodeInvalidPartitions, Name: "INVALID_PARTITIONS",
+		Description: "Invalid number of partitions", Retriable: false,
+	},
+	ErrorCodeInvalidReplicationFactor: {
+		Code: ErrorCodeInvalidReplicationFactor, Name: "INVALID_REPLICATION_FACTOR",
+		Description: "Invalid replication factor", Retriable: false,
+	},
+	ErrorCodeInvalidRecord: {
+		Code: ErrorCodeInvalidRecord, Name: "INVALID_RECORD",
+		Description: "Invalid record", Retriable: false,
+	},
+	ErrorCodeConnectionRefused: {
+		Code: ErrorCodeConnectionRefused, Name: "CONNECTION_REFUSED",
+		Description: "Connection refused", Retriable: true,
+	},
+	ErrorCodeConnectionTimeout: {
+		Code: ErrorCodeConnectionTimeout, Name: "CONNECTION_TIMEOUT",
+		Description: "Connection timeout", Retriable: true,
+	},
+	ErrorCodeReadTimeout: {
+		Code: ErrorCodeReadTimeout, Name: "READ_TIMEOUT",
+		Description: "Read operation timeout", Retriable: true,
+	},
+	ErrorCodeWriteTimeout: {
+		Code: ErrorCodeWriteTimeout, Name: "WRITE_TIMEOUT",
+		Description: "Write operation timeout", Retriable: true,
+	},
+	ErrorCodeIllegalGeneration: {
+		Code: ErrorCodeIllegalGeneration, Name: "ILLEGAL_GENERATION",
+		Description: "Illegal generation", Retriable: false,
+	},
+	ErrorCodeInconsistentGroupProtocol: {
+		Code: ErrorCodeInconsistentGroupProtocol, Name: "INCONSISTENT_GROUP_PROTOCOL",
+		Description: "Inconsistent group protocol", Retriable: false,
+	},
+	ErrorCodeMemberIDRequired: {
+		Code: ErrorCodeMemberIDRequired, Name: "MEMBER_ID_REQUIRED",
+		Description: "Member ID required", Retriable: false,
+	},
+	ErrorCodeFencedInstanceID: {
+		Code: ErrorCodeFencedInstanceID, Name: "FENCED_INSTANCE_ID",
+		Description: "Instance ID fenced", Retriable: false,
+	},
+	ErrorCodeGroupMaxSizeReached: {
+		Code: ErrorCodeGroupMaxSizeReached, Name: "GROUP_MAX_SIZE_REACHED",
+		Description: "Group max size reached", Retriable: false,
+	},
+	ErrorCodeUnstableOffsetCommit: {
+		Code: ErrorCodeUnstableOffsetCommit, Name: "UNSTABLE_OFFSET_COMMIT",
+		Description: "Offset commit during rebalance", Retriable: true,
+	},
+}
+
+// GetErrorInfo returns error information for the given error code
+func GetErrorInfo(code int16) ErrorInfo {
+	if info, exists := KafkaErrors[code]; exists {
+		return info
+	}
+	return ErrorInfo{
+		Code: code, Name: "UNKNOWN", Description: "Unknown error code", Retriable: false,
+	}
+}
+
+// IsRetriableError returns true if the error is retriable
+func IsRetriableError(code int16) bool {
+	return GetErrorInfo(code).Retriable
+}
+
+// BuildErrorResponse builds a standard Kafka error response (2-byte error code only).
+// Prefer BuildAPIErrorResponse for API-aware error bodies.
+func BuildErrorResponse(correlationID uint32, errorCode int16) []byte {
+	response := make([]byte, 0, 8)
+
+	// NOTE: Correlation ID is handled by writeResponseWithCorrelationID
+	// Do NOT include it in the response body
+
+	// Error code (2 bytes)
+	errorCodeBytes := make([]byte, 2)
+	binary.BigEndian.PutUint16(errorCodeBytes, uint16(errorCode))
+	response = append(response, errorCodeBytes...)
+
+	return response
+}
+
+// BuildAPIErrorResponse builds a minimal-but-valid error response body whose
+// layout matches the schema the client expects for the given API key and
+// version.  The correlation ID and header-level tagged fields are NOT included
+// (those are added by writeResponseWithHeader).
+func BuildAPIErrorResponse(apiKey, apiVersion uint16, errorCode int16) []byte {
+	ec := make([]byte, 2)
+	binary.BigEndian.PutUint16(ec, uint16(errorCode))
+	throttle := []byte{0, 0, 0, 0} // throttle_time_ms = 0
+	emptyArr := []byte{0, 0, 0, 0} // regular array length = 0
+	nullStr := []byte{0xFF, 0xFF}  // nullable string = null
+	emptyStr := []byte{0, 0}       // string length = 0
+
+	switch APIKey(apiKey) {
+
+	// --- error_code is the first body field -----------------------------------
+
+	case APIKeyApiVersions:
+		// error_code(2) + api_keys_array + [throttle_time_ms v1+] [+ tagged_fields v3+]
+		buf := append([]byte{}, ec...)
+		if apiVersion >= 3 {
+			buf = append(buf, 1) // compact array length=0 (varint 1 = 0+1)
+		} else {
+			buf = append(buf, emptyArr...)
+		}
+		if apiVersion >= 1 {
+			buf = append(buf, throttle...)
+		}
+		if apiVersion >= 3 {
+			buf = append(buf, 0) // body-level tagged fields
+		}
+		return buf
+
+	// --- throttle_time_ms(4) + error_code(2) + trailing fields ----------------
+
+	case APIKeyFindCoordinator:
+		// [throttle v1+] + error_code + error_msg + node_id + host + port
+		buf := make([]byte, 0, 24)
+		if apiVersion >= 1 {
+			buf = append(buf, throttle...)
+		}
+		buf = append(buf, ec...)
+		buf = append(buf, nullStr...)             // error_message
+		buf = append(buf, 0xFF, 0xFF, 0xFF, 0xFF) // node_id = -1
+		buf = append(buf, emptyStr...)            // host
+		buf = append(buf, 0, 0, 0, 0)             // port = 0
+		return buf
+
+	case APIKeyJoinGroup:
+		// [throttle v2+] + error_code + generation_id + protocol_name + leader + member_id + members[]
+		buf := make([]byte, 0, 24)
+		if apiVersion >= 2 {
+			buf = append(buf, throttle...)
+		}
+		buf = append(buf, ec...)
+		buf = append(buf, 0xFF, 0xFF, 0xFF, 0xFF) // generation_id = -1
+		buf = append(buf, nullStr...)             // protocol_name
+		buf = append(buf, emptyStr...)            // leader
+		buf = append(buf, emptyStr...)            // member_id
+		buf = append(buf, emptyArr...)            // members = []
+		return buf
+
+	case APIKeySyncGroup:
+		// [throttle v1+] + error_code + assignment
+		buf := make([]byte, 0, 12)
+		if apiVersion >= 1 {
+			buf = append(buf, throttle...)
+		}
+		buf = append(buf, ec...)
+		buf = append(buf, 0, 0, 0, 0) // assignment bytes length = 0
+		return buf
+
+	case APIKeyHeartbeat:
+		// [throttle v1+] + error_code
+		buf := make([]byte, 0, 8)
+		if apiVersion >= 1 {
+			buf = append(buf, throttle...)
+		}
+		buf = append(buf, ec...)
+		return buf
+
+	case APIKeyLeaveGroup:
+		// [throttle v1+] + error_code [+ members v3+]
+		buf := make([]byte, 0, 12)
+		if apiVersion >= 1 {
+			buf = append(buf, throttle...)
+		}
+		buf = append(buf, ec...)
+		if apiVersion >= 3 {
+			buf = append(buf, emptyArr...) // members = []
+		}
+		return buf
+
+	case APIKeyInitProducerId:
+		// throttle + error_code + producer_id + producer_epoch
+		buf := make([]byte, 0, 18)
+		buf = append(buf, throttle...)
+		buf = append(buf, ec...)
+		buf = append(buf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF) // producer_id = -1
+		buf = append(buf, 0xFF, 0xFF)                                     // producer_epoch = -1
+		return buf
+
+	case APIKeyListGroups:
+		// [throttle v1+] + error_code + groups[]
+		buf := make([]byte, 0, 12)
+		if apiVersion >= 1 {
+			buf = append(buf, throttle...)
+		}
+		buf = append(buf, ec...)
+		buf = append(buf, emptyArr...) // groups = []
+		return buf
+
+	case APIKeyDescribeCluster:
+		// throttle + error_code + error_msg + cluster_id + controller_id + brokers[] + cluster_authorized_operations
+		buf := make([]byte, 0, 24)
+		buf = append(buf, throttle...)
+		buf = append(buf, ec...)
+		buf = append(buf, nullStr...)             // error_message
+		buf = append(buf, emptyStr...)            // cluster_id
+		buf = append(buf, 0xFF, 0xFF, 0xFF, 0xFF) // controller_id = -1
+		buf = append(buf, emptyArr...)            // brokers = []
+		buf = append(buf, 0, 0, 0, 0)             // cluster_authorized_operations
+		return buf
+
+	// --- array-based responses (no top-level error_code) ----------------------
+
+	case APIKeyProduce:
+		// topics[] + [throttle v1+]
+		buf := append([]byte{}, emptyArr...)
+		if apiVersion >= 1 {
+			buf = append(buf, throttle...)
+		}
+		return buf
+
+	case APIKeyFetch:
+		// [throttle v1+] [+ error_code + session_id v7+] + topics[]
+		buf := make([]byte, 0, 16)
+		if apiVersion >= 1 {
+			buf = append(buf, throttle...)
+		}
+		if apiVersion >= 7 {
+			buf = append(buf, ec...)      // error_code
+			buf = append(buf, 0, 0, 0, 0) // session_id = 0
+		}
+		buf = append(buf, emptyArr...) // topics = []
+		return buf
+
+	case APIKeyMetadata:
+		// [throttle v3+] + brokers[] + [cluster_id v2+] + [controller_id v1+] + topics[]
+		buf := make([]byte, 0, 24)
+		if apiVersion >= 3 {
+			buf = append(buf, throttle...)
+		}
+		buf = append(buf, emptyArr...) // brokers = []
+		if apiVersion >= 2 {
+			buf = append(buf, nullStr...) // cluster_id
+		}
+		if apiVersion >= 1 {
+			buf = append(buf, 0xFF, 0xFF, 0xFF, 0xFF) // controller_id = -1
+		}
+		buf = append(buf, emptyArr...) // topics = []
+		return buf
+
+	case APIKeyListOffsets:
+		// [throttle v2+] + topics[]
+		buf := make([]byte, 0, 8)
+		if apiVersion >= 2 {
+			buf = append(buf, throttle...)
+		}
+		buf = append(buf, emptyArr...)
+		return buf
+
+	case APIKeyOffsetCommit:
+		// [throttle v3+] + topics[]
+		buf := make([]byte, 0, 8)
+		if apiVersion >= 3 {
+			buf = append(buf, throttle...)
+		}
+		buf = append(buf, emptyArr...)
+		return buf
+
+	case APIKeyOffsetFetch:
+		// [throttle v3+] + topics[] [+ error_code v2+]
+		buf := make([]byte, 0, 12)
+		if apiVersion >= 3 {
+			buf = append(buf, throttle...)
+		}
+		buf = append(buf, emptyArr...) // topics = []
+		if apiVersion >= 2 {
+			buf = append(buf, ec...) // error_code
+		}
+		return buf
+
+	case APIKeyCreateTopics:
+		// throttle + topics[]
+		buf := append([]byte{}, throttle...)
+		buf = append(buf, emptyArr...)
+		return buf
+
+	case APIKeyDeleteTopics:
+		// throttle + topics[]
+		buf := append([]byte{}, throttle...)
+		buf = append(buf, emptyArr...)
+		return buf
+
+	case APIKeyDescribeGroups:
+		// throttle + groups[]
+		buf := append([]byte{}, throttle...)
+		buf = append(buf, emptyArr...)
+		return buf
+
+	case APIKeyDescribeConfigs:
+		// throttle + resources[]
+		buf := append([]byte{}, throttle...)
+		buf = append(buf, emptyArr...)
+		return buf
+
+	default:
+		// Unknown API — emit just the error code as a best-effort fallback
+		return append([]byte{}, ec...)
+	}
+}
+
+// BuildErrorResponseWithMessage builds a Kafka error response with error message
+func BuildErrorResponseWithMessage(correlationID uint32, errorCode int16, message string) []byte {
+	response := BuildErrorResponse(correlationID, errorCode)
+
+	// Error message (2 bytes length + message)
+	if message == "" {
+		response = append(response, 0xFF, 0xFF) // Null string
+	} else {
+		messageLen := uint16(len(message))
+		messageLenBytes := make([]byte, 2)
+		binary.BigEndian.PutUint16(messageLenBytes, messageLen)
+		response = append(response, messageLenBytes...)
+		response = append(response, []byte(message)...)
+	}
+
+	return response
+}
+
+// ClassifyNetworkError classifies network errors into appropriate Kafka error codes
+func ClassifyNetworkError(err error) int16 {
+	if err == nil {
+		return ErrorCodeNone
+	}
+
+	// Check for network errors
+	if netErr, ok := err.(net.Error); ok {
+		if netErr.Timeout() {
+			return ErrorCodeRequestTimedOut
+		}
+		return ErrorCodeNetworkException
+	}
+
+	// Check for specific error types
+	switch err.Error() {
+	case "connection refused":
+		return ErrorCodeConnectionRefused
+	case "connection timeout":
+		return ErrorCodeConnectionTimeout
+	default:
+		return ErrorCodeUnknownServerError
+	}
+}
+
+// TimeoutConfig holds timeout configuration for connections and operations
+type TimeoutConfig struct {
+	ConnectionTimeout time.Duration // Timeout for establishing connections
+	ReadTimeout       time.Duration // Timeout for read operations
+	WriteTimeout      time.Duration // Timeout for write operations
+	RequestTimeout    time.Duration // Overall request timeout
+}
+
+// DefaultTimeoutConfig returns default timeout configuration
+func DefaultTimeoutConfig() TimeoutConfig {
+	return TimeoutConfig{
+		ConnectionTimeout: 30 * time.Second,
+		ReadTimeout:       10 * time.Second,
+		WriteTimeout:      10 * time.Second,
+		RequestTimeout:    30 * time.Second,
+	}
+}
+
+// HandleTimeoutError handles timeout errors and returns appropriate error code
+func HandleTimeoutError(err error, operation string) int16 {
+	if err == nil {
+		return ErrorCodeNone
+	}
+
+	// Handle context timeout errors
+	if err == context.DeadlineExceeded {
+		switch operation {
+		case "read":
+			return ErrorCodeReadTimeout
+		case "write":
+			return ErrorCodeWriteTimeout
+		case "connect":
+			return ErrorCodeConnectionTimeout
+		default:
+			return ErrorCodeRequestTimedOut
+		}
+	}
+
+	if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
+		switch operation {
+		case "read":
+			return ErrorCodeReadTimeout
+		case "write":
+			return ErrorCodeWriteTimeout
+		case "connect":
+			return ErrorCodeConnectionTimeout
+		default:
+			return ErrorCodeRequestTimedOut
+		}
+	}
+
+	return ClassifyNetworkError(err)
+}

weed/mq/kafka/protocol/fetch_multibatch.go

@@ -0,0 +1,624 @@
+package protocol
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/binary"
+	"fmt"
+	"hash/crc32"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/compression"
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/integration"
+)
+
+// MultiBatchFetcher handles fetching multiple record batches with size limits
+type MultiBatchFetcher struct {
+	handler *Handler
+}
+
+// NewMultiBatchFetcher creates a new multi-batch fetcher
+func NewMultiBatchFetcher(handler *Handler) *MultiBatchFetcher {
+	return &MultiBatchFetcher{handler: handler}
+}
+
+// FetchResult represents the result of a multi-batch fetch operation
+type FetchResult struct {
+	RecordBatches []byte // Concatenated record batches
+	NextOffset    int64  // Next offset to fetch from
+	TotalSize     int32  // Total size of all batches
+	BatchCount    int    // Number of batches included
+}
+
+// FetchMultipleBatches fetches multiple record batches up to maxBytes limit
+// ctx controls the fetch timeout (should match Kafka fetch request's MaxWaitTime)
+func (f *MultiBatchFetcher) FetchMultipleBatches(ctx context.Context, topicName string, partitionID int32, startOffset, highWaterMark int64, maxBytes int32) (*FetchResult, error) {
+
+	if startOffset >= highWaterMark {
+		return &FetchResult{
+			RecordBatches: []byte{},
+			NextOffset:    startOffset,
+			TotalSize:     0,
+			BatchCount:    0,
+		}, nil
+	}
+
+	// Minimum size for basic response headers and one empty batch
+	minResponseSize := int32(200)
+	if maxBytes < minResponseSize {
+		maxBytes = minResponseSize
+	}
+
+	var combinedBatches []byte
+	currentOffset := startOffset
+	totalSize := int32(0)
+	batchCount := 0
+
+	// Estimate records per batch based on maxBytes available
+	// Assume average message size + batch overhead
+	// Client requested maxBytes, we should use most of it
+	// Start with larger batches to maximize throughput
+	estimatedMsgSize := int32(1024)                        // Typical message size with overhead
+	recordsPerBatch := (maxBytes - 200) / estimatedMsgSize // Use available space efficiently
+	if recordsPerBatch < 100 {
+		recordsPerBatch = 100 // Minimum 100 records per batch
+	}
+	if recordsPerBatch > 10000 {
+		recordsPerBatch = 10000 // Cap at 10k records per batch to avoid huge memory allocations
+	}
+	maxBatchesPerFetch := int((maxBytes - 200) / (estimatedMsgSize * 10)) // Reasonable limit
+	if maxBatchesPerFetch < 5 {
+		maxBatchesPerFetch = 5 // At least 5 batches
+	}
+	if maxBatchesPerFetch > 100 {
+		maxBatchesPerFetch = 100 // At most 100 batches
+	}
+
+	for batchCount < maxBatchesPerFetch && currentOffset < highWaterMark {
+
+		// Calculate remaining space
+		remainingBytes := maxBytes - totalSize
+		if remainingBytes < 100 { // Need at least 100 bytes for a minimal batch
+			break
+		}
+
+		// Adapt records per batch based on remaining space
+		// If we have less space remaining, fetch fewer records to avoid going over
+		currentBatchSize := recordsPerBatch
+		if remainingBytes < recordsPerBatch*estimatedMsgSize {
+			currentBatchSize = remainingBytes / estimatedMsgSize
+			if currentBatchSize < 1 {
+				currentBatchSize = 1
+			}
+		}
+
+		// Calculate how many records to fetch for this batch
+		recordsAvailable := highWaterMark - currentOffset
+		if recordsAvailable <= 0 {
+			break
+		}
+
+		recordsToFetch := currentBatchSize
+		if int64(recordsToFetch) > recordsAvailable {
+			recordsToFetch = int32(recordsAvailable)
+		}
+
+		// Check if handler is nil
+		if f.handler == nil {
+			break
+		}
+		if f.handler.seaweedMQHandler == nil {
+			break
+		}
+
+		// Fetch records for this batch
+		// Pass context to respect Kafka fetch request's MaxWaitTime
+		smqRecords, err := f.handler.seaweedMQHandler.GetStoredRecords(ctx, topicName, partitionID, currentOffset, int(recordsToFetch))
+
+		if err != nil || len(smqRecords) == 0 {
+			break
+		}
+
+		// Note: we construct the batch and check actual size after construction
+
+		// Construct record batch
+		batch := f.constructSingleRecordBatch(topicName, currentOffset, smqRecords)
+		batchSize := int32(len(batch))
+
+		// Double-check actual size doesn't exceed maxBytes
+		if totalSize+batchSize > maxBytes && batchCount > 0 {
+			break
+		}
+
+		// Add this batch to combined result
+		combinedBatches = append(combinedBatches, batch...)
+		totalSize += batchSize
+		currentOffset += int64(len(smqRecords))
+		batchCount++
+
+		// If this is a small batch, we might be at the end
+		if len(smqRecords) < int(recordsPerBatch) {
+			break
+		}
+	}
+
+	result := &FetchResult{
+		RecordBatches: combinedBatches,
+		NextOffset:    currentOffset,
+		TotalSize:     totalSize,
+		BatchCount:    batchCount,
+	}
+
+	return result, nil
+}
+
+// constructSingleRecordBatch creates a single record batch from SMQ records
+func (f *MultiBatchFetcher) constructSingleRecordBatch(topicName string, baseOffset int64, smqRecords []integration.SMQRecord) []byte {
+	if len(smqRecords) == 0 {
+		return f.constructEmptyRecordBatch(baseOffset)
+	}
+
+	// Create record batch using the SMQ records
+	batch := make([]byte, 0, 512)
+
+	// Record batch header
+	baseOffsetBytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(baseOffsetBytes, uint64(baseOffset))
+	batch = append(batch, baseOffsetBytes...) // base offset (8 bytes)
+
+	// Calculate batch length (will be filled after we know the size)
+	batchLengthPos := len(batch)
+	batch = append(batch, 0, 0, 0, 0) // batch length placeholder (4 bytes)
+
+	// Partition leader epoch (4 bytes) - use 0 (real Kafka uses 0, not -1)
+	batch = append(batch, 0x00, 0x00, 0x00, 0x00)
+
+	// Magic byte (1 byte) - v2 format
+	batch = append(batch, 2)
+
+	// CRC placeholder (4 bytes) - will be calculated later
+	crcPos := len(batch)
+	batch = append(batch, 0, 0, 0, 0)
+
+	// Attributes (2 bytes) - no compression, etc.
+	batch = append(batch, 0, 0)
+
+	// Last offset delta (4 bytes)
+	lastOffsetDelta := int32(len(smqRecords) - 1)
+	lastOffsetDeltaBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(lastOffsetDeltaBytes, uint32(lastOffsetDelta))
+	batch = append(batch, lastOffsetDeltaBytes...)
+
+	// Base timestamp (8 bytes) - convert from nanoseconds to milliseconds for Kafka compatibility
+	baseTimestamp := smqRecords[0].GetTimestamp() / 1000000 // Convert nanoseconds to milliseconds
+	baseTimestampBytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(baseTimestampBytes, uint64(baseTimestamp))
+	batch = append(batch, baseTimestampBytes...)
+
+	// Max timestamp (8 bytes) - convert from nanoseconds to milliseconds for Kafka compatibility
+	maxTimestamp := baseTimestamp
+	if len(smqRecords) > 1 {
+		maxTimestamp = smqRecords[len(smqRecords)-1].GetTimestamp() / 1000000 // Convert nanoseconds to milliseconds
+	}
+	maxTimestampBytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(maxTimestampBytes, uint64(maxTimestamp))
+	batch = append(batch, maxTimestampBytes...)
+
+	// Producer ID (8 bytes) - use -1 for no producer ID
+	batch = append(batch, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF)
+
+	// Producer epoch (2 bytes) - use -1 for no producer epoch
+	batch = append(batch, 0xFF, 0xFF)
+
+	// Base sequence (4 bytes) - use -1 for no base sequence
+	batch = append(batch, 0xFF, 0xFF, 0xFF, 0xFF)
+
+	// Records count (4 bytes)
+	recordCountBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(recordCountBytes, uint32(len(smqRecords)))
+	batch = append(batch, recordCountBytes...)
+
+	// Add individual records from SMQ records
+	for i, smqRecord := range smqRecords {
+		// Build individual record
+		recordBytes := make([]byte, 0, 128)
+
+		// Record attributes (1 byte)
+		recordBytes = append(recordBytes, 0)
+
+		// Timestamp delta (varint) - calculate from base timestamp (both in milliseconds)
+		recordTimestampMs := smqRecord.GetTimestamp() / 1000000 // Convert nanoseconds to milliseconds
+		timestampDelta := recordTimestampMs - baseTimestamp     // Both in milliseconds now
+		recordBytes = append(recordBytes, encodeVarint(timestampDelta)...)
+
+		// Offset delta (varint)
+		offsetDelta := int64(i)
+		recordBytes = append(recordBytes, encodeVarint(offsetDelta)...)
+
+		// Key length and key (varint + data) - decode RecordValue to get original Kafka message
+		key := f.handler.decodeRecordValueToKafkaMessage(topicName, smqRecord.GetKey())
+		if key == nil {
+			recordBytes = append(recordBytes, encodeVarint(-1)...) // null key
+		} else {
+			recordBytes = append(recordBytes, encodeVarint(int64(len(key)))...)
+			recordBytes = append(recordBytes, key...)
+		}
+
+		// Value length and value (varint + data) - decode RecordValue to get original Kafka message
+		value := f.handler.decodeRecordValueToKafkaMessage(topicName, smqRecord.GetValue())
+
+		if value == nil {
+			recordBytes = append(recordBytes, encodeVarint(-1)...) // null value
+		} else {
+			recordBytes = append(recordBytes, encodeVarint(int64(len(value)))...)
+			recordBytes = append(recordBytes, value...)
+		}
+
+		// Headers count (varint) - 0 headers
+		recordBytes = append(recordBytes, encodeVarint(0)...)
+
+		// Prepend record length (varint)
+		recordLength := int64(len(recordBytes))
+		batch = append(batch, encodeVarint(recordLength)...)
+		batch = append(batch, recordBytes...)
+	}
+
+	// Fill in the batch length
+	batchLength := uint32(len(batch) - batchLengthPos - 4)
+	binary.BigEndian.PutUint32(batch[batchLengthPos:batchLengthPos+4], batchLength)
+
+	// Debug: Log reconstructed batch (only at high verbosity)
+	if glog.V(4) {
+		fmt.Printf("\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n")
+		fmt.Printf("📏 RECONSTRUCTED BATCH: topic=%s baseOffset=%d size=%d bytes, recordCount=%d\n",
+			topicName, baseOffset, len(batch), len(smqRecords))
+	}
+
+	if glog.V(4) && len(batch) >= 61 {
+		fmt.Printf("  Header Structure:\n")
+		fmt.Printf("    Base Offset (0-7):     %x\n", batch[0:8])
+		fmt.Printf("    Batch Length (8-11):   %x\n", batch[8:12])
+		fmt.Printf("    Leader Epoch (12-15):  %x\n", batch[12:16])
+		fmt.Printf("    Magic (16):            %x\n", batch[16:17])
+		fmt.Printf("    CRC (17-20):           %x (WILL BE CALCULATED)\n", batch[17:21])
+		fmt.Printf("    Attributes (21-22):    %x\n", batch[21:23])
+		fmt.Printf("    Last Offset Delta (23-26): %x\n", batch[23:27])
+		fmt.Printf("    Base Timestamp (27-34): %x\n", batch[27:35])
+		fmt.Printf("    Max Timestamp (35-42):  %x\n", batch[35:43])
+		fmt.Printf("    Producer ID (43-50):    %x\n", batch[43:51])
+		fmt.Printf("    Producer Epoch (51-52): %x\n", batch[51:53])
+		fmt.Printf("    Base Sequence (53-56):  %x\n", batch[53:57])
+		fmt.Printf("    Record Count (57-60):   %x\n", batch[57:61])
+		if len(batch) > 61 {
+			fmt.Printf("    Records Section (61+):  %x... (%d bytes)\n",
+				batch[61:min(81, len(batch))], len(batch)-61)
+		}
+	}
+
+	// Calculate CRC32 for the batch
+	// Per Kafka spec: CRC covers ONLY from attributes offset (byte 21) onwards
+	// See: DefaultRecordBatch.java computeChecksum() - Crc32C.compute(buffer, ATTRIBUTES_OFFSET, ...)
+	crcData := batch[crcPos+4:] // Skip CRC field itself, include rest
+	crc := crc32.Checksum(crcData, crc32.MakeTable(crc32.Castagnoli))
+
+	// CRC debug (only at high verbosity)
+	if glog.V(4) {
+		batchLengthValue := binary.BigEndian.Uint32(batch[8:12])
+		expectedTotalSize := 12 + int(batchLengthValue)
+		actualTotalSize := len(batch)
+
+		fmt.Printf("\n  === CRC CALCULATION DEBUG ===\n")
+		fmt.Printf("    Batch length field (bytes 8-11): %d\n", batchLengthValue)
+		fmt.Printf("    Expected total batch size: %d bytes (12 + %d)\n", expectedTotalSize, batchLengthValue)
+		fmt.Printf("    Actual batch size: %d bytes\n", actualTotalSize)
+		fmt.Printf("    CRC position: byte %d\n", crcPos)
+		fmt.Printf("    CRC data range: bytes %d to %d (%d bytes)\n", crcPos+4, actualTotalSize-1, len(crcData))
+
+		if expectedTotalSize != actualTotalSize {
+			fmt.Printf("    SIZE MISMATCH: %d bytes difference!\n", actualTotalSize-expectedTotalSize)
+		}
+
+		if crcPos != 17 {
+			fmt.Printf("    CRC POSITION WRONG: expected 17, got %d!\n", crcPos)
+		}
+
+		fmt.Printf("    CRC data (first 100 bytes of %d):\n", len(crcData))
+		dumpSize := 100
+		if len(crcData) < dumpSize {
+			dumpSize = len(crcData)
+		}
+		for i := 0; i < dumpSize; i += 20 {
+			end := i + 20
+			if end > dumpSize {
+				end = dumpSize
+			}
+			fmt.Printf("      [%3d-%3d]: %x\n", i, end-1, crcData[i:end])
+		}
+
+		manualCRC := crc32.Checksum(crcData, crc32.MakeTable(crc32.Castagnoli))
+		fmt.Printf("    Calculated CRC: 0x%08x\n", crc)
+		fmt.Printf("    Manual verify:  0x%08x", manualCRC)
+		if crc == manualCRC {
+			fmt.Printf(" OK\n")
+		} else {
+			fmt.Printf(" MISMATCH!\n")
+		}
+
+		if actualTotalSize <= 200 {
+			fmt.Printf("    Complete batch hex dump (%d bytes):\n", actualTotalSize)
+			for i := 0; i < actualTotalSize; i += 16 {
+				end := i + 16
+				if end > actualTotalSize {
+					end = actualTotalSize
+				}
+				fmt.Printf("      %04d: %x\n", i, batch[i:end])
+			}
+		}
+		fmt.Printf("  === END CRC DEBUG ===\n\n")
+	}
+
+	binary.BigEndian.PutUint32(batch[crcPos:crcPos+4], crc)
+
+	if glog.V(4) {
+		fmt.Printf("    Final CRC (17-20):     %x (calculated over %d bytes)\n", batch[17:21], len(crcData))
+
+		// VERIFICATION: Read back what we just wrote
+		writtenCRC := binary.BigEndian.Uint32(batch[17:21])
+		fmt.Printf("    VERIFICATION: CRC we calculated=0x%x, CRC written to batch=0x%x", crc, writtenCRC)
+		if crc == writtenCRC {
+			fmt.Printf(" OK\n")
+		} else {
+			fmt.Printf(" MISMATCH!\n")
+		}
+
+		// DEBUG: Hash the entire batch to check if reconstructions are identical
+		batchHash := crc32.ChecksumIEEE(batch)
+		fmt.Printf("    BATCH IDENTITY: hash=0x%08x size=%d topic=%s baseOffset=%d recordCount=%d\n",
+			batchHash, len(batch), topicName, baseOffset, len(smqRecords))
+
+		// DEBUG: Show first few record keys/values to verify consistency
+		if len(smqRecords) > 0 && strings.Contains(topicName, "loadtest") {
+			fmt.Printf("    RECORD SAMPLES:\n")
+			for i := 0; i < min(3, len(smqRecords)); i++ {
+				keyPreview := smqRecords[i].GetKey()
+				if len(keyPreview) > 20 {
+					keyPreview = keyPreview[:20]
+				}
+				valuePreview := smqRecords[i].GetValue()
+				if len(valuePreview) > 40 {
+					valuePreview = valuePreview[:40]
+				}
+				fmt.Printf("      [%d] keyLen=%d valueLen=%d keyHex=%x valueHex=%x\n",
+					i, len(smqRecords[i].GetKey()), len(smqRecords[i].GetValue()),
+					keyPreview, valuePreview)
+			}
+		}
+
+		fmt.Printf("    Batch for topic=%s baseOffset=%d recordCount=%d\n", topicName, baseOffset, len(smqRecords))
+		fmt.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n")
+	}
+
+	return batch
+}
+
+// constructEmptyRecordBatch creates an empty record batch
+func (f *MultiBatchFetcher) constructEmptyRecordBatch(baseOffset int64) []byte {
+	// Create minimal empty record batch
+	batch := make([]byte, 0, 61)
+
+	// Base offset (8 bytes)
+	baseOffsetBytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(baseOffsetBytes, uint64(baseOffset))
+	batch = append(batch, baseOffsetBytes...)
+
+	// Batch length (4 bytes) - will be filled at the end
+	lengthPos := len(batch)
+	batch = append(batch, 0, 0, 0, 0)
+
+	// Partition leader epoch (4 bytes) - -1
+	batch = append(batch, 0xFF, 0xFF, 0xFF, 0xFF)
+
+	// Magic byte (1 byte) - version 2
+	batch = append(batch, 2)
+
+	// CRC32 (4 bytes) - placeholder
+	crcPos := len(batch)
+	batch = append(batch, 0, 0, 0, 0)
+
+	// Attributes (2 bytes) - no compression, no transactional
+	batch = append(batch, 0, 0)
+
+	// Last offset delta (4 bytes) - -1 for empty batch
+	batch = append(batch, 0xFF, 0xFF, 0xFF, 0xFF)
+
+	// Base timestamp (8 bytes)
+	timestamp := uint64(1640995200000) // Fixed timestamp for empty batches
+	timestampBytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(timestampBytes, timestamp)
+	batch = append(batch, timestampBytes...)
+
+	// Max timestamp (8 bytes) - same as base for empty batch
+	batch = append(batch, timestampBytes...)
+
+	// Producer ID (8 bytes) - -1 for non-transactional
+	batch = append(batch, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF)
+
+	// Producer Epoch (2 bytes) - -1 for non-transactional
+	batch = append(batch, 0xFF, 0xFF)
+
+	// Base Sequence (4 bytes) - -1 for non-transactional
+	batch = append(batch, 0xFF, 0xFF, 0xFF, 0xFF)
+
+	// Record count (4 bytes) - 0 for empty batch
+	batch = append(batch, 0, 0, 0, 0)
+
+	// Fill in the batch length
+	batchLength := len(batch) - 12 // Exclude base offset and length field itself
+	binary.BigEndian.PutUint32(batch[lengthPos:lengthPos+4], uint32(batchLength))
+
+	// Calculate CRC32 for the batch
+	// Per Kafka spec: CRC covers ONLY from attributes offset (byte 21) onwards
+	// See: DefaultRecordBatch.java computeChecksum() - Crc32C.compute(buffer, ATTRIBUTES_OFFSET, ...)
+	crcData := batch[crcPos+4:] // Skip CRC field itself, include rest
+	crc := crc32.Checksum(crcData, crc32.MakeTable(crc32.Castagnoli))
+	binary.BigEndian.PutUint32(batch[crcPos:crcPos+4], crc)
+
+	return batch
+}
+
+// CompressedBatchResult represents a compressed record batch result
+type CompressedBatchResult struct {
+	CompressedData []byte
+	OriginalSize   int32
+	CompressedSize int32
+	Codec          compression.CompressionCodec
+}
+
+// CreateCompressedBatch creates a compressed record batch (basic support)
+func (f *MultiBatchFetcher) CreateCompressedBatch(baseOffset int64, smqRecords []integration.SMQRecord, codec compression.CompressionCodec) (*CompressedBatchResult, error) {
+	if codec == compression.None {
+		// No compression requested
+		batch := f.constructSingleRecordBatch("", baseOffset, smqRecords)
+		return &CompressedBatchResult{
+			CompressedData: batch,
+			OriginalSize:   int32(len(batch)),
+			CompressedSize: int32(len(batch)),
+			Codec:          compression.None,
+		}, nil
+	}
+
+	// For Phase 5, implement basic GZIP compression support
+	originalBatch := f.constructSingleRecordBatch("", baseOffset, smqRecords)
+	originalSize := int32(len(originalBatch))
+
+	compressedData, err := f.compressData(originalBatch, codec)
+	if err != nil {
+		// Fall back to uncompressed if compression fails
+		return &CompressedBatchResult{
+			CompressedData: originalBatch,
+			OriginalSize:   originalSize,
+			CompressedSize: originalSize,
+			Codec:          compression.None,
+		}, nil
+	}
+
+	// Create compressed record batch with proper headers
+	compressedBatch := f.constructCompressedRecordBatch(baseOffset, compressedData, codec, originalSize)
+
+	return &CompressedBatchResult{
+		CompressedData: compressedBatch,
+		OriginalSize:   originalSize,
+		CompressedSize: int32(len(compressedBatch)),
+		Codec:          codec,
+	}, nil
+}
+
+// constructCompressedRecordBatch creates a record batch with compressed records
+func (f *MultiBatchFetcher) constructCompressedRecordBatch(baseOffset int64, compressedRecords []byte, codec compression.CompressionCodec, originalSize int32) []byte {
+	// Validate size to prevent overflow
+	const maxBatchSize = 1 << 30 // 1 GB limit
+	if len(compressedRecords) > maxBatchSize-100 {
+		glog.Errorf("Compressed records too large: %d bytes", len(compressedRecords))
+		return nil
+	}
+	batch := make([]byte, 0, len(compressedRecords)+100)
+
+	// Record batch header is similar to regular batch
+	baseOffsetBytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(baseOffsetBytes, uint64(baseOffset))
+	batch = append(batch, baseOffsetBytes...)
+
+	// Batch length (4 bytes) - will be filled later
+	batchLengthPos := len(batch)
+	batch = append(batch, 0, 0, 0, 0)
+
+	// Partition leader epoch (4 bytes)
+	batch = append(batch, 0xFF, 0xFF, 0xFF, 0xFF)
+
+	// Magic byte (1 byte) - v2 format
+	batch = append(batch, 2)
+
+	// CRC placeholder (4 bytes)
+	crcPos := len(batch)
+	batch = append(batch, 0, 0, 0, 0)
+
+	// Attributes (2 bytes) - set compression bits
+	var compressionBits uint16
+	switch codec {
+	case compression.Gzip:
+		compressionBits = 1
+	case compression.Snappy:
+		compressionBits = 2
+	case compression.Lz4:
+		compressionBits = 3
+	case compression.Zstd:
+		compressionBits = 4
+	default:
+		compressionBits = 0 // no compression
+	}
+	batch = append(batch, byte(compressionBits>>8), byte(compressionBits))
+
+	// Last offset delta (4 bytes) - for compressed batches, this represents the logical record count
+	batch = append(batch, 0, 0, 0, 0) // Will be set based on logical records
+
+	// Timestamps (16 bytes) - use current time for compressed batches
+	timestamp := uint64(1640995200000)
+	timestampBytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(timestampBytes, timestamp)
+	batch = append(batch, timestampBytes...) // first timestamp
+	batch = append(batch, timestampBytes...) // max timestamp
+
+	// Producer fields (14 bytes total)
+	batch = append(batch, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF) // producer ID
+	batch = append(batch, 0xFF, 0xFF)                                     // producer epoch
+	batch = append(batch, 0xFF, 0xFF, 0xFF, 0xFF)                         // base sequence
+
+	// Record count (4 bytes) - for compressed batches, this is the number of logical records
+	batch = append(batch, 0, 0, 0, 1) // Placeholder: treat as 1 logical record
+
+	// Compressed records data
+	batch = append(batch, compressedRecords...)
+
+	// Fill in the batch length
+	batchLength := uint32(len(batch) - batchLengthPos - 4)
+	binary.BigEndian.PutUint32(batch[batchLengthPos:batchLengthPos+4], batchLength)
+
+	// Calculate CRC32 for the batch
+	// Per Kafka spec: CRC covers ONLY from attributes offset (byte 21) onwards
+	// See: DefaultRecordBatch.java computeChecksum() - Crc32C.compute(buffer, ATTRIBUTES_OFFSET, ...)
+	crcData := batch[crcPos+4:] // Skip CRC field itself, include rest
+	crc := crc32.Checksum(crcData, crc32.MakeTable(crc32.Castagnoli))
+	binary.BigEndian.PutUint32(batch[crcPos:crcPos+4], crc)
+
+	return batch
+}
+
+// compressData compresses data using the specified codec (basic implementation)
+func (f *MultiBatchFetcher) compressData(data []byte, codec compression.CompressionCodec) ([]byte, error) {
+	// For Phase 5, implement basic compression support
+	switch codec {
+	case compression.None:
+		return data, nil
+	case compression.Gzip:
+		// Implement actual GZIP compression
+		var buf bytes.Buffer
+		gzipWriter := gzip.NewWriter(&buf)
+
+		if _, err := gzipWriter.Write(data); err != nil {
+			gzipWriter.Close()
+			return nil, fmt.Errorf("gzip compression write failed: %w", err)
+		}
+
+		if err := gzipWriter.Close(); err != nil {
+			return nil, fmt.Errorf("gzip compression close failed: %w", err)
+		}
+
+		compressed := buf.Bytes()
+
+		return compressed, nil
+	default:
+		return nil, fmt.Errorf("unsupported compression codec: %d", codec)
+	}
+}

weed/mq/kafka/protocol/fetch_partition_reader.go

@@ -0,0 +1,316 @@
+package protocol
+
+import (
+	"context"
+	"math"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+// partitionReader maintains a persistent connection to a single topic-partition
+// and streams records forward, eliminating repeated offset lookups
+// Pre-fetches and buffers records for instant serving
+type partitionReader struct {
+	topicName     string
+	partitionID   int32
+	currentOffset int64
+	fetchChan     chan *partitionFetchRequest
+	closeChan     chan struct{}
+
+	// Pre-fetch buffer support
+	recordBuffer chan *bufferedRecords // Buffered pre-fetched records
+	bufferMu     sync.Mutex            // Protects offset access
+
+	handler *Handler
+	connCtx *ConnectionContext
+}
+
+// bufferedRecords represents a batch of pre-fetched records
+type bufferedRecords struct {
+	recordBatch []byte
+	startOffset int64
+	endOffset   int64
+}
+
+// partitionFetchRequest represents a request to fetch data from this partition
+type partitionFetchRequest struct {
+	requestedOffset int64
+	maxBytes        int32
+	maxWaitMs       int32 // MaxWaitTime from Kafka fetch request
+	resultChan      chan *partitionFetchResult
+	isSchematized   bool
+	apiVersion      uint16
+}
+
+// newPartitionReader creates and starts a new partition reader with pre-fetch buffering
+func newPartitionReader(ctx context.Context, handler *Handler, connCtx *ConnectionContext, topicName string, partitionID int32, startOffset int64) *partitionReader {
+	pr := &partitionReader{
+		topicName:     topicName,
+		partitionID:   partitionID,
+		currentOffset: startOffset,
+		fetchChan:     make(chan *partitionFetchRequest, 200), // Buffer 200 requests to handle Schema Registry's rapid polling in slow CI environments
+		closeChan:     make(chan struct{}),
+		recordBuffer:  make(chan *bufferedRecords, 5), // Buffer 5 batches of records
+		handler:       handler,
+		connCtx:       connCtx,
+	}
+
+	// Start the pre-fetch goroutine that continuously fetches ahead
+	go pr.preFetchLoop(ctx)
+
+	// Start the request handler goroutine
+	go pr.handleRequests(ctx)
+
+	glog.V(4).Infof("[%s] Created partition reader for %s[%d] starting at offset %d (sequential with ch=200)",
+		connCtx.ConnectionID, topicName, partitionID, startOffset)
+
+	return pr
+}
+
+// preFetchLoop is disabled for SMQ backend to prevent subscriber storms
+// SMQ reads from disk and creating multiple concurrent subscribers causes
+// broker overload and partition shutdowns. Fetch requests are handled
+// on-demand in serveFetchRequest instead.
+func (pr *partitionReader) preFetchLoop(ctx context.Context) {
+	defer func() {
+		glog.V(4).Infof("[%s] Pre-fetch loop exiting for %s[%d]",
+			pr.connCtx.ConnectionID, pr.topicName, pr.partitionID)
+		close(pr.recordBuffer)
+	}()
+
+	// Wait for shutdown - no continuous pre-fetching to avoid overwhelming the broker
+	select {
+	case <-ctx.Done():
+		return
+	case <-pr.closeChan:
+		return
+	}
+}
+
+// handleRequests serves fetch requests SEQUENTIALLY to prevent subscriber storm
+// Sequential processing is essential for SMQ backend because:
+// 1. GetStoredRecords may create a new subscriber on each call
+// 2. Concurrent calls create multiple subscribers for the same partition
+// 3. This overwhelms the broker and causes partition shutdowns
+func (pr *partitionReader) handleRequests(ctx context.Context) {
+	defer func() {
+		glog.V(4).Infof("[%s] Request handler exiting for %s[%d]",
+			pr.connCtx.ConnectionID, pr.topicName, pr.partitionID)
+	}()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-pr.closeChan:
+			return
+		case req := <-pr.fetchChan:
+			// Process sequentially to prevent subscriber storm
+			pr.serveFetchRequest(ctx, req)
+		}
+	}
+}
+
+// serveFetchRequest fetches data on-demand (no pre-fetching)
+func (pr *partitionReader) serveFetchRequest(ctx context.Context, req *partitionFetchRequest) {
+	startTime := time.Now()
+	result := &partitionFetchResult{}
+
+	defer func() {
+		result.fetchDuration = time.Since(startTime)
+
+		// Send result back to client
+		select {
+		case req.resultChan <- result:
+			// Successfully sent
+		case <-ctx.Done():
+			glog.Warningf("[%s] Context cancelled while sending result for %s[%d]",
+				pr.connCtx.ConnectionID, pr.topicName, pr.partitionID)
+		case <-time.After(50 * time.Millisecond):
+			glog.Warningf("[%s] Timeout sending result for %s[%d] - CLIENT MAY HAVE DISCONNECTED",
+				pr.connCtx.ConnectionID, pr.topicName, pr.partitionID)
+		}
+	}()
+
+	// Get high water mark
+	hwmUnknown := false
+	hwm, hwmErr := pr.handler.seaweedMQHandler.GetLatestOffset(pr.topicName, pr.partitionID)
+	if hwmErr != nil {
+		// HWM lookup can fail when the partition has been deactivated between consumer
+		// sessions. Proceed with the fetch anyway — the broker will return the correct
+		// data (or empty) based on its own state. Use math.MaxInt64 as sentinel so
+		// FetchMultipleBatches doesn't artificially cap recordsAvailable, and we
+		// don't hit the early-return below. The actual HWM will be derived from
+		// the fetch result (newOffset) after the read.
+		glog.Warningf("[%s] HWM lookup failed for %s[%d]: %v — will attempt fetch anyway",
+			pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, hwmErr)
+		hwm = math.MaxInt64
+		hwmUnknown = true
+	}
+	result.highWaterMark = hwm
+
+	glog.V(2).Infof("[%s] HWM for %s[%d]: %d (requested: %d)",
+		pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, hwm, req.requestedOffset)
+
+	// If requested offset >= HWM, return immediately with empty result
+	// This prevents overwhelming the broker with futile read attempts when no data is available
+	if req.requestedOffset >= hwm {
+		result.recordBatch = []byte{}
+		glog.V(3).Infof("[%s] Requested offset %d >= HWM %d, returning empty",
+			pr.connCtx.ConnectionID, req.requestedOffset, hwm)
+		return
+	}
+
+	// Update tracking offset to match requested offset
+	pr.bufferMu.Lock()
+	if req.requestedOffset != pr.currentOffset {
+		glog.V(3).Infof("[%s] Updating currentOffset for %s[%d]: %d -> %d",
+			pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, pr.currentOffset, req.requestedOffset)
+		pr.currentOffset = req.requestedOffset
+	}
+	pr.bufferMu.Unlock()
+
+	// Fetch on-demand - no pre-fetching to avoid overwhelming the broker
+	recordBatch, newOffset := pr.readRecords(ctx, req.requestedOffset, req.maxBytes, req.maxWaitMs, hwm)
+
+	// When HWM was unknown, derive a reasonable value from the fetch result
+	// so the client sees a meaningful high water mark instead of MaxInt64.
+	if hwmUnknown {
+		if newOffset > req.requestedOffset {
+			result.highWaterMark = newOffset // best estimate: end of what we read
+		} else {
+			result.highWaterMark = req.requestedOffset // no data found
+		}
+	}
+
+	// Log what we got back - DETAILED for diagnostics
+	if len(recordBatch) == 0 {
+		if req.requestedOffset < result.highWaterMark {
+			glog.Warningf("[%s] FETCH GAP for %s[%d]: group=%s member=%s requested offset=%d below HWM=%d but readRecords returned empty (maxWaitMs=%d)",
+				pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, pr.connCtx.ConsumerGroup, pr.connCtx.MemberID,
+				req.requestedOffset, result.highWaterMark, req.maxWaitMs)
+		} else {
+			glog.V(2).Infof("[%s] FETCH %s[%d]: readRecords returned EMPTY (offset=%d, hwm=%d)",
+				pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, req.requestedOffset, result.highWaterMark)
+		}
+		result.recordBatch = []byte{}
+	} else {
+		result.recordBatch = recordBatch
+		pr.bufferMu.Lock()
+		pr.currentOffset = newOffset
+		pr.bufferMu.Unlock()
+	}
+}
+
+// readRecords reads records forward using the multi-batch fetcher
+func (pr *partitionReader) readRecords(ctx context.Context, fromOffset int64, maxBytes int32, maxWaitMs int32, highWaterMark int64) ([]byte, int64) {
+	fetchStartTime := time.Now()
+
+	// Create context with timeout based on Kafka fetch request's MaxWaitTime
+	// This ensures we wait exactly as long as the client requested
+	fetchCtx := ctx
+	if maxWaitMs > 0 {
+		var cancel context.CancelFunc
+		// Use 1.5x the client timeout to account for internal processing overhead
+		// This prevents legitimate slow reads from being killed by client timeout
+		internalTimeoutMs := int32(float64(maxWaitMs) * 1.5)
+		if internalTimeoutMs > 5000 {
+			internalTimeoutMs = 5000 // Cap at 5 seconds
+		}
+		fetchCtx, cancel = context.WithTimeout(ctx, time.Duration(internalTimeoutMs)*time.Millisecond)
+		defer cancel()
+	}
+
+	// Use multi-batch fetcher for better MaxBytes compliance
+	multiFetcher := NewMultiBatchFetcher(pr.handler)
+	startTime := time.Now()
+	fetchResult, err := multiFetcher.FetchMultipleBatches(
+		fetchCtx,
+		pr.topicName,
+		pr.partitionID,
+		fromOffset,
+		highWaterMark,
+		maxBytes,
+	)
+	fetchDuration := time.Since(startTime)
+
+	// Log slow fetches (potential hangs)
+	if fetchDuration > 2*time.Second {
+		glog.Warningf("[%s] SLOW FETCH for %s[%d]: offset=%d took %.2fs (maxWait=%dms, HWM=%d)",
+			pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, fromOffset, fetchDuration.Seconds(), maxWaitMs, highWaterMark)
+	}
+
+	if err == nil && fetchResult.TotalSize > 0 {
+		glog.V(4).Infof("[%s] Multi-batch fetch for %s[%d]: %d batches, %d bytes, offset %d -> %d (duration: %v)",
+			pr.connCtx.ConnectionID, pr.topicName, pr.partitionID,
+			fetchResult.BatchCount, fetchResult.TotalSize, fromOffset, fetchResult.NextOffset, fetchDuration)
+		return fetchResult.RecordBatches, fetchResult.NextOffset
+	}
+
+	// Multi-batch failed - try single batch with a fresh timeout
+	glog.Warningf("[%s] Multi-batch fetch failed for %s[%d] offset=%d after %v, falling back to single-batch (err: %v)",
+		pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, fromOffset, fetchDuration, err)
+
+	// Compute the remaining time budget for the fallback. If the parent
+	// context carries a deadline, honour it; otherwise derive remaining
+	// from maxWaitMs minus elapsed time. This prevents the fallback from
+	// restarting the full budget after the multi-batch fetch already
+	// consumed part of it.
+	var remaining time.Duration
+	if deadline, ok := ctx.Deadline(); ok {
+		remaining = time.Until(deadline)
+	} else {
+		remaining = time.Duration(maxWaitMs)*time.Millisecond - time.Since(fetchStartTime)
+	}
+	if remaining <= 0 {
+		// Budget exhausted — skip the fallback entirely.
+		glog.V(2).Infof("[%s] No remaining budget for fallback on %s[%d] (maxWait=%dms, elapsed=%v)",
+			pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, maxWaitMs, time.Since(fetchStartTime))
+		return []byte{}, fromOffset
+	}
+	// Clamp: floor of 2s so disk reads via gRPC have a realistic chance,
+	// but never exceed 10s to bound data-plane blocking.
+	fallbackTimeout := remaining
+	if fallbackTimeout < 2*time.Second {
+		fallbackTimeout = 2 * time.Second
+	}
+	if fallbackTimeout > 10*time.Second {
+		fallbackTimeout = 10 * time.Second
+	}
+	fallbackCtx, fallbackCancel := context.WithTimeout(ctx, fallbackTimeout)
+	defer fallbackCancel()
+	fallbackStartTime := time.Now()
+	smqRecords, err := pr.handler.seaweedMQHandler.GetStoredRecords(fallbackCtx, pr.topicName, pr.partitionID, fromOffset, 10)
+	fallbackDuration := time.Since(fallbackStartTime)
+
+	if fallbackDuration > 2*time.Second {
+		glog.Warningf("[%s] SLOW FALLBACK for %s[%d]: offset=%d took %.2fs",
+			pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, fromOffset, fallbackDuration.Seconds())
+	}
+
+	if err != nil {
+		glog.Errorf("[%s] CRITICAL: Both multi-batch AND fallback failed for %s[%d] offset=%d: %v",
+			pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, fromOffset, err)
+		return []byte{}, fromOffset
+	}
+
+	if len(smqRecords) > 0 {
+		recordBatch := pr.handler.constructRecordBatchFromSMQ(pr.topicName, fromOffset, smqRecords)
+		nextOffset := fromOffset + int64(len(smqRecords))
+		glog.V(3).Infof("[%s] Fallback succeeded: got %d records for %s[%d] offset %d -> %d (total: %v)",
+			pr.connCtx.ConnectionID, len(smqRecords), pr.topicName, pr.partitionID, fromOffset, nextOffset, time.Since(fetchStartTime))
+		return recordBatch, nextOffset
+	}
+
+	// No records available
+	glog.V(3).Infof("[%s] No records available for %s[%d] offset=%d after multi-batch and fallback (total: %v)",
+		pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, fromOffset, time.Since(fetchStartTime))
+	return []byte{}, fromOffset
+}
+
+// close signals the reader to shut down
+func (pr *partitionReader) close() {
+	close(pr.closeChan)
+}

weed/mq/kafka/protocol/find_coordinator.go

@@ -0,0 +1,498 @@
+package protocol
+
+import (
+	"encoding/binary"
+	"fmt"
+	"net"
+	"strconv"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+// CoordinatorRegistryInterface defines the interface for coordinator registry operations
+type CoordinatorRegistryInterface interface {
+	IsLeader() bool
+	GetLeaderAddress() string
+	WaitForLeader(timeout time.Duration) (string, error)
+	AssignCoordinator(consumerGroup string, requestingGateway string) (*CoordinatorAssignment, error)
+	GetCoordinator(consumerGroup string) (*CoordinatorAssignment, error)
+}
+
+// CoordinatorAssignment represents a consumer group coordinator assignment
+type CoordinatorAssignment struct {
+	ConsumerGroup     string
+	CoordinatorAddr   string
+	CoordinatorNodeID int32
+	AssignedAt        time.Time
+	LastHeartbeat     time.Time
+}
+
+func (h *Handler) handleFindCoordinator(correlationID uint32, apiVersion uint16, requestBody []byte) ([]byte, error) {
+	glog.V(2).Infof("FindCoordinator: version=%d, correlation=%d, bodyLen=%d", apiVersion, correlationID, len(requestBody))
+	switch apiVersion {
+	case 0:
+		glog.V(4).Infof("FindCoordinator - Routing to V0 handler")
+		return h.handleFindCoordinatorV0(correlationID, requestBody)
+	case 1, 2:
+		glog.V(4).Infof("FindCoordinator - Routing to V1-2 handler (non-flexible)")
+		return h.handleFindCoordinatorV2(correlationID, requestBody)
+	case 3:
+		glog.V(4).Infof("FindCoordinator - Routing to V3 handler (flexible)")
+		return h.handleFindCoordinatorV3(correlationID, requestBody)
+	default:
+		return nil, fmt.Errorf("FindCoordinator version %d not supported", apiVersion)
+	}
+}
+
+func (h *Handler) handleFindCoordinatorV0(correlationID uint32, requestBody []byte) ([]byte, error) {
+	// Parse FindCoordinator v0 request: Key (STRING) only
+
+	if len(requestBody) < 2 { // need at least Key length
+		return nil, fmt.Errorf("FindCoordinator request too short")
+	}
+
+	offset := 0
+
+	if len(requestBody) < offset+2 { // coordinator_key_size(2)
+		return nil, fmt.Errorf("FindCoordinator request missing data (need %d bytes, have %d)", offset+2, len(requestBody))
+	}
+
+	// Parse coordinator key (group ID for consumer groups)
+	coordinatorKeySize := binary.BigEndian.Uint16(requestBody[offset : offset+2])
+	offset += 2
+
+	if len(requestBody) < offset+int(coordinatorKeySize) {
+		return nil, fmt.Errorf("FindCoordinator request missing coordinator key (need %d bytes, have %d)", offset+int(coordinatorKeySize), len(requestBody))
+	}
+
+	coordinatorKey := string(requestBody[offset : offset+int(coordinatorKeySize)])
+	offset += int(coordinatorKeySize)
+
+	// Parse coordinator type (v1+ only, default to 0 for consumer groups in v0)
+	_ = int8(0) // Consumer group coordinator (unused in v0)
+
+	// Find the appropriate coordinator for this group
+	coordinatorHost, coordinatorPort, nodeID, err := h.findCoordinatorForGroup(coordinatorKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find coordinator for group %s: %w", coordinatorKey, err)
+	}
+
+	// Return hostname instead of IP address for client connectivity
+	// Clients need to connect to the same hostname they originally connected to
+	_ = coordinatorHost // originalHost
+	coordinatorHost = h.getClientConnectableHost(coordinatorHost)
+
+	// Build response
+	response := make([]byte, 0, 64)
+
+	// NOTE: Correlation ID is handled by writeResponseWithHeader
+	// Do NOT include it in the response body
+
+	// FindCoordinator v0 Response Format (NO throttle_time_ms, NO error_message):
+	// - error_code (INT16)
+	// - node_id (INT32)
+	// - host (STRING)
+	// - port (INT32)
+
+	// Error code (2 bytes, 0 = no error)
+	response = append(response, 0, 0)
+
+	// Coordinator node_id (4 bytes) - use direct bit conversion for int32 to uint32
+	nodeIDBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(nodeIDBytes, uint32(int32(nodeID)))
+	response = append(response, nodeIDBytes...)
+
+	// Coordinator host (string)
+	hostLen := uint16(len(coordinatorHost))
+	response = append(response, byte(hostLen>>8), byte(hostLen))
+	response = append(response, []byte(coordinatorHost)...)
+
+	// Coordinator port (4 bytes) - validate port range
+	if coordinatorPort < 0 || coordinatorPort > 65535 {
+		return nil, fmt.Errorf("invalid port number: %d", coordinatorPort)
+	}
+	portBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(portBytes, uint32(coordinatorPort))
+	response = append(response, portBytes...)
+
+	return response, nil
+}
+
+func (h *Handler) handleFindCoordinatorV2(correlationID uint32, requestBody []byte) ([]byte, error) {
+	// Parse FindCoordinator request (v0-2 non-flex): Key (STRING), v1+ adds KeyType (INT8)
+
+	if len(requestBody) < 2 { // need at least Key length
+		return nil, fmt.Errorf("FindCoordinator request too short")
+	}
+
+	offset := 0
+
+	if len(requestBody) < offset+2 { // coordinator_key_size(2)
+		return nil, fmt.Errorf("FindCoordinator request missing data (need %d bytes, have %d)", offset+2, len(requestBody))
+	}
+
+	// Parse coordinator key (group ID for consumer groups)
+	coordinatorKeySize := binary.BigEndian.Uint16(requestBody[offset : offset+2])
+	offset += 2
+
+	if len(requestBody) < offset+int(coordinatorKeySize) {
+		return nil, fmt.Errorf("FindCoordinator request missing coordinator key (need %d bytes, have %d)", offset+int(coordinatorKeySize), len(requestBody))
+	}
+
+	coordinatorKey := string(requestBody[offset : offset+int(coordinatorKeySize)])
+	offset += int(coordinatorKeySize)
+
+	// Coordinator type present in v1+ (INT8). If absent, default 0.
+	if offset < len(requestBody) {
+		_ = requestBody[offset] // coordinatorType
+		offset++                // Move past the coordinator type byte
+	}
+
+	// Find the appropriate coordinator for this group
+	coordinatorHost, coordinatorPort, nodeID, err := h.findCoordinatorForGroup(coordinatorKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find coordinator for group %s: %w", coordinatorKey, err)
+	}
+
+	// Return hostname instead of IP address for client connectivity
+	// Clients need to connect to the same hostname they originally connected to
+	_ = coordinatorHost // originalHost
+	coordinatorHost = h.getClientConnectableHost(coordinatorHost)
+
+	response := make([]byte, 0, 64)
+
+	// NOTE: Correlation ID is handled by writeResponseWithHeader
+	// Do NOT include it in the response body
+
+	// FindCoordinator v2 Response Format:
+	// - throttle_time_ms (INT32)
+	// - error_code (INT16)
+	// - error_message (STRING) - nullable
+	// - node_id (INT32)
+	// - host (STRING)
+	// - port (INT32)
+
+	// Throttle time (4 bytes, 0 = no throttling)
+	response = append(response, 0, 0, 0, 0)
+
+	// Error code (2 bytes, 0 = no error)
+	response = append(response, 0, 0)
+
+	// Error message (nullable string) - null for success
+	response = append(response, 0xff, 0xff) // -1 length indicates null
+
+	// Coordinator node_id (4 bytes) - use direct bit conversion for int32 to uint32
+	nodeIDBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(nodeIDBytes, uint32(int32(nodeID)))
+	response = append(response, nodeIDBytes...)
+
+	// Coordinator host (string)
+	hostLen := uint16(len(coordinatorHost))
+	response = append(response, byte(hostLen>>8), byte(hostLen))
+	response = append(response, []byte(coordinatorHost)...)
+
+	// Coordinator port (4 bytes) - validate port range
+	if coordinatorPort < 0 || coordinatorPort > 65535 {
+		return nil, fmt.Errorf("invalid port number: %d", coordinatorPort)
+	}
+	portBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(portBytes, uint32(coordinatorPort))
+	response = append(response, portBytes...)
+
+	// Debug logging (hex dump removed to reduce CPU usage)
+	if glog.V(4) {
+		glog.V(4).Infof("FindCoordinator v2: Built response - bodyLen=%d, host='%s' (len=%d), port=%d, nodeID=%d",
+			len(response), coordinatorHost, len(coordinatorHost), coordinatorPort, nodeID)
+	}
+
+	return response, nil
+}
+
+func (h *Handler) handleFindCoordinatorV3(correlationID uint32, requestBody []byte) ([]byte, error) {
+	// Parse FindCoordinator v3 request (flexible version):
+	// - Key (COMPACT_STRING with varint length+1)
+	// - KeyType (INT8)
+	// - Tagged fields (varint)
+
+	if len(requestBody) < 2 {
+		return nil, fmt.Errorf("FindCoordinator v3 request too short")
+	}
+
+	// HEX DUMP for debugging
+	glog.V(4).Infof("FindCoordinator V3 request body (first 50 bytes): % x", requestBody[:min(50, len(requestBody))])
+	glog.V(4).Infof("FindCoordinator V3 request body length: %d", len(requestBody))
+
+	offset := 0
+
+	// The first byte is the tagged fields from the REQUEST HEADER that weren't consumed
+	// Skip the tagged fields count (should be 0x00 for no tagged fields)
+	if len(requestBody) > 0 && requestBody[0] == 0x00 {
+		glog.V(4).Infof("FindCoordinator V3: Skipping header tagged fields byte (0x00)")
+		offset = 1
+	}
+
+	// Parse coordinator key (compact string: varint length+1)
+	glog.V(4).Infof("FindCoordinator V3: About to decode varint from bytes: % x", requestBody[offset:min(offset+5, len(requestBody))])
+	coordinatorKeyLen, bytesRead, err := DecodeUvarint(requestBody[offset:])
+	if err != nil || bytesRead <= 0 {
+		return nil, fmt.Errorf("failed to decode coordinator key length: %w (bytes: % x)", err, requestBody[offset:min(offset+5, len(requestBody))])
+	}
+	offset += bytesRead
+
+	glog.V(4).Infof("FindCoordinator V3: coordinatorKeyLen (varint)=%d, bytesRead=%d, offset now=%d", coordinatorKeyLen, bytesRead, offset)
+	glog.V(4).Infof("FindCoordinator V3: Next bytes after varint: % x", requestBody[offset:min(offset+20, len(requestBody))])
+
+	if coordinatorKeyLen == 0 {
+		return nil, fmt.Errorf("coordinator key cannot be null in v3")
+	}
+	// Compact strings in Kafka use length+1 encoding:
+	// varint=0 means null, varint=1 means empty string, varint=n+1 means string of length n
+	coordinatorKeyLen-- // Decode: actual length = varint - 1
+
+	glog.V(4).Infof("FindCoordinator V3: actual coordinatorKeyLen after decoding: %d", coordinatorKeyLen)
+
+	if len(requestBody) < offset+int(coordinatorKeyLen) {
+		return nil, fmt.Errorf("FindCoordinator v3 request missing coordinator key")
+	}
+
+	coordinatorKey := string(requestBody[offset : offset+int(coordinatorKeyLen)])
+	offset += int(coordinatorKeyLen)
+
+	// Parse coordinator type (INT8)
+	if offset < len(requestBody) {
+		_ = requestBody[offset] // coordinatorType
+		offset++
+	}
+
+	// Skip tagged fields (we don't need them for now)
+	if offset < len(requestBody) {
+		_, bytesRead, tagErr := DecodeUvarint(requestBody[offset:])
+		if tagErr == nil && bytesRead > 0 {
+			offset += bytesRead
+			// TODO: Parse tagged fields if needed
+		}
+	}
+
+	// Find the appropriate coordinator for this group
+	coordinatorHost, coordinatorPort, nodeID, err := h.findCoordinatorForGroup(coordinatorKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find coordinator for group %s: %w", coordinatorKey, err)
+	}
+
+	// Return hostname instead of IP address for client connectivity
+	_ = coordinatorHost // originalHost
+	coordinatorHost = h.getClientConnectableHost(coordinatorHost)
+
+	// Build response (v3 is flexible, uses compact strings and tagged fields)
+	response := make([]byte, 0, 64)
+
+	// NOTE: Correlation ID is handled by writeResponseWithHeader
+	// Do NOT include it in the response body
+
+	// FindCoordinator v3 Response Format (FLEXIBLE):
+	// - throttle_time_ms (INT32)
+	// - error_code (INT16)
+	// - error_message (COMPACT_NULLABLE_STRING with varint length+1, 0 = null)
+	// - node_id (INT32)
+	// - host (COMPACT_STRING with varint length+1)
+	// - port (INT32)
+	// - tagged_fields (varint, 0 = no tags)
+
+	// Throttle time (4 bytes, 0 = no throttling)
+	response = append(response, 0, 0, 0, 0)
+
+	// Error code (2 bytes, 0 = no error)
+	response = append(response, 0, 0)
+
+	// Error message (compact nullable string) - null for success
+	// Compact nullable string: 0 = null, 1 = empty string, n+1 = string of length n
+	response = append(response, 0) // 0 = null
+
+	// Coordinator node_id (4 bytes) - use direct bit conversion for int32 to uint32
+	nodeIDBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(nodeIDBytes, uint32(int32(nodeID)))
+	response = append(response, nodeIDBytes...)
+
+	// Coordinator host (compact string: varint length+1)
+	hostLen := uint32(len(coordinatorHost))
+	response = append(response, EncodeUvarint(hostLen+1)...) // +1 for compact string encoding
+	response = append(response, []byte(coordinatorHost)...)
+
+	// Coordinator port (4 bytes) - validate port range
+	if coordinatorPort < 0 || coordinatorPort > 65535 {
+		return nil, fmt.Errorf("invalid port number: %d", coordinatorPort)
+	}
+	portBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(portBytes, uint32(coordinatorPort))
+	response = append(response, portBytes...)
+
+	// Tagged fields (0 = no tags)
+	response = append(response, 0)
+
+	return response, nil
+}
+
+// findCoordinatorForGroup determines the coordinator gateway for a consumer group
+// Uses gateway leader for distributed coordinator assignment (first-come-first-serve)
+func (h *Handler) findCoordinatorForGroup(groupID string) (host string, port int, nodeID int32, err error) {
+	// Get the coordinator registry from the handler
+	registry := h.GetCoordinatorRegistry()
+	if registry == nil {
+		// Fallback to current gateway if no registry available
+		gatewayAddr := h.GetGatewayAddress()
+		if gatewayAddr == "" {
+			return "", 0, 0, fmt.Errorf("no coordinator registry and no gateway address configured")
+		}
+		host, port, err := h.parseGatewayAddress(gatewayAddr)
+		if err != nil {
+			return "", 0, 0, fmt.Errorf("failed to parse gateway address: %w", err)
+		}
+		nodeID = 1
+		return host, port, nodeID, nil
+	}
+
+	// If this gateway is the leader, handle the assignment directly
+	if registry.IsLeader() {
+		return h.handleCoordinatorAssignmentAsLeader(groupID, registry)
+	}
+
+	// If not the leader, contact the leader to get/assign coordinator
+	// But first check if we can quickly become the leader or if there's already a leader
+	if leader := registry.GetLeaderAddress(); leader != "" {
+		// If the leader is this gateway, handle assignment directly
+		if leader == h.GetGatewayAddress() {
+			return h.handleCoordinatorAssignmentAsLeader(groupID, registry)
+		}
+	}
+	return h.requestCoordinatorFromLeader(groupID, registry)
+}
+
+// handleCoordinatorAssignmentAsLeader handles coordinator assignment when this gateway is the leader
+func (h *Handler) handleCoordinatorAssignmentAsLeader(groupID string, registry CoordinatorRegistryInterface) (host string, port int, nodeID int32, err error) {
+	// Check if coordinator already exists
+	if assignment, err := registry.GetCoordinator(groupID); err == nil && assignment != nil {
+		return h.parseAddress(assignment.CoordinatorAddr, assignment.CoordinatorNodeID)
+	}
+
+	// No coordinator exists, assign the requesting gateway (first-come-first-serve)
+	currentGateway := h.GetGatewayAddress()
+	if currentGateway == "" {
+		return "", 0, 0, fmt.Errorf("no gateway address configured for coordinator assignment")
+	}
+	assignment, err := registry.AssignCoordinator(groupID, currentGateway)
+	if err != nil {
+		// Fallback to current gateway on assignment error
+		host, port, parseErr := h.parseGatewayAddress(currentGateway)
+		if parseErr != nil {
+			return "", 0, 0, fmt.Errorf("failed to parse gateway address after assignment error: %w", parseErr)
+		}
+		nodeID = 1
+		return host, port, nodeID, nil
+	}
+
+	return h.parseAddress(assignment.CoordinatorAddr, assignment.CoordinatorNodeID)
+}
+
+// requestCoordinatorFromLeader requests coordinator assignment from the gateway leader
+// If no leader exists, it waits for leader election to complete
+func (h *Handler) requestCoordinatorFromLeader(groupID string, registry CoordinatorRegistryInterface) (host string, port int, nodeID int32, err error) {
+	// Wait for leader election to complete with a longer timeout for Schema Registry compatibility
+	_, err = h.waitForLeader(registry, 10*time.Second) // 10 second timeout for enterprise clients
+	if err != nil {
+		gatewayAddr := h.GetGatewayAddress()
+		if gatewayAddr == "" {
+			return "", 0, 0, fmt.Errorf("failed to wait for leader and no gateway address configured: %w", err)
+		}
+		host, port, parseErr := h.parseGatewayAddress(gatewayAddr)
+		if parseErr != nil {
+			return "", 0, 0, fmt.Errorf("failed to parse gateway address after leader wait timeout: %w", parseErr)
+		}
+		nodeID = 1
+		return host, port, nodeID, nil
+	}
+
+	// Since we don't have direct RPC between gateways yet, and the leader might be this gateway,
+	// check if we became the leader during the wait
+	if registry.IsLeader() {
+		return h.handleCoordinatorAssignmentAsLeader(groupID, registry)
+	}
+
+	// For now, if we can't directly contact the leader (no inter-gateway RPC yet),
+	// use current gateway as fallback. In a full implementation, this would make
+	// an RPC call to the leader gateway.
+	gatewayAddr := h.GetGatewayAddress()
+	if gatewayAddr == "" {
+		return "", 0, 0, fmt.Errorf("no gateway address configured for fallback coordinator")
+	}
+	host, port, parseErr := h.parseGatewayAddress(gatewayAddr)
+	if parseErr != nil {
+		return "", 0, 0, fmt.Errorf("failed to parse gateway address for fallback: %w", parseErr)
+	}
+	nodeID = 1
+	return host, port, nodeID, nil
+}
+
+// waitForLeader waits for a leader to be elected, with timeout
+func (h *Handler) waitForLeader(registry CoordinatorRegistryInterface, timeout time.Duration) (leaderAddress string, err error) {
+
+	// Use the registry's efficient wait mechanism
+	leaderAddress, err = registry.WaitForLeader(timeout)
+	if err != nil {
+		return "", err
+	}
+
+	return leaderAddress, nil
+}
+
+// parseGatewayAddress parses a gateway address string (host:port) into host and port
+func (h *Handler) parseGatewayAddress(address string) (host string, port int, err error) {
+	// Use net.SplitHostPort for proper IPv6 support
+	hostStr, portStr, err := net.SplitHostPort(address)
+	if err != nil {
+		return "", 0, fmt.Errorf("invalid gateway address format: %s", address)
+	}
+
+	port, err = strconv.Atoi(portStr)
+	if err != nil {
+		return "", 0, fmt.Errorf("invalid port in gateway address %s: %v", address, err)
+	}
+
+	return hostStr, port, nil
+}
+
+// parseAddress parses a gateway address and returns host, port, and nodeID
+func (h *Handler) parseAddress(address string, nodeID int32) (host string, port int, nid int32, err error) {
+	// Reuse the correct parseGatewayAddress implementation
+	host, port, err = h.parseGatewayAddress(address)
+	if err != nil {
+		return "", 0, 0, err
+	}
+	nid = nodeID
+	return host, port, nid, nil
+}
+
+// getClientConnectableHost returns the hostname that clients can connect to
+// This ensures that FindCoordinator returns the same hostname the client originally connected to
+func (h *Handler) getClientConnectableHost(coordinatorHost string) string {
+	// If the coordinator host is an IP address, return the original gateway hostname
+	// This prevents clients from switching to IP addresses which creates new connections
+	if net.ParseIP(coordinatorHost) != nil {
+		// It's an IP address, return the original gateway hostname
+		gatewayAddr := h.GetGatewayAddress()
+		if host, _, err := h.parseGatewayAddress(gatewayAddr); err == nil {
+			// If the gateway address is also an IP, return the IP directly
+			// This handles local/test environments where hostnames aren't resolvable
+			if net.ParseIP(host) != nil {
+				// Both are IPs, return the actual IP address
+				return coordinatorHost
+			}
+			return host
+		}
+		// Fallback to the coordinator host IP itself
+		return coordinatorHost
+	}
+
+	// It's already a hostname, return as-is
+	return coordinatorHost
+}

weed/mq/kafka/protocol/flexible_versions.go

@@ -0,0 +1,479 @@
+package protocol
+
+import (
+	"encoding/binary"
+	"fmt"
+)
+
+// FlexibleVersions provides utilities for handling Kafka flexible versions protocol
+// Flexible versions use compact arrays/strings and tagged fields for backward compatibility
+
+// CompactArrayLength encodes a length for compact arrays
+// Compact arrays encode length as length+1, where 0 means empty array
+func CompactArrayLength(length uint32) []byte {
+	// Compact arrays use length+1 encoding (0 = null, 1 = empty, n+1 = array of length n)
+	// For an empty array (length=0), we return 1 (not 0, which would be null)
+	return EncodeUvarint(length + 1)
+}
+
+// DecodeCompactArrayLength decodes a compact array length
+// Returns the actual length and number of bytes consumed
+func DecodeCompactArrayLength(data []byte) (uint32, int, error) {
+	if len(data) == 0 {
+		return 0, 0, fmt.Errorf("no data for compact array length")
+	}
+
+	if data[0] == 0 {
+		return 0, 1, nil // Empty array
+	}
+
+	length, consumed, err := DecodeUvarint(data)
+	if err != nil {
+		return 0, 0, fmt.Errorf("decode compact array length: %w", err)
+	}
+
+	if length == 0 {
+		return 0, consumed, fmt.Errorf("invalid compact array length encoding")
+	}
+
+	return length - 1, consumed, nil
+}
+
+// CompactStringLength encodes a length for compact strings
+// Compact strings encode length as length+1, where 0 means null string
+func CompactStringLength(length int) []byte {
+	if length < 0 {
+		return []byte{0} // Null string
+	}
+	return EncodeUvarint(uint32(length + 1))
+}
+
+// DecodeCompactStringLength decodes a compact string length
+// Returns the actual length (-1 for null), and number of bytes consumed
+func DecodeCompactStringLength(data []byte) (int, int, error) {
+	if len(data) == 0 {
+		return 0, 0, fmt.Errorf("no data for compact string length")
+	}
+
+	if data[0] == 0 {
+		return -1, 1, nil // Null string
+	}
+
+	length, consumed, err := DecodeUvarint(data)
+	if err != nil {
+		return 0, 0, fmt.Errorf("decode compact string length: %w", err)
+	}
+
+	if length == 0 {
+		return 0, consumed, fmt.Errorf("invalid compact string length encoding")
+	}
+
+	return int(length - 1), consumed, nil
+}
+
+// EncodeUvarint encodes an unsigned integer using variable-length encoding
+// This is used for compact arrays, strings, and tagged fields
+func EncodeUvarint(value uint32) []byte {
+	var buf []byte
+	for value >= 0x80 {
+		buf = append(buf, byte(value)|0x80)
+		value >>= 7
+	}
+	buf = append(buf, byte(value))
+	return buf
+}
+
+// DecodeUvarint decodes a variable-length unsigned integer
+// Returns the decoded value and number of bytes consumed
+func DecodeUvarint(data []byte) (uint32, int, error) {
+	var value uint32
+	var shift uint
+	var consumed int
+
+	for i, b := range data {
+		consumed = i + 1
+		value |= uint32(b&0x7F) << shift
+
+		if (b & 0x80) == 0 {
+			return value, consumed, nil
+		}
+
+		shift += 7
+		if shift >= 32 {
+			return 0, consumed, fmt.Errorf("uvarint overflow")
+		}
+	}
+
+	return 0, consumed, fmt.Errorf("incomplete uvarint")
+}
+
+// TaggedField represents a tagged field in flexible versions
+type TaggedField struct {
+	Tag  uint32
+	Data []byte
+}
+
+// TaggedFields represents a collection of tagged fields
+type TaggedFields struct {
+	Fields []TaggedField
+}
+
+// EncodeTaggedFields encodes tagged fields for flexible versions
+func (tf *TaggedFields) Encode() []byte {
+	if len(tf.Fields) == 0 {
+		return []byte{0} // Empty tagged fields
+	}
+
+	var buf []byte
+
+	// Number of tagged fields
+	buf = append(buf, EncodeUvarint(uint32(len(tf.Fields)))...)
+
+	for _, field := range tf.Fields {
+		// Tag
+		buf = append(buf, EncodeUvarint(field.Tag)...)
+		// Size
+		buf = append(buf, EncodeUvarint(uint32(len(field.Data)))...)
+		// Data
+		buf = append(buf, field.Data...)
+	}
+
+	return buf
+}
+
+// DecodeTaggedFields decodes tagged fields from flexible versions
+func DecodeTaggedFields(data []byte) (*TaggedFields, int, error) {
+	if len(data) == 0 {
+		return &TaggedFields{}, 0, fmt.Errorf("no data for tagged fields")
+	}
+
+	if data[0] == 0 {
+		return &TaggedFields{}, 1, nil // Empty tagged fields
+	}
+
+	offset := 0
+
+	// Number of tagged fields
+	numFields, consumed, err := DecodeUvarint(data[offset:])
+	if err != nil {
+		return nil, 0, fmt.Errorf("decode tagged fields count: %w", err)
+	}
+	offset += consumed
+
+	fields := make([]TaggedField, numFields)
+
+	for i := uint32(0); i < numFields; i++ {
+		// Tag
+		tag, consumed, err := DecodeUvarint(data[offset:])
+		if err != nil {
+			return nil, 0, fmt.Errorf("decode tagged field %d tag: %w", i, err)
+		}
+		offset += consumed
+
+		// Size
+		size, consumed, err := DecodeUvarint(data[offset:])
+		if err != nil {
+			return nil, 0, fmt.Errorf("decode tagged field %d size: %w", i, err)
+		}
+		offset += consumed
+
+		// Data
+		if offset+int(size) > len(data) {
+			// More detailed error information
+			return nil, 0, fmt.Errorf("tagged field %d data truncated: need %d bytes at offset %d, but only %d total bytes available", i, size, offset, len(data))
+		}
+
+		fields[i] = TaggedField{
+			Tag:  tag,
+			Data: data[offset : offset+int(size)],
+		}
+		offset += int(size)
+	}
+
+	return &TaggedFields{Fields: fields}, offset, nil
+}
+
+// IsFlexibleVersion determines if an API version uses flexible versions
+// This is API-specific and based on when each API adopted flexible versions
+func IsFlexibleVersion(apiKey, apiVersion uint16) bool {
+	switch APIKey(apiKey) {
+	case APIKeyApiVersions:
+		return apiVersion >= 3
+	case APIKeyMetadata:
+		return apiVersion >= 9
+	case APIKeyFetch:
+		return apiVersion >= 12
+	case APIKeyProduce:
+		return apiVersion >= 9
+	case APIKeyJoinGroup:
+		return apiVersion >= 6
+	case APIKeySyncGroup:
+		return apiVersion >= 4
+	case APIKeyOffsetCommit:
+		return apiVersion >= 8
+	case APIKeyOffsetFetch:
+		return apiVersion >= 6
+	case APIKeyFindCoordinator:
+		return apiVersion >= 3
+	case APIKeyHeartbeat:
+		return apiVersion >= 4
+	case APIKeyLeaveGroup:
+		return apiVersion >= 4
+	case APIKeyCreateTopics:
+		return apiVersion >= 2
+	case APIKeyDeleteTopics:
+		return apiVersion >= 4
+	default:
+		return false
+	}
+}
+
+// FlexibleString encodes a string for flexible versions (compact format)
+func FlexibleString(s string) []byte {
+	// Compact strings use length+1 encoding (0 = null, 1 = empty, n+1 = string of length n)
+	// For an empty string (s=""), we return length+1 = 1 (not 0, which would be null)
+	var buf []byte
+	buf = append(buf, CompactStringLength(len(s))...)
+	buf = append(buf, []byte(s)...)
+	return buf
+}
+
+// parseCompactString parses a compact string from flexible protocol
+// Returns the string bytes and the number of bytes consumed
+func parseCompactString(data []byte) ([]byte, int) {
+	if len(data) == 0 {
+		return nil, 0
+	}
+
+	// Parse compact string length (unsigned varint - no zigzag decoding!)
+	length, consumed := decodeUnsignedVarint(data)
+	if consumed == 0 {
+		return nil, 0
+	}
+
+	// Debug logging for compact string parsing
+
+	if length == 0 {
+		// Null string (length 0 means null)
+		return nil, consumed
+	}
+
+	// In compact strings, length is actual length + 1
+	// So length 1 means empty string, length > 1 means non-empty
+	if length == 0 {
+		return nil, consumed // Already handled above
+	}
+	actualLength := int(length - 1)
+	if actualLength < 0 {
+		return nil, 0
+	}
+
+	if actualLength == 0 {
+		// Empty string (length was 1)
+		return []byte{}, consumed
+	}
+
+	if consumed+actualLength > len(data) {
+		return nil, 0
+	}
+
+	result := data[consumed : consumed+actualLength]
+	return result, consumed + actualLength
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+// decodeUnsignedVarint decodes an unsigned varint (no zigzag decoding)
+func decodeUnsignedVarint(data []byte) (uint64, int) {
+	if len(data) == 0 {
+		return 0, 0
+	}
+
+	var result uint64
+	var shift uint
+	var bytesRead int
+
+	for i, b := range data {
+		if i > 9 { // varints can be at most 10 bytes
+			return 0, 0 // invalid varint
+		}
+
+		bytesRead++
+		result |= uint64(b&0x7F) << shift
+
+		if (b & 0x80) == 0 {
+			// Most significant bit is 0, we're done
+			return result, bytesRead
+		}
+
+		shift += 7
+	}
+
+	return 0, 0 // incomplete varint
+}
+
+// FlexibleNullableString encodes a nullable string for flexible versions
+func FlexibleNullableString(s *string) []byte {
+	if s == nil {
+		return []byte{0} // Null string
+	}
+	return FlexibleString(*s)
+}
+
+// DecodeFlexibleString decodes a flexible string
+// Returns the string (empty for null) and bytes consumed
+func DecodeFlexibleString(data []byte) (string, int, error) {
+	length, consumed, err := DecodeCompactStringLength(data)
+	if err != nil {
+		return "", 0, err
+	}
+
+	if length < 0 {
+		return "", consumed, nil // Null string -> empty string
+	}
+
+	if consumed+length > len(data) {
+		return "", 0, fmt.Errorf("string data truncated")
+	}
+
+	return string(data[consumed : consumed+length]), consumed + length, nil
+}
+
+// FlexibleVersionHeader handles the request header parsing for flexible versions
+type FlexibleVersionHeader struct {
+	APIKey        uint16
+	APIVersion    uint16
+	CorrelationID uint32
+	ClientID      *string
+	TaggedFields  *TaggedFields
+}
+
+// parseRegularHeader parses a regular (non-flexible) Kafka request header
+func parseRegularHeader(data []byte) (*FlexibleVersionHeader, []byte, error) {
+	if len(data) < 8 {
+		return nil, nil, fmt.Errorf("header too short")
+	}
+
+	header := &FlexibleVersionHeader{}
+	offset := 0
+
+	// API Key (2 bytes)
+	header.APIKey = binary.BigEndian.Uint16(data[offset : offset+2])
+	offset += 2
+
+	// API Version (2 bytes)
+	header.APIVersion = binary.BigEndian.Uint16(data[offset : offset+2])
+	offset += 2
+
+	// Correlation ID (4 bytes)
+	header.CorrelationID = binary.BigEndian.Uint32(data[offset : offset+4])
+	offset += 4
+
+	// Regular versions use standard strings
+	if len(data) < offset+2 {
+		return nil, nil, fmt.Errorf("missing client_id length")
+	}
+
+	clientIDLen := int16(binary.BigEndian.Uint16(data[offset : offset+2]))
+	offset += 2
+
+	if clientIDLen >= 0 {
+		if len(data) < offset+int(clientIDLen) {
+			return nil, nil, fmt.Errorf("client_id truncated")
+		}
+		clientID := string(data[offset : offset+int(clientIDLen)])
+		header.ClientID = &clientID
+		offset += int(clientIDLen)
+	}
+
+	return header, data[offset:], nil
+}
+
+// ParseRequestHeader parses a Kafka request header, handling both regular and flexible versions
+func ParseRequestHeader(data []byte) (*FlexibleVersionHeader, []byte, error) {
+	if len(data) < 8 {
+		return nil, nil, fmt.Errorf("header too short")
+	}
+
+	header := &FlexibleVersionHeader{}
+	offset := 0
+
+	// API Key (2 bytes)
+	header.APIKey = binary.BigEndian.Uint16(data[offset : offset+2])
+	offset += 2
+
+	// API Version (2 bytes)
+	header.APIVersion = binary.BigEndian.Uint16(data[offset : offset+2])
+	offset += 2
+
+	// Correlation ID (4 bytes)
+	header.CorrelationID = binary.BigEndian.Uint32(data[offset : offset+4])
+	offset += 4
+
+	// Client ID handling depends on flexible version
+	isFlexible := IsFlexibleVersion(header.APIKey, header.APIVersion)
+
+	if isFlexible {
+		// Flexible versions use compact strings
+		clientID, consumed, err := DecodeFlexibleString(data[offset:])
+		if err != nil {
+			return nil, nil, fmt.Errorf("decode flexible client_id: %w", err)
+		}
+		offset += consumed
+
+		if clientID != "" {
+			header.ClientID = &clientID
+		}
+
+		// Parse tagged fields in header
+		taggedFields, consumed, err := DecodeTaggedFields(data[offset:])
+		if err != nil {
+			// If tagged fields parsing fails, this might be a regular header sent by kafka-go
+			// Fall back to regular header parsing
+			return parseRegularHeader(data)
+		}
+		offset += consumed
+		header.TaggedFields = taggedFields
+
+	} else {
+		// Regular versions use standard strings
+		if len(data) < offset+2 {
+			return nil, nil, fmt.Errorf("missing client_id length")
+		}
+
+		clientIDLen := int16(binary.BigEndian.Uint16(data[offset : offset+2]))
+		offset += 2
+
+		if clientIDLen >= 0 {
+			if len(data) < offset+int(clientIDLen) {
+				return nil, nil, fmt.Errorf("client_id truncated")
+			}
+
+			clientID := string(data[offset : offset+int(clientIDLen)])
+			header.ClientID = &clientID
+			offset += int(clientIDLen)
+		}
+		// No tagged fields in regular versions
+	}
+
+	return header, data[offset:], nil
+}
+
+// EncodeFlexibleResponse encodes a response with proper flexible version formatting
+func EncodeFlexibleResponse(correlationID uint32, data []byte, hasTaggedFields bool) []byte {
+	response := make([]byte, 4)
+	binary.BigEndian.PutUint32(response, correlationID)
+	response = append(response, data...)
+
+	if hasTaggedFields {
+		// Add empty tagged fields for flexible responses
+		response = append(response, 0)
+	}
+
+	return response
+}

weed/mq/kafka/protocol/group_introspection.go

@@ -0,0 +1,447 @@
+package protocol
+
+import (
+	"encoding/binary"
+	"fmt"
+)
+
+// handleDescribeGroups handles DescribeGroups API (key 15)
+func (h *Handler) handleDescribeGroups(correlationID uint32, apiVersion uint16, requestBody []byte) ([]byte, error) {
+
+	// Parse request
+	request, err := h.parseDescribeGroupsRequest(requestBody, apiVersion)
+	if err != nil {
+		return nil, fmt.Errorf("parse DescribeGroups request: %w", err)
+	}
+
+	// Build response
+	response := DescribeGroupsResponse{
+		ThrottleTimeMs: 0,
+		Groups:         make([]DescribeGroupsGroup, 0, len(request.GroupIDs)),
+	}
+
+	// Get group information for each requested group
+	for _, groupID := range request.GroupIDs {
+		group := h.describeGroup(groupID)
+		response.Groups = append(response.Groups, group)
+	}
+
+	return h.buildDescribeGroupsResponse(response, correlationID, apiVersion), nil
+}
+
+// handleListGroups handles ListGroups API (key 16)
+func (h *Handler) handleListGroups(correlationID uint32, apiVersion uint16, requestBody []byte) ([]byte, error) {
+
+	// Parse request (ListGroups has minimal request structure)
+	request, err := h.parseListGroupsRequest(requestBody, apiVersion)
+	if err != nil {
+		return nil, fmt.Errorf("parse ListGroups request: %w", err)
+	}
+
+	// Build response
+	response := ListGroupsResponse{
+		ThrottleTimeMs: 0,
+		ErrorCode:      0,
+		Groups:         h.listAllGroups(request.StatesFilter),
+	}
+
+	return h.buildListGroupsResponse(response, correlationID, apiVersion), nil
+}
+
+// describeGroup gets detailed information about a specific group
+func (h *Handler) describeGroup(groupID string) DescribeGroupsGroup {
+	// Get group information from coordinator
+	if h.groupCoordinator == nil {
+		return DescribeGroupsGroup{
+			ErrorCode: 15, // GROUP_COORDINATOR_NOT_AVAILABLE
+			GroupID:   groupID,
+			State:     "Dead",
+		}
+	}
+
+	group := h.groupCoordinator.GetGroup(groupID)
+	if group == nil {
+		return DescribeGroupsGroup{
+			ErrorCode:    25, // UNKNOWN_GROUP_ID
+			GroupID:      groupID,
+			State:        "Dead",
+			ProtocolType: "",
+			Protocol:     "",
+			Members:      []DescribeGroupsMember{},
+		}
+	}
+
+	// Convert group to response format
+	members := make([]DescribeGroupsMember, 0, len(group.Members))
+	for memberID, member := range group.Members {
+		// Convert assignment to bytes (simplified)
+		var assignmentBytes []byte
+		if len(member.Assignment) > 0 {
+			// In a real implementation, this would serialize the assignment properly
+			assignmentBytes = []byte(fmt.Sprintf("assignment:%d", len(member.Assignment)))
+		}
+
+		members = append(members, DescribeGroupsMember{
+			MemberID:         memberID,
+			GroupInstanceID:  member.GroupInstanceID, // Now supports static membership
+			ClientID:         member.ClientID,
+			ClientHost:       member.ClientHost,
+			MemberMetadata:   member.Metadata,
+			MemberAssignment: assignmentBytes,
+		})
+	}
+
+	// Convert group state to string
+	var stateStr string
+	switch group.State {
+	case 0: // Assuming 0 is Empty
+		stateStr = "Empty"
+	case 1: // Assuming 1 is PreparingRebalance
+		stateStr = "PreparingRebalance"
+	case 2: // Assuming 2 is CompletingRebalance
+		stateStr = "CompletingRebalance"
+	case 3: // Assuming 3 is Stable
+		stateStr = "Stable"
+	default:
+		stateStr = "Dead"
+	}
+
+	return DescribeGroupsGroup{
+		ErrorCode:     0,
+		GroupID:       groupID,
+		State:         stateStr,
+		ProtocolType:  "consumer", // Default protocol type
+		Protocol:      group.Protocol,
+		Members:       members,
+		AuthorizedOps: []int32{}, // Empty for now
+	}
+}
+
+// listAllGroups gets a list of all consumer groups
+func (h *Handler) listAllGroups(statesFilter []string) []ListGroupsGroup {
+	if h.groupCoordinator == nil {
+		return []ListGroupsGroup{}
+	}
+
+	allGroupIDs := h.groupCoordinator.ListGroups()
+	groups := make([]ListGroupsGroup, 0, len(allGroupIDs))
+
+	for _, groupID := range allGroupIDs {
+		// Get the full group details
+		group := h.groupCoordinator.GetGroup(groupID)
+		if group == nil {
+			continue
+		}
+
+		// Convert group state to string
+		var stateStr string
+		switch group.State {
+		case 0:
+			stateStr = "Empty"
+		case 1:
+			stateStr = "PreparingRebalance"
+		case 2:
+			stateStr = "CompletingRebalance"
+		case 3:
+			stateStr = "Stable"
+		default:
+			stateStr = "Dead"
+		}
+
+		// Apply state filter if provided
+		if len(statesFilter) > 0 {
+			matchesFilter := false
+			for _, state := range statesFilter {
+				if stateStr == state {
+					matchesFilter = true
+					break
+				}
+			}
+			if !matchesFilter {
+				continue
+			}
+		}
+
+		groups = append(groups, ListGroupsGroup{
+			GroupID:      group.ID,
+			ProtocolType: "consumer", // Default protocol type
+			GroupState:   stateStr,
+		})
+	}
+
+	return groups
+}
+
+// Request/Response structures
+
+type DescribeGroupsRequest struct {
+	GroupIDs             []string
+	IncludeAuthorizedOps bool
+}
+
+type DescribeGroupsResponse struct {
+	ThrottleTimeMs int32
+	Groups         []DescribeGroupsGroup
+}
+
+type DescribeGroupsGroup struct {
+	ErrorCode     int16
+	GroupID       string
+	State         string
+	ProtocolType  string
+	Protocol      string
+	Members       []DescribeGroupsMember
+	AuthorizedOps []int32
+}
+
+type DescribeGroupsMember struct {
+	MemberID         string
+	GroupInstanceID  *string
+	ClientID         string
+	ClientHost       string
+	MemberMetadata   []byte
+	MemberAssignment []byte
+}
+
+type ListGroupsRequest struct {
+	StatesFilter []string
+}
+
+type ListGroupsResponse struct {
+	ThrottleTimeMs int32
+	ErrorCode      int16
+	Groups         []ListGroupsGroup
+}
+
+type ListGroupsGroup struct {
+	GroupID      string
+	ProtocolType string
+	GroupState   string
+}
+
+// Parsing functions
+
+func (h *Handler) parseDescribeGroupsRequest(data []byte, apiVersion uint16) (*DescribeGroupsRequest, error) {
+	offset := 0
+	request := &DescribeGroupsRequest{}
+
+	// Skip client_id if present (depends on version)
+	if len(data) < 4 {
+		return nil, fmt.Errorf("request too short")
+	}
+
+	// Group IDs array
+	groupCount := binary.BigEndian.Uint32(data[offset : offset+4])
+	offset += 4
+
+	request.GroupIDs = make([]string, groupCount)
+	for i := uint32(0); i < groupCount; i++ {
+		if offset+2 > len(data) {
+			return nil, fmt.Errorf("invalid group ID at index %d", i)
+		}
+
+		groupIDLen := binary.BigEndian.Uint16(data[offset : offset+2])
+		offset += 2
+
+		if offset+int(groupIDLen) > len(data) {
+			return nil, fmt.Errorf("group ID too long at index %d", i)
+		}
+
+		request.GroupIDs[i] = string(data[offset : offset+int(groupIDLen)])
+		offset += int(groupIDLen)
+	}
+
+	// Include authorized operations (v3+)
+	if apiVersion >= 3 && offset < len(data) {
+		request.IncludeAuthorizedOps = data[offset] != 0
+	}
+
+	return request, nil
+}
+
+func (h *Handler) parseListGroupsRequest(data []byte, apiVersion uint16) (*ListGroupsRequest, error) {
+	request := &ListGroupsRequest{}
+
+	// ListGroups v4+ includes states filter
+	if apiVersion >= 4 && len(data) >= 4 {
+		offset := 0
+		statesCount := binary.BigEndian.Uint32(data[offset : offset+4])
+		offset += 4
+
+		if statesCount > 0 {
+			request.StatesFilter = make([]string, statesCount)
+			for i := uint32(0); i < statesCount; i++ {
+				if offset+2 > len(data) {
+					break
+				}
+
+				stateLen := binary.BigEndian.Uint16(data[offset : offset+2])
+				offset += 2
+
+				if offset+int(stateLen) > len(data) {
+					break
+				}
+
+				request.StatesFilter[i] = string(data[offset : offset+int(stateLen)])
+				offset += int(stateLen)
+			}
+		}
+	}
+
+	return request, nil
+}
+
+// Response building functions
+
+func (h *Handler) buildDescribeGroupsResponse(response DescribeGroupsResponse, correlationID uint32, apiVersion uint16) []byte {
+	buf := make([]byte, 0, 1024)
+
+	// Correlation ID
+	correlationIDBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(correlationIDBytes, correlationID)
+	buf = append(buf, correlationIDBytes...)
+
+	// Throttle time (v1+)
+	if apiVersion >= 1 {
+		throttleBytes := make([]byte, 4)
+		binary.BigEndian.PutUint32(throttleBytes, uint32(response.ThrottleTimeMs))
+		buf = append(buf, throttleBytes...)
+	}
+
+	// Groups array
+	groupCountBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(groupCountBytes, uint32(len(response.Groups)))
+	buf = append(buf, groupCountBytes...)
+
+	for _, group := range response.Groups {
+		// Error code
+		buf = append(buf, byte(group.ErrorCode>>8), byte(group.ErrorCode))
+
+		// Group ID
+		groupIDLen := uint16(len(group.GroupID))
+		buf = append(buf, byte(groupIDLen>>8), byte(groupIDLen))
+		buf = append(buf, []byte(group.GroupID)...)
+
+		// State
+		stateLen := uint16(len(group.State))
+		buf = append(buf, byte(stateLen>>8), byte(stateLen))
+		buf = append(buf, []byte(group.State)...)
+
+		// Protocol type
+		protocolTypeLen := uint16(len(group.ProtocolType))
+		buf = append(buf, byte(protocolTypeLen>>8), byte(protocolTypeLen))
+		buf = append(buf, []byte(group.ProtocolType)...)
+
+		// Protocol
+		protocolLen := uint16(len(group.Protocol))
+		buf = append(buf, byte(protocolLen>>8), byte(protocolLen))
+		buf = append(buf, []byte(group.Protocol)...)
+
+		// Members array
+		memberCountBytes := make([]byte, 4)
+		binary.BigEndian.PutUint32(memberCountBytes, uint32(len(group.Members)))
+		buf = append(buf, memberCountBytes...)
+
+		for _, member := range group.Members {
+			// Member ID
+			memberIDLen := uint16(len(member.MemberID))
+			buf = append(buf, byte(memberIDLen>>8), byte(memberIDLen))
+			buf = append(buf, []byte(member.MemberID)...)
+
+			// Group instance ID (v4+, nullable)
+			if apiVersion >= 4 {
+				if member.GroupInstanceID != nil {
+					instanceIDLen := uint16(len(*member.GroupInstanceID))
+					buf = append(buf, byte(instanceIDLen>>8), byte(instanceIDLen))
+					buf = append(buf, []byte(*member.GroupInstanceID)...)
+				} else {
+					buf = append(buf, 0xFF, 0xFF) // null
+				}
+			}
+
+			// Client ID
+			clientIDLen := uint16(len(member.ClientID))
+			buf = append(buf, byte(clientIDLen>>8), byte(clientIDLen))
+			buf = append(buf, []byte(member.ClientID)...)
+
+			// Client host
+			clientHostLen := uint16(len(member.ClientHost))
+			buf = append(buf, byte(clientHostLen>>8), byte(clientHostLen))
+			buf = append(buf, []byte(member.ClientHost)...)
+
+			// Member metadata
+			metadataLen := uint32(len(member.MemberMetadata))
+			metadataLenBytes := make([]byte, 4)
+			binary.BigEndian.PutUint32(metadataLenBytes, metadataLen)
+			buf = append(buf, metadataLenBytes...)
+			buf = append(buf, member.MemberMetadata...)
+
+			// Member assignment
+			assignmentLen := uint32(len(member.MemberAssignment))
+			assignmentLenBytes := make([]byte, 4)
+			binary.BigEndian.PutUint32(assignmentLenBytes, assignmentLen)
+			buf = append(buf, assignmentLenBytes...)
+			buf = append(buf, member.MemberAssignment...)
+		}
+
+		// Authorized operations (v3+)
+		if apiVersion >= 3 {
+			opsCountBytes := make([]byte, 4)
+			binary.BigEndian.PutUint32(opsCountBytes, uint32(len(group.AuthorizedOps)))
+			buf = append(buf, opsCountBytes...)
+
+			for _, op := range group.AuthorizedOps {
+				opBytes := make([]byte, 4)
+				binary.BigEndian.PutUint32(opBytes, uint32(op))
+				buf = append(buf, opBytes...)
+			}
+		}
+	}
+
+	return buf
+}
+
+func (h *Handler) buildListGroupsResponse(response ListGroupsResponse, correlationID uint32, apiVersion uint16) []byte {
+	buf := make([]byte, 0, 512)
+
+	// Correlation ID
+	correlationIDBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(correlationIDBytes, correlationID)
+	buf = append(buf, correlationIDBytes...)
+
+	// Throttle time (v1+)
+	if apiVersion >= 1 {
+		throttleBytes := make([]byte, 4)
+		binary.BigEndian.PutUint32(throttleBytes, uint32(response.ThrottleTimeMs))
+		buf = append(buf, throttleBytes...)
+	}
+
+	// Error code
+	buf = append(buf, byte(response.ErrorCode>>8), byte(response.ErrorCode))
+
+	// Groups array
+	groupCountBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(groupCountBytes, uint32(len(response.Groups)))
+	buf = append(buf, groupCountBytes...)
+
+	for _, group := range response.Groups {
+		// Group ID
+		groupIDLen := uint16(len(group.GroupID))
+		buf = append(buf, byte(groupIDLen>>8), byte(groupIDLen))
+		buf = append(buf, []byte(group.GroupID)...)
+
+		// Protocol type
+		protocolTypeLen := uint16(len(group.ProtocolType))
+		buf = append(buf, byte(protocolTypeLen>>8), byte(protocolTypeLen))
+		buf = append(buf, []byte(group.ProtocolType)...)
+
+		// Group state (v4+)
+		if apiVersion >= 4 {
+			groupStateLen := uint16(len(group.GroupState))
+			buf = append(buf, byte(groupStateLen>>8), byte(groupStateLen))
+			buf = append(buf, []byte(group.GroupState)...)
+		}
+	}
+
+	return buf
+}

weed/mq/kafka/protocol/heartbeat_response_format_test.go

@@ -0,0 +1,182 @@
+package protocol
+
+import (
+	"encoding/binary"
+	"testing"
+)
+
+// TestHeartbeatResponseFormat_V0 verifies Heartbeat v0 response format
+// v0: error_code (2 bytes) - NO throttle_time_ms
+func TestHeartbeatResponseFormat_V0(t *testing.T) {
+	h := &Handler{}
+	response := HeartbeatResponse{
+		CorrelationID: 12345,
+		ErrorCode:     ErrorCodeNone,
+	}
+
+	result := h.buildHeartbeatResponseV(response, 0)
+
+	// v0 should only have error_code (2 bytes)
+	if len(result) != 2 {
+		t.Errorf("Heartbeat v0 response length = %d, want 2 bytes (error_code only)", len(result))
+	}
+
+	// Verify error code
+	errorCode := int16(binary.BigEndian.Uint16(result[0:2]))
+	if errorCode != ErrorCodeNone {
+		t.Errorf("Heartbeat v0 error_code = %d, want %d", errorCode, ErrorCodeNone)
+	}
+}
+
+// TestHeartbeatResponseFormat_V1ToV3 verifies Heartbeat v1-v3 response format
+// v1-v3: throttle_time_ms (4 bytes) -> error_code (2 bytes)
+// CRITICAL: throttle_time_ms comes FIRST in v1+
+func TestHeartbeatResponseFormat_V1ToV3(t *testing.T) {
+	testCases := []struct {
+		apiVersion uint16
+		name       string
+	}{
+		{1, "v1"},
+		{2, "v2"},
+		{3, "v3"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			h := &Handler{}
+			response := HeartbeatResponse{
+				CorrelationID: 12345,
+				ErrorCode:     ErrorCodeNone,
+			}
+
+			result := h.buildHeartbeatResponseV(response, tc.apiVersion)
+
+			// v1-v3 should have throttle_time_ms (4 bytes) + error_code (2 bytes) = 6 bytes
+			if len(result) != 6 {
+				t.Errorf("Heartbeat %s response length = %d, want 6 bytes", tc.name, len(result))
+			}
+
+			// CRITICAL: Verify field order - throttle_time_ms BEFORE error_code
+			// Bytes 0-3: throttle_time_ms (should be 0)
+			throttleTime := int32(binary.BigEndian.Uint32(result[0:4]))
+			if throttleTime != 0 {
+				t.Errorf("Heartbeat %s throttle_time_ms = %d, want 0", tc.name, throttleTime)
+			}
+
+			// Bytes 4-5: error_code (should be 0 = ErrorCodeNone)
+			errorCode := int16(binary.BigEndian.Uint16(result[4:6]))
+			if errorCode != ErrorCodeNone {
+				t.Errorf("Heartbeat %s error_code = %d, want %d", tc.name, errorCode, ErrorCodeNone)
+			}
+		})
+	}
+}
+
+// TestHeartbeatResponseFormat_V4Plus verifies Heartbeat v4+ response format (flexible)
+// v4+: throttle_time_ms (4 bytes) -> error_code (2 bytes) -> tagged_fields (varint)
+func TestHeartbeatResponseFormat_V4Plus(t *testing.T) {
+	testCases := []struct {
+		apiVersion uint16
+		name       string
+	}{
+		{4, "v4"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			h := &Handler{}
+			response := HeartbeatResponse{
+				CorrelationID: 12345,
+				ErrorCode:     ErrorCodeNone,
+			}
+
+			result := h.buildHeartbeatResponseV(response, tc.apiVersion)
+
+			// v4+ should have throttle_time_ms (4 bytes) + error_code (2 bytes) + tagged_fields (1 byte for empty) = 7 bytes
+			if len(result) != 7 {
+				t.Errorf("Heartbeat %s response length = %d, want 7 bytes", tc.name, len(result))
+			}
+
+			// Verify field order - throttle_time_ms BEFORE error_code
+			// Bytes 0-3: throttle_time_ms (should be 0)
+			throttleTime := int32(binary.BigEndian.Uint32(result[0:4]))
+			if throttleTime != 0 {
+				t.Errorf("Heartbeat %s throttle_time_ms = %d, want 0", tc.name, throttleTime)
+			}
+
+			// Bytes 4-5: error_code (should be 0 = ErrorCodeNone)
+			errorCode := int16(binary.BigEndian.Uint16(result[4:6]))
+			if errorCode != ErrorCodeNone {
+				t.Errorf("Heartbeat %s error_code = %d, want %d", tc.name, errorCode, ErrorCodeNone)
+			}
+
+			// Byte 6: tagged_fields (should be 0x00 for empty)
+			taggedFields := result[6]
+			if taggedFields != 0x00 {
+				t.Errorf("Heartbeat %s tagged_fields = 0x%02x, want 0x00", tc.name, taggedFields)
+			}
+		})
+	}
+}
+
+// TestHeartbeatResponseFormat_ErrorCode verifies error codes are correctly encoded
+func TestHeartbeatResponseFormat_ErrorCode(t *testing.T) {
+	testCases := []struct {
+		errorCode int16
+		name      string
+	}{
+		{ErrorCodeNone, "None"},
+		{ErrorCodeUnknownMemberID, "UnknownMemberID"},
+		{ErrorCodeIllegalGeneration, "IllegalGeneration"},
+		{ErrorCodeRebalanceInProgress, "RebalanceInProgress"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			h := &Handler{}
+			response := HeartbeatResponse{
+				CorrelationID: 12345,
+				ErrorCode:     tc.errorCode,
+			}
+
+			// Test with v3 (non-flexible)
+			result := h.buildHeartbeatResponseV(response, 3)
+
+			// Bytes 4-5: error_code
+			errorCode := int16(binary.BigEndian.Uint16(result[4:6]))
+			if errorCode != tc.errorCode {
+				t.Errorf("Heartbeat v3 error_code = %d, want %d", errorCode, tc.errorCode)
+			}
+		})
+	}
+}
+
+// TestHeartbeatResponseFormat_BugReproduce reproduces the original bug
+// This test documents the bug where error_code was placed BEFORE throttle_time_ms in v1-v3
+func TestHeartbeatResponseFormat_BugReproduce(t *testing.T) {
+	t.Skip("This test documents the original bug - skip to avoid false failures")
+
+	// Original buggy implementation would have:
+	// v1-v3: error_code (2 bytes) -> throttle_time_ms (4 bytes)
+	// This caused Sarama to read error_code bytes as throttle_time_ms, resulting in huge throttle values
+
+	// Example: error_code = 0 (0x0000) would be read as throttle_time_ms = 0
+	// But if there were any non-zero bytes, it would cause massive throttle times
+
+	// But if error_code was non-zero, e.g., ErrorCodeIllegalGeneration = 22:
+	buggyResponseWithError := []byte{
+		0x00, 0x16, // error_code = 22 (0x0016)
+		0x00, 0x00, 0x00, 0x00, // throttle_time_ms = 0
+	}
+
+	// Sarama would read:
+	// - Bytes 0-3 as throttle_time_ms: 0x00160000 = 1441792 ms = 24 minutes!
+	throttleTimeMs := binary.BigEndian.Uint32(buggyResponseWithError[0:4])
+	if throttleTimeMs != 1441792 {
+		t.Errorf("Buggy format would cause throttle_time_ms = %d ms (%.1f minutes), want 1441792 ms (24 minutes)",
+			throttleTimeMs, float64(throttleTimeMs)/60000)
+	}
+
+	t.Logf("Original bug: error_code=22 would be misread as throttle_time_ms=%d ms (%.1f minutes)",
+		throttleTimeMs, float64(throttleTimeMs)/60000)
+}

weed/mq/kafka/protocol/metadata_blocking_test.go

@@ -0,0 +1,373 @@
+package protocol
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/integration"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+// TestMetadataRequestBlocking documents the original bug where Metadata requests hang
+// when the backend (broker/filer) ListTopics call blocks indefinitely.
+// This test is kept for documentation purposes and to verify the mock handler behavior.
+//
+// NOTE: The actual fix is in the broker's ListTopics implementation (weed/mq/broker/broker_grpc_lookup.go)
+// which adds a 2-second timeout for filer operations. This test uses a mock handler that
+// bypasses that fix, so it still demonstrates the original blocking behavior.
+func TestMetadataRequestBlocking(t *testing.T) {
+	t.Skip("This test documents the original bug. The fix is in the broker's ListTopics with filer timeout. Run TestMetadataRequestWithFastMock to verify fast path works.")
+
+	t.Log("Testing Metadata handler with blocking backend...")
+
+	// Create a handler with a mock backend that blocks on ListTopics
+	handler := &Handler{
+		seaweedMQHandler: &BlockingMockHandler{
+			blockDuration: 10 * time.Second, // Simulate slow backend
+		},
+	}
+
+	// Call handleMetadata in a goroutine so we can timeout
+	responseChan := make(chan []byte, 1)
+	errorChan := make(chan error, 1)
+
+	go func() {
+		// Build a simple Metadata v1 request body (empty topics array = all topics)
+		requestBody := []byte{0, 0, 0, 0} // Empty topics array
+		response, err := handler.handleMetadata(1, 1, requestBody)
+		if err != nil {
+			errorChan <- err
+		} else {
+			responseChan <- response
+		}
+	}()
+
+	// Wait for response with timeout
+	select {
+	case response := <-responseChan:
+		t.Logf("Metadata response received (%d bytes) - backend responded", len(response))
+		t.Error("UNEXPECTED: Response received before timeout - backend should have blocked")
+	case err := <-errorChan:
+		t.Logf("Metadata returned error: %v", err)
+		t.Error("UNEXPECTED: Error received - expected blocking, not error")
+	case <-time.After(3 * time.Second):
+		t.Logf("✓ BUG REPRODUCED: Metadata request blocked for 3+ seconds")
+		t.Logf("  Root cause: seaweedMQHandler.ListTopics() blocks indefinitely when broker/filer is slow")
+		t.Logf("  Impact: Entire control plane processor goroutine is frozen")
+		t.Logf("  Fix implemented: Broker's ListTopics now has 2-second timeout for filer operations")
+		// This is expected behavior with blocking mock - demonstrates the original issue
+	}
+}
+
+// TestMetadataRequestWithFastMock verifies that Metadata requests complete quickly
+// when the backend responds promptly (the common case)
+func TestMetadataRequestWithFastMock(t *testing.T) {
+	t.Log("Testing Metadata handler with fast-responding backend...")
+
+	// Create a handler with a fast mock (simulates in-memory topics only)
+	handler := &Handler{
+		seaweedMQHandler: &FastMockHandler{
+			topics: []string{"test-topic-1", "test-topic-2"},
+		},
+	}
+
+	// Call handleMetadata and measure time
+	start := time.Now()
+	requestBody := []byte{0, 0, 0, 0} // Empty topics array = list all
+	response, err := handler.handleMetadata(1, 1, requestBody)
+	duration := time.Since(start)
+
+	if err != nil {
+		t.Errorf("Metadata returned error: %v", err)
+	} else if response == nil {
+		t.Error("Metadata returned nil response")
+	} else {
+		t.Logf("✓ Metadata completed in %v (%d bytes)", duration, len(response))
+		if duration > 500*time.Millisecond {
+			t.Errorf("Metadata took too long: %v (should be < 500ms for fast backend)", duration)
+		}
+	}
+}
+
+// TestMetadataRequestWithTimeoutFix tests that Metadata requests with timeout-aware backend
+// complete within reasonable time even when underlying storage is slow
+func TestMetadataRequestWithTimeoutFix(t *testing.T) {
+	t.Log("Testing Metadata handler with timeout-aware backend...")
+
+	// Create a handler with a timeout-aware mock
+	// This simulates the broker's ListTopics with 2-second filer timeout
+	handler := &Handler{
+		seaweedMQHandler: &TimeoutAwareMockHandler{
+			timeout:       2 * time.Second,
+			blockDuration: 10 * time.Second, // Backend is slow but timeout kicks in
+		},
+	}
+
+	// Call handleMetadata and measure time
+	start := time.Now()
+	requestBody := []byte{0, 0, 0, 0} // Empty topics array
+	response, err := handler.handleMetadata(1, 1, requestBody)
+	duration := time.Since(start)
+
+	t.Logf("Metadata completed in %v", duration)
+
+	if err != nil {
+		t.Logf("✓ Metadata returned error after timeout: %v", err)
+		// This is acceptable - error response is better than hanging
+	} else if response != nil {
+		t.Logf("✓ Metadata returned response (%d bytes) without blocking", len(response))
+		// Backend timed out but still returned in-memory topics
+		if duration > 3*time.Second {
+			t.Errorf("Metadata took too long: %v (should timeout at ~2s)", duration)
+		}
+	} else {
+		t.Error("Metadata returned nil response and nil error - unexpected")
+	}
+}
+
+// FastMockHandler simulates a fast backend (in-memory topics only)
+type FastMockHandler struct {
+	topics []string
+}
+
+func (h *FastMockHandler) ListTopics() []string {
+	// Fast response - simulates in-memory topics
+	return h.topics
+}
+
+func (h *FastMockHandler) TopicExists(name string) bool {
+	for _, topic := range h.topics {
+		if topic == name {
+			return true
+		}
+	}
+	return false
+}
+
+func (h *FastMockHandler) CreateTopic(name string, partitions int32) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (h *FastMockHandler) CreateTopicWithSchemas(name string, partitions int32, keyRecordType *schema_pb.RecordType, valueRecordType *schema_pb.RecordType) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (h *FastMockHandler) DeleteTopic(name string) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (h *FastMockHandler) GetTopicInfo(name string) (*integration.KafkaTopicInfo, bool) {
+	return nil, false
+}
+
+func (h *FastMockHandler) ProduceRecord(ctx context.Context, topicName string, partitionID int32, key, value []byte) (int64, error) {
+	return 0, fmt.Errorf("not implemented")
+}
+
+func (h *FastMockHandler) ProduceRecordValue(ctx context.Context, topicName string, partitionID int32, key []byte, recordValueBytes []byte) (int64, error) {
+	return 0, fmt.Errorf("not implemented")
+}
+
+func (h *FastMockHandler) GetStoredRecords(ctx context.Context, topic string, partition int32, fromOffset int64, maxRecords int) ([]integration.SMQRecord, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (h *FastMockHandler) GetEarliestOffset(topic string, partition int32) (int64, error) {
+	return 0, fmt.Errorf("not implemented")
+}
+
+func (h *FastMockHandler) GetLatestOffset(topic string, partition int32) (int64, error) {
+	return 0, fmt.Errorf("not implemented")
+}
+
+func (h *FastMockHandler) WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (h *FastMockHandler) GetBrokerAddresses() []string {
+	return []string{"localhost:17777"}
+}
+
+func (h *FastMockHandler) CreatePerConnectionBrokerClient() (*integration.BrokerClient, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (h *FastMockHandler) SetProtocolHandler(handler integration.ProtocolHandler) {
+	// No-op
+}
+
+func (h *FastMockHandler) InvalidateTopicExistsCache(topic string) {
+	// No-op for mock
+}
+
+func (h *FastMockHandler) Close() error {
+	return nil
+}
+
+// BlockingMockHandler simulates a backend that blocks indefinitely on ListTopics
+type BlockingMockHandler struct {
+	blockDuration time.Duration
+}
+
+func (h *BlockingMockHandler) ListTopics() []string {
+	// Simulate backend blocking (e.g., waiting for unresponsive broker/filer)
+	time.Sleep(h.blockDuration)
+	return []string{}
+}
+
+func (h *BlockingMockHandler) TopicExists(name string) bool {
+	return false
+}
+
+func (h *BlockingMockHandler) CreateTopic(name string, partitions int32) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (h *BlockingMockHandler) CreateTopicWithSchemas(name string, partitions int32, keyRecordType *schema_pb.RecordType, valueRecordType *schema_pb.RecordType) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (h *BlockingMockHandler) DeleteTopic(name string) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (h *BlockingMockHandler) GetTopicInfo(name string) (*integration.KafkaTopicInfo, bool) {
+	return nil, false
+}
+
+func (h *BlockingMockHandler) ProduceRecord(ctx context.Context, topicName string, partitionID int32, key, value []byte) (int64, error) {
+	return 0, fmt.Errorf("not implemented")
+}
+
+func (h *BlockingMockHandler) ProduceRecordValue(ctx context.Context, topicName string, partitionID int32, key []byte, recordValueBytes []byte) (int64, error) {
+	return 0, fmt.Errorf("not implemented")
+}
+
+func (h *BlockingMockHandler) GetStoredRecords(ctx context.Context, topic string, partition int32, fromOffset int64, maxRecords int) ([]integration.SMQRecord, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (h *BlockingMockHandler) GetEarliestOffset(topic string, partition int32) (int64, error) {
+	return 0, fmt.Errorf("not implemented")
+}
+
+func (h *BlockingMockHandler) GetLatestOffset(topic string, partition int32) (int64, error) {
+	return 0, fmt.Errorf("not implemented")
+}
+
+func (h *BlockingMockHandler) WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (h *BlockingMockHandler) GetBrokerAddresses() []string {
+	return []string{"localhost:17777"}
+}
+
+func (h *BlockingMockHandler) CreatePerConnectionBrokerClient() (*integration.BrokerClient, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (h *BlockingMockHandler) SetProtocolHandler(handler integration.ProtocolHandler) {
+	// No-op
+}
+
+func (h *BlockingMockHandler) InvalidateTopicExistsCache(topic string) {
+	// No-op for mock
+}
+
+func (h *BlockingMockHandler) Close() error {
+	return nil
+}
+
+// TimeoutAwareMockHandler demonstrates expected behavior with timeout
+type TimeoutAwareMockHandler struct {
+	timeout       time.Duration
+	blockDuration time.Duration
+}
+
+func (h *TimeoutAwareMockHandler) ListTopics() []string {
+	// Simulate timeout-aware backend
+	ctx, cancel := context.WithTimeout(context.Background(), h.timeout)
+	defer cancel()
+
+	done := make(chan bool)
+	go func() {
+		time.Sleep(h.blockDuration)
+		done <- true
+	}()
+
+	select {
+	case <-done:
+		return []string{}
+	case <-ctx.Done():
+		// Timeout - return empty list rather than blocking forever
+		return []string{}
+	}
+}
+
+func (h *TimeoutAwareMockHandler) TopicExists(name string) bool {
+	return false
+}
+
+func (h *TimeoutAwareMockHandler) CreateTopic(name string, partitions int32) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (h *TimeoutAwareMockHandler) CreateTopicWithSchemas(name string, partitions int32, keyRecordType *schema_pb.RecordType, valueRecordType *schema_pb.RecordType) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (h *TimeoutAwareMockHandler) DeleteTopic(name string) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (h *TimeoutAwareMockHandler) GetTopicInfo(name string) (*integration.KafkaTopicInfo, bool) {
+	return nil, false
+}
+
+func (h *TimeoutAwareMockHandler) ProduceRecord(ctx context.Context, topicName string, partitionID int32, key, value []byte) (int64, error) {
+	return 0, fmt.Errorf("not implemented")
+}
+
+func (h *TimeoutAwareMockHandler) ProduceRecordValue(ctx context.Context, topicName string, partitionID int32, key []byte, recordValueBytes []byte) (int64, error) {
+	return 0, fmt.Errorf("not implemented")
+}
+
+func (h *TimeoutAwareMockHandler) GetStoredRecords(ctx context.Context, topic string, partition int32, fromOffset int64, maxRecords int) ([]integration.SMQRecord, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (h *TimeoutAwareMockHandler) GetEarliestOffset(topic string, partition int32) (int64, error) {
+	return 0, fmt.Errorf("not implemented")
+}
+
+func (h *TimeoutAwareMockHandler) GetLatestOffset(topic string, partition int32) (int64, error) {
+	return 0, fmt.Errorf("not implemented")
+}
+
+func (h *TimeoutAwareMockHandler) WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (h *TimeoutAwareMockHandler) GetBrokerAddresses() []string {
+	return []string{"localhost:17777"}
+}
+
+func (h *TimeoutAwareMockHandler) CreatePerConnectionBrokerClient() (*integration.BrokerClient, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (h *TimeoutAwareMockHandler) SetProtocolHandler(handler integration.ProtocolHandler) {
+	// No-op
+}
+
+func (h *TimeoutAwareMockHandler) InvalidateTopicExistsCache(topic string) {
+	// No-op for mock
+}
+
+func (h *TimeoutAwareMockHandler) Close() error {
+	return nil
+}

weed/mq/kafka/protocol/metrics.go

@@ -0,0 +1,233 @@
+package protocol
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// Metrics tracks basic request/error/latency statistics for Kafka protocol operations
+type Metrics struct {
+	// Request counters by API key
+	requestCounts map[uint16]*int64
+	errorCounts   map[uint16]*int64
+
+	// Latency tracking
+	latencySum   map[uint16]*int64 // Total latency in microseconds
+	latencyCount map[uint16]*int64 // Number of requests for average calculation
+
+	// Connection metrics
+	activeConnections int64
+	totalConnections  int64
+
+	// Mutex for map operations
+	mu sync.RWMutex
+
+	// Start time for uptime calculation
+	startTime time.Time
+}
+
+// APIMetrics represents metrics for a specific API
+type APIMetrics struct {
+	APIKey       uint16  `json:"api_key"`
+	APIName      string  `json:"api_name"`
+	RequestCount int64   `json:"request_count"`
+	ErrorCount   int64   `json:"error_count"`
+	AvgLatencyMs float64 `json:"avg_latency_ms"`
+}
+
+// ConnectionMetrics represents connection-related metrics
+type ConnectionMetrics struct {
+	ActiveConnections int64     `json:"active_connections"`
+	TotalConnections  int64     `json:"total_connections"`
+	UptimeSeconds     int64     `json:"uptime_seconds"`
+	StartTime         time.Time `json:"start_time"`
+}
+
+// MetricsSnapshot represents a complete metrics snapshot
+type MetricsSnapshot struct {
+	APIs        []APIMetrics      `json:"apis"`
+	Connections ConnectionMetrics `json:"connections"`
+	Timestamp   time.Time         `json:"timestamp"`
+}
+
+// NewMetrics creates a new metrics tracker
+func NewMetrics() *Metrics {
+	return &Metrics{
+		requestCounts: make(map[uint16]*int64),
+		errorCounts:   make(map[uint16]*int64),
+		latencySum:    make(map[uint16]*int64),
+		latencyCount:  make(map[uint16]*int64),
+		startTime:     time.Now(),
+	}
+}
+
+// RecordRequest records a successful request with latency
+func (m *Metrics) RecordRequest(apiKey uint16, latency time.Duration) {
+	m.ensureCounters(apiKey)
+
+	atomic.AddInt64(m.requestCounts[apiKey], 1)
+	atomic.AddInt64(m.latencySum[apiKey], latency.Microseconds())
+	atomic.AddInt64(m.latencyCount[apiKey], 1)
+}
+
+// RecordError records an error for a specific API
+func (m *Metrics) RecordError(apiKey uint16, latency time.Duration) {
+	m.ensureCounters(apiKey)
+
+	atomic.AddInt64(m.requestCounts[apiKey], 1)
+	atomic.AddInt64(m.errorCounts[apiKey], 1)
+	atomic.AddInt64(m.latencySum[apiKey], latency.Microseconds())
+	atomic.AddInt64(m.latencyCount[apiKey], 1)
+}
+
+// RecordConnection records a new connection
+func (m *Metrics) RecordConnection() {
+	atomic.AddInt64(&m.activeConnections, 1)
+	atomic.AddInt64(&m.totalConnections, 1)
+}
+
+// RecordDisconnection records a connection closure
+func (m *Metrics) RecordDisconnection() {
+	atomic.AddInt64(&m.activeConnections, -1)
+}
+
+// GetSnapshot returns a complete metrics snapshot
+func (m *Metrics) GetSnapshot() MetricsSnapshot {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	apis := make([]APIMetrics, 0, len(m.requestCounts))
+
+	for apiKey, requestCount := range m.requestCounts {
+		requests := atomic.LoadInt64(requestCount)
+		errors := atomic.LoadInt64(m.errorCounts[apiKey])
+		latencySum := atomic.LoadInt64(m.latencySum[apiKey])
+		latencyCount := atomic.LoadInt64(m.latencyCount[apiKey])
+
+		var avgLatencyMs float64
+		if latencyCount > 0 {
+			avgLatencyMs = float64(latencySum) / float64(latencyCount) / 1000.0 // Convert to milliseconds
+		}
+
+		apis = append(apis, APIMetrics{
+			APIKey:       apiKey,
+			APIName:      getAPIName(APIKey(apiKey)),
+			RequestCount: requests,
+			ErrorCount:   errors,
+			AvgLatencyMs: avgLatencyMs,
+		})
+	}
+
+	return MetricsSnapshot{
+		APIs: apis,
+		Connections: ConnectionMetrics{
+			ActiveConnections: atomic.LoadInt64(&m.activeConnections),
+			TotalConnections:  atomic.LoadInt64(&m.totalConnections),
+			UptimeSeconds:     int64(time.Since(m.startTime).Seconds()),
+			StartTime:         m.startTime,
+		},
+		Timestamp: time.Now(),
+	}
+}
+
+// GetAPIMetrics returns metrics for a specific API
+func (m *Metrics) GetAPIMetrics(apiKey uint16) APIMetrics {
+	m.ensureCounters(apiKey)
+
+	requests := atomic.LoadInt64(m.requestCounts[apiKey])
+	errors := atomic.LoadInt64(m.errorCounts[apiKey])
+	latencySum := atomic.LoadInt64(m.latencySum[apiKey])
+	latencyCount := atomic.LoadInt64(m.latencyCount[apiKey])
+
+	var avgLatencyMs float64
+	if latencyCount > 0 {
+		avgLatencyMs = float64(latencySum) / float64(latencyCount) / 1000.0
+	}
+
+	return APIMetrics{
+		APIKey:       apiKey,
+		APIName:      getAPIName(APIKey(apiKey)),
+		RequestCount: requests,
+		ErrorCount:   errors,
+		AvgLatencyMs: avgLatencyMs,
+	}
+}
+
+// GetConnectionMetrics returns connection-related metrics
+func (m *Metrics) GetConnectionMetrics() ConnectionMetrics {
+	return ConnectionMetrics{
+		ActiveConnections: atomic.LoadInt64(&m.activeConnections),
+		TotalConnections:  atomic.LoadInt64(&m.totalConnections),
+		UptimeSeconds:     int64(time.Since(m.startTime).Seconds()),
+		StartTime:         m.startTime,
+	}
+}
+
+// Reset resets all metrics (useful for testing)
+func (m *Metrics) Reset() {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	for apiKey := range m.requestCounts {
+		atomic.StoreInt64(m.requestCounts[apiKey], 0)
+		atomic.StoreInt64(m.errorCounts[apiKey], 0)
+		atomic.StoreInt64(m.latencySum[apiKey], 0)
+		atomic.StoreInt64(m.latencyCount[apiKey], 0)
+	}
+
+	atomic.StoreInt64(&m.activeConnections, 0)
+	atomic.StoreInt64(&m.totalConnections, 0)
+	m.startTime = time.Now()
+}
+
+// ensureCounters ensures that counters exist for the given API key
+func (m *Metrics) ensureCounters(apiKey uint16) {
+	m.mu.RLock()
+	if _, exists := m.requestCounts[apiKey]; exists {
+		m.mu.RUnlock()
+		return
+	}
+	m.mu.RUnlock()
+
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	// Double-check after acquiring write lock
+	if _, exists := m.requestCounts[apiKey]; exists {
+		return
+	}
+
+	m.requestCounts[apiKey] = new(int64)
+	m.errorCounts[apiKey] = new(int64)
+	m.latencySum[apiKey] = new(int64)
+	m.latencyCount[apiKey] = new(int64)
+}
+
+// Global metrics instance
+var globalMetrics = NewMetrics()
+
+// GetGlobalMetrics returns the global metrics instance
+func GetGlobalMetrics() *Metrics {
+	return globalMetrics
+}
+
+// RecordRequestMetrics is a convenience function to record request metrics globally
+func RecordRequestMetrics(apiKey uint16, latency time.Duration) {
+	globalMetrics.RecordRequest(apiKey, latency)
+}
+
+// RecordErrorMetrics is a convenience function to record error metrics globally
+func RecordErrorMetrics(apiKey uint16, latency time.Duration) {
+	globalMetrics.RecordError(apiKey, latency)
+}
+
+// RecordConnectionMetrics is a convenience function to record connection metrics globally
+func RecordConnectionMetrics() {
+	globalMetrics.RecordConnection()
+}
+
+// RecordDisconnectionMetrics is a convenience function to record disconnection metrics globally
+func RecordDisconnectionMetrics() {
+	globalMetrics.RecordDisconnection()
+}

weed/mq/kafka/protocol/offset_fetch_pattern_test.go

@@ -0,0 +1,219 @@
+package protocol
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+// TestOffsetCommitFetchPattern verifies the critical pattern:
+// 1. Consumer reads messages 0-N
+// 2. Consumer commits offset N
+// 3. Consumer fetches messages starting from N+1
+// 4. No message loss or duplication
+//
+// This tests for the root cause of the "consumer stalling" issue where
+// consumers stop fetching after certain offsets.
+func TestOffsetCommitFetchPattern(t *testing.T) {
+	t.Skip("Integration test - requires mock broker setup")
+
+	// Setup
+	const (
+		topic        = "test-topic"
+		partition    = int32(0)
+		messageCount = 1000
+		batchSize    = 50
+		groupID      = "test-group"
+	)
+
+	// Mock store for offsets
+	offsetStore := make(map[string]int64)
+	offsetKey := fmt.Sprintf("%s/%s/%d", groupID, topic, partition)
+
+	// Simulate message production
+	messages := make([][]byte, messageCount)
+	for i := 0; i < messageCount; i++ {
+		messages[i] = []byte(fmt.Sprintf("message-%d", i))
+	}
+
+	// Test: Sequential consumption with offset commits
+	t.Run("SequentialConsumption", func(t *testing.T) {
+		consumedOffsets := make(map[int64]bool)
+		nextOffset := int64(0)
+
+		for nextOffset < int64(messageCount) {
+			// Step 1: Fetch batch of messages starting from nextOffset
+			endOffset := nextOffset + int64(batchSize)
+			if endOffset > int64(messageCount) {
+				endOffset = int64(messageCount)
+			}
+
+			fetchedCount := endOffset - nextOffset
+			if fetchedCount <= 0 {
+				t.Fatalf("Fetch returned no messages at offset %d (HWM=%d)", nextOffset, messageCount)
+			}
+
+			// Simulate fetching messages
+			for i := nextOffset; i < endOffset; i++ {
+				if consumedOffsets[i] {
+					t.Errorf("DUPLICATE: Message at offset %d already consumed", i)
+				}
+				consumedOffsets[i] = true
+			}
+
+			// Step 2: Commit the last offset in this batch
+			lastConsumedOffset := endOffset - 1
+			offsetStore[offsetKey] = lastConsumedOffset
+			t.Logf("Batch %d: Consumed offsets %d-%d, committed offset %d",
+				nextOffset/int64(batchSize), nextOffset, lastConsumedOffset, lastConsumedOffset)
+
+			// Step 3: Verify offset is correctly stored
+			storedOffset, exists := offsetStore[offsetKey]
+			if !exists || storedOffset != lastConsumedOffset {
+				t.Errorf("Offset not stored correctly: stored=%v, expected=%d", storedOffset, lastConsumedOffset)
+			}
+
+			// Step 4: Next fetch should start from lastConsumedOffset + 1
+			nextOffset = lastConsumedOffset + 1
+		}
+
+		// Verify all messages were consumed exactly once
+		if len(consumedOffsets) != messageCount {
+			t.Errorf("Not all messages consumed: got %d, expected %d", len(consumedOffsets), messageCount)
+		}
+
+		for i := 0; i < messageCount; i++ {
+			if !consumedOffsets[int64(i)] {
+				t.Errorf("Message at offset %d not consumed", i)
+			}
+		}
+	})
+
+	t.Logf("✅ Sequential consumption pattern verified successfully")
+}
+
+// TestOffsetFetchAfterCommit verifies that after committing offset N,
+// the next fetch returns offset N+1 onwards (not empty, not error)
+func TestOffsetFetchAfterCommit(t *testing.T) {
+	t.Skip("Integration test - requires mock broker setup")
+
+	t.Run("FetchAfterCommit", func(t *testing.T) {
+		// Simulate: Commit offset 163, then fetch offset 164
+		committedOffset := int64(163)
+		nextFetchOffset := committedOffset + 1
+
+		t.Logf("After committing offset %d, fetching from offset %d", committedOffset, nextFetchOffset)
+
+		// This is where consumers are getting stuck!
+		// They commit offset 163, then fetch 164+, but get empty response
+
+		// Expected: Fetch(164) returns records starting from offset 164
+		// Actual Bug: Fetch(164) returns empty, consumer stops fetching
+
+		if nextFetchOffset > committedOffset+100 {
+			t.Errorf("POTENTIAL BUG: Fetch offset %d is way beyond committed offset %d",
+				nextFetchOffset, committedOffset)
+		}
+
+		t.Logf("✅ Offset fetch request looks correct: committed=%d, next_fetch=%d",
+			committedOffset, nextFetchOffset)
+	})
+}
+
+// TestOffsetPersistencePattern verifies that offsets are correctly
+// persisted and recovered across restarts
+func TestOffsetPersistencePattern(t *testing.T) {
+	t.Skip("Integration test - requires mock broker setup")
+
+	t.Run("OffsetRecovery", func(t *testing.T) {
+		const (
+			groupID   = "test-group"
+			topic     = "test-topic"
+			partition = int32(0)
+		)
+
+		offsetStore := make(map[string]int64)
+		offsetKey := fmt.Sprintf("%s/%s/%d", groupID, topic, partition)
+
+		// Scenario 1: First consumer session
+		// Consume messages 0-99, commit offset 99
+		offsetStore[offsetKey] = 99
+		t.Logf("Session 1: Committed offset 99")
+
+		// Scenario 2: Consumer restarts (consumer group rebalancing)
+		// Should recover offset 99 from storage
+		recoveredOffset, exists := offsetStore[offsetKey]
+		if !exists || recoveredOffset != 99 {
+			t.Errorf("Failed to recover offset: expected 99, got %v", recoveredOffset)
+		}
+
+		// Scenario 3: Continue consuming from offset 100
+		// This is where the bug manifests! Consumer might:
+		// A) Correctly fetch from 100
+		// B) Try to fetch from 99 (duplicate)
+		// C) Get stuck and not fetch at all
+		nextOffset := recoveredOffset + 1
+		if nextOffset != 100 {
+			t.Errorf("Incorrect next offset after recovery: expected 100, got %d", nextOffset)
+		}
+
+		t.Logf("✅ Offset recovery pattern works: recovered %d, next fetch at %d", recoveredOffset, nextOffset)
+	})
+}
+
+// TestOffsetCommitConsistency verifies that offset commits are atomic
+// and don't cause partial updates
+func TestOffsetCommitConsistency(t *testing.T) {
+	t.Skip("Integration test - requires mock broker setup")
+
+	t.Run("AtomicCommit", func(t *testing.T) {
+		type OffsetCommit struct {
+			Group     string
+			Topic     string
+			Partition int32
+			Offset    int64
+			Timestamp int64
+		}
+
+		commits := []OffsetCommit{
+			{"group1", "topic1", 0, 100, time.Now().UnixNano()},
+			{"group1", "topic1", 1, 150, time.Now().UnixNano()},
+			{"group1", "topic1", 2, 120, time.Now().UnixNano()},
+		}
+
+		// All commits should succeed or all fail (atomicity)
+		for _, commit := range commits {
+			key := fmt.Sprintf("%s/%s/%d", commit.Group, commit.Topic, commit.Partition)
+			t.Logf("Committing %s at offset %d", key, commit.Offset)
+
+			// Verify offset is correctly persisted
+			// (In real test, would read from SMQ storage)
+		}
+
+		t.Logf("✅ Offset commit consistency verified")
+	})
+}
+
+// TestLongPollWithOffsetCommit verifies long-poll semantics work correctly
+// with offset commits (no throttling confusion)
+func TestLongPollWithOffsetCommit(t *testing.T) {
+	t.Skip("Integration test - requires mock broker setup")
+
+	t.Run("LongPollNoThrottling", func(t *testing.T) {
+		// Critical: long-poll duration should NOT be reported as throttleTimeMs
+		// This was bug 8969b4509
+
+		// Simulate long-poll wait (no data available)
+		time.Sleep(100 * time.Millisecond)
+
+		// throttleTimeMs should be 0 (NOT elapsed duration!)
+		throttleTimeMs := int32(0) // CORRECT
+		// throttleTimeMs := int32(elapsed / time.Millisecond) // WRONG (previous bug)
+
+		if throttleTimeMs > 0 {
+			t.Errorf("Long-poll elapsed time should NOT be reported as throttle: %d ms", throttleTimeMs)
+		}
+
+		t.Logf("✅ Long-poll not confused with throttling")
+	})
+}

weed/mq/kafka/protocol/offset_management.go

@@ -0,0 +1,738 @@
+package protocol
+
+import (
+	"encoding/binary"
+	"fmt"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/consumer"
+)
+
+// ConsumerOffsetKey uniquely identifies a consumer offset
+type ConsumerOffsetKey struct {
+	ConsumerGroup         string
+	Topic                 string
+	Partition             int32
+	ConsumerGroupInstance string // Optional - for static group membership
+}
+
+// OffsetCommit API (key 8) - Commit consumer group offsets
+// This API allows consumers to persist their current position in topic partitions
+
+// OffsetCommitRequest represents an OffsetCommit request from a Kafka client
+type OffsetCommitRequest struct {
+	GroupID         string
+	GenerationID    int32
+	MemberID        string
+	GroupInstanceID string // Optional static membership ID
+	RetentionTime   int64  // Offset retention time (-1 for broker default)
+	Topics          []OffsetCommitTopic
+}
+
+// OffsetCommitTopic represents topic-level offset commit data
+type OffsetCommitTopic struct {
+	Name       string
+	Partitions []OffsetCommitPartition
+}
+
+// OffsetCommitPartition represents partition-level offset commit data
+type OffsetCommitPartition struct {
+	Index       int32  // Partition index
+	Offset      int64  // Offset to commit
+	LeaderEpoch int32  // Leader epoch (-1 if not available)
+	Metadata    string // Optional metadata
+}
+
+// OffsetCommitResponse represents an OffsetCommit response to a Kafka client
+type OffsetCommitResponse struct {
+	CorrelationID uint32
+	Topics        []OffsetCommitTopicResponse
+}
+
+// OffsetCommitTopicResponse represents topic-level offset commit response
+type OffsetCommitTopicResponse struct {
+	Name       string
+	Partitions []OffsetCommitPartitionResponse
+}
+
+// OffsetCommitPartitionResponse represents partition-level offset commit response
+type OffsetCommitPartitionResponse struct {
+	Index     int32
+	ErrorCode int16
+}
+
+// OffsetFetch API (key 9) - Fetch consumer group committed offsets
+// This API allows consumers to retrieve their last committed positions
+
+// OffsetFetchRequest represents an OffsetFetch request from a Kafka client
+type OffsetFetchRequest struct {
+	GroupID         string
+	GroupInstanceID string // Optional static membership ID
+	Topics          []OffsetFetchTopic
+	RequireStable   bool // Only fetch stable offsets
+}
+
+// OffsetFetchTopic represents topic-level offset fetch data
+type OffsetFetchTopic struct {
+	Name       string
+	Partitions []int32 // Partition indices to fetch (empty = all partitions)
+}
+
+// OffsetFetchResponse represents an OffsetFetch response to a Kafka client
+type OffsetFetchResponse struct {
+	CorrelationID uint32
+	Topics        []OffsetFetchTopicResponse
+	ErrorCode     int16 // Group-level error
+}
+
+// OffsetFetchTopicResponse represents topic-level offset fetch response
+type OffsetFetchTopicResponse struct {
+	Name       string
+	Partitions []OffsetFetchPartitionResponse
+}
+
+// OffsetFetchPartitionResponse represents partition-level offset fetch response
+type OffsetFetchPartitionResponse struct {
+	Index       int32
+	Offset      int64  // Committed offset (-1 if no offset)
+	LeaderEpoch int32  // Leader epoch (-1 if not available)
+	Metadata    string // Optional metadata
+	ErrorCode   int16  // Partition-level error
+}
+
+// Error codes specific to offset management are imported from errors.go
+
+func (h *Handler) handleOffsetCommit(correlationID uint32, apiVersion uint16, requestBody []byte) ([]byte, error) {
+	// Parse OffsetCommit request
+	req, err := h.parseOffsetCommitRequest(requestBody, apiVersion)
+	if err != nil {
+		return h.buildOffsetCommitErrorResponse(correlationID, ErrorCodeInvalidCommitOffsetSize, apiVersion), nil
+	}
+
+	// Validate request
+	if req.GroupID == "" || req.MemberID == "" {
+		return h.buildOffsetCommitErrorResponse(correlationID, ErrorCodeInvalidGroupID, apiVersion), nil
+	}
+
+	// Get or create consumer group
+	// Some Kafka clients (like kafka-go Reader) commit offsets without formally joining
+	// the group via JoinGroup/SyncGroup. We need to support these "simple consumer" use cases.
+	group := h.groupCoordinator.GetOrCreateGroup(req.GroupID)
+
+	group.Mu.Lock()
+	defer group.Mu.Unlock()
+
+	// Update group's last activity
+	group.LastActivity = time.Now()
+
+	// Check generation compatibility
+	// Allow commits for empty groups (no active members) to support simple consumers
+	// that commit offsets without formal group membership
+	groupIsEmpty := len(group.Members) == 0
+	generationMatches := groupIsEmpty || (req.GenerationID == group.Generation)
+
+	glog.V(3).Infof("[OFFSET_COMMIT] Group check: id=%s reqGen=%d groupGen=%d members=%d empty=%v matches=%v",
+		req.GroupID, req.GenerationID, group.Generation, len(group.Members), groupIsEmpty, generationMatches)
+
+	// Process offset commits
+	resp := OffsetCommitResponse{
+		CorrelationID: correlationID,
+		Topics:        make([]OffsetCommitTopicResponse, 0, len(req.Topics)),
+	}
+
+	for _, t := range req.Topics {
+		topicResp := OffsetCommitTopicResponse{
+			Name:       t.Name,
+			Partitions: make([]OffsetCommitPartitionResponse, 0, len(t.Partitions)),
+		}
+
+		for _, p := range t.Partitions {
+
+			// Create consumer offset key for SMQ storage (not used immediately)
+			key := ConsumerOffsetKey{
+				Topic:                 t.Name,
+				Partition:             p.Index,
+				ConsumerGroup:         req.GroupID,
+				ConsumerGroupInstance: req.GroupInstanceID,
+			}
+
+			// Commit offset synchronously for immediate consistency
+			var errCode int16 = ErrorCodeNone
+			if generationMatches {
+				// Store in in-memory map for immediate response
+				// This is the primary committed offset position for consumers
+				if err := h.commitOffset(group, t.Name, p.Index, p.Offset, p.Metadata); err != nil {
+					errCode = ErrorCodeOffsetMetadataTooLarge
+					glog.V(2).Infof("[OFFSET_COMMIT] Failed to commit offset: group=%s topic=%s partition=%d offset=%d err=%v",
+						req.GroupID, t.Name, p.Index, p.Offset, err)
+				} else {
+					// Also persist to SMQ storage for durability across broker restarts
+					// This is done synchronously to ensure offset is not lost
+					if err := h.commitOffsetToSMQ(key, p.Offset, p.Metadata); err != nil {
+						// Log the error but don't fail the commit
+						// In-memory commit is the source of truth for active consumers
+						// SMQ persistence is best-effort for crash recovery
+						glog.V(3).Infof("[OFFSET_COMMIT] SMQ persist failed (non-fatal): group=%s topic=%s partition=%d offset=%d err=%v",
+							req.GroupID, t.Name, p.Index, p.Offset, err)
+					}
+					glog.V(3).Infof("[OFFSET_COMMIT] Committed: group=%s topic=%s partition=%d offset=%d gen=%d",
+						req.GroupID, t.Name, p.Index, p.Offset, group.Generation)
+				}
+			} else {
+				// Do not store commit if generation mismatch
+				errCode = 22 // IllegalGeneration
+				glog.V(2).Infof("[OFFSET_COMMIT] Rejected - generation mismatch: group=%s expected=%d got=%d members=%d",
+					req.GroupID, group.Generation, req.GenerationID, len(group.Members))
+			}
+
+			topicResp.Partitions = append(topicResp.Partitions, OffsetCommitPartitionResponse{
+				Index:     p.Index,
+				ErrorCode: errCode,
+			})
+		}
+
+		resp.Topics = append(resp.Topics, topicResp)
+	}
+
+	return h.buildOffsetCommitResponse(resp, apiVersion), nil
+}
+
+func (h *Handler) handleOffsetFetch(correlationID uint32, apiVersion uint16, requestBody []byte) ([]byte, error) {
+	// Parse OffsetFetch request
+	request, err := h.parseOffsetFetchRequest(requestBody)
+	if err != nil {
+		return h.buildOffsetFetchErrorResponse(correlationID, ErrorCodeInvalidGroupID), nil
+	}
+
+	// Validate request
+	if request.GroupID == "" {
+		return h.buildOffsetFetchErrorResponse(correlationID, ErrorCodeInvalidGroupID), nil
+	}
+
+	// Get or create consumer group
+	// IMPORTANT: Use GetOrCreateGroup (not GetGroup) to allow fetching persisted offsets
+	// even if the group doesn't exist in memory yet. This is critical for consumer restarts.
+	// Kafka allows offset fetches for groups that haven't joined yet (e.g., simple consumers).
+	group := h.groupCoordinator.GetOrCreateGroup(request.GroupID)
+
+	group.Mu.RLock()
+	defer group.Mu.RUnlock()
+
+	glog.V(4).Infof("[OFFSET_FETCH] Request: group=%s topics=%d", request.GroupID, len(request.Topics))
+
+	// Build response
+	response := OffsetFetchResponse{
+		CorrelationID: correlationID,
+		Topics:        make([]OffsetFetchTopicResponse, 0, len(request.Topics)),
+		ErrorCode:     ErrorCodeNone,
+	}
+
+	for _, topic := range request.Topics {
+		topicResponse := OffsetFetchTopicResponse{
+			Name:       topic.Name,
+			Partitions: make([]OffsetFetchPartitionResponse, 0),
+		}
+
+		// If no partitions specified, fetch all partitions for the topic
+		partitionsToFetch := topic.Partitions
+		if len(partitionsToFetch) == 0 {
+			// Get all partitions for this topic from group's offset commits
+			if topicOffsets, exists := group.OffsetCommits[topic.Name]; exists {
+				for partition := range topicOffsets {
+					partitionsToFetch = append(partitionsToFetch, partition)
+				}
+			}
+		}
+
+		// Fetch offsets for requested partitions
+		for _, partition := range partitionsToFetch {
+			var fetchedOffset int64 = -1
+			var metadata string = ""
+			var errorCode int16 = ErrorCodeNone
+
+			// Try fetching from in-memory cache first (works for both mock and SMQ backends)
+			if off, meta, err := h.fetchOffset(group, topic.Name, partition); err == nil && off >= 0 {
+				fetchedOffset = off
+				metadata = meta
+				glog.V(4).Infof("[OFFSET_FETCH] Found in memory: group=%s topic=%s partition=%d offset=%d",
+					request.GroupID, topic.Name, partition, off)
+			} else {
+				// Fallback: try fetching from SMQ persistent storage
+				// This handles cases where offsets are stored in SMQ but not yet loaded into memory
+				key := ConsumerOffsetKey{
+					Topic:                 topic.Name,
+					Partition:             partition,
+					ConsumerGroup:         request.GroupID,
+					ConsumerGroupInstance: request.GroupInstanceID,
+				}
+				if off, meta, err := h.fetchOffsetFromSMQ(key); err == nil && off >= 0 {
+					fetchedOffset = off
+					metadata = meta
+					glog.V(3).Infof("[OFFSET_FETCH] Found in storage: group=%s topic=%s partition=%d offset=%d",
+						request.GroupID, topic.Name, partition, off)
+				} else {
+					glog.V(3).Infof("[OFFSET_FETCH] No offset found: group=%s topic=%s partition=%d (will start from auto.offset.reset)",
+						request.GroupID, topic.Name, partition)
+				}
+				// No offset found in either location (-1 indicates no committed offset)
+			}
+
+			partitionResponse := OffsetFetchPartitionResponse{
+				Index:       partition,
+				Offset:      fetchedOffset,
+				LeaderEpoch: 0, // Default epoch for SeaweedMQ (single leader model)
+				Metadata:    metadata,
+				ErrorCode:   errorCode,
+			}
+			topicResponse.Partitions = append(topicResponse.Partitions, partitionResponse)
+		}
+
+		response.Topics = append(response.Topics, topicResponse)
+	}
+
+	return h.buildOffsetFetchResponse(response, apiVersion), nil
+}
+
+func (h *Handler) parseOffsetCommitRequest(data []byte, apiVersion uint16) (*OffsetCommitRequest, error) {
+	if len(data) < 8 {
+		return nil, fmt.Errorf("request too short")
+	}
+
+	offset := 0
+
+	// GroupID (string)
+	groupIDLength := int(binary.BigEndian.Uint16(data[offset:]))
+	offset += 2
+	if offset+groupIDLength > len(data) {
+		return nil, fmt.Errorf("invalid group ID length")
+	}
+	groupID := string(data[offset : offset+groupIDLength])
+	offset += groupIDLength
+
+	// Generation ID (4 bytes)
+	if offset+4 > len(data) {
+		return nil, fmt.Errorf("missing generation ID")
+	}
+	generationID := int32(binary.BigEndian.Uint32(data[offset:]))
+	offset += 4
+
+	// MemberID (string)
+	if offset+2 > len(data) {
+		return nil, fmt.Errorf("missing member ID length")
+	}
+	memberIDLength := int(binary.BigEndian.Uint16(data[offset:]))
+	offset += 2
+	if offset+memberIDLength > len(data) {
+		return nil, fmt.Errorf("invalid member ID length")
+	}
+	memberID := string(data[offset : offset+memberIDLength])
+	offset += memberIDLength
+
+	// RetentionTime (8 bytes) - exists in v0-v4, removed in v5+
+	var retentionTime int64 = -1
+	if apiVersion <= 4 {
+		if len(data) < offset+8 {
+			return nil, fmt.Errorf("missing retention time for v%d", apiVersion)
+		}
+		retentionTime = int64(binary.BigEndian.Uint64(data[offset : offset+8]))
+		offset += 8
+	}
+
+	// GroupInstanceID (nullable string) - ONLY in version 3+
+	var groupInstanceID string
+	if apiVersion >= 3 {
+		if offset+2 > len(data) {
+			return nil, fmt.Errorf("missing group instance ID length")
+		}
+		groupInstanceIDLength := int(int16(binary.BigEndian.Uint16(data[offset:])))
+		offset += 2
+		if groupInstanceIDLength == -1 {
+			// Null string
+			groupInstanceID = ""
+		} else if groupInstanceIDLength > 0 {
+			if offset+groupInstanceIDLength > len(data) {
+				return nil, fmt.Errorf("invalid group instance ID length")
+			}
+			groupInstanceID = string(data[offset : offset+groupInstanceIDLength])
+			offset += groupInstanceIDLength
+		}
+	}
+
+	// Topics array
+	var topicsCount uint32
+	if len(data) >= offset+4 {
+		topicsCount = binary.BigEndian.Uint32(data[offset : offset+4])
+		offset += 4
+	}
+
+	topics := make([]OffsetCommitTopic, 0, topicsCount)
+
+	for i := uint32(0); i < topicsCount && offset < len(data); i++ {
+		// Parse topic name
+		if len(data) < offset+2 {
+			break
+		}
+		topicNameLength := binary.BigEndian.Uint16(data[offset : offset+2])
+		offset += 2
+
+		if len(data) < offset+int(topicNameLength) {
+			break
+		}
+		topicName := string(data[offset : offset+int(topicNameLength)])
+		offset += int(topicNameLength)
+
+		// Parse partitions array
+		if len(data) < offset+4 {
+			break
+		}
+		partitionsCount := binary.BigEndian.Uint32(data[offset : offset+4])
+		offset += 4
+
+		partitions := make([]OffsetCommitPartition, 0, partitionsCount)
+
+		for j := uint32(0); j < partitionsCount && offset < len(data); j++ {
+			// Parse partition index (4 bytes)
+			if len(data) < offset+4 {
+				break
+			}
+			partitionIndex := int32(binary.BigEndian.Uint32(data[offset : offset+4]))
+			offset += 4
+
+			// Parse committed offset (8 bytes)
+			if len(data) < offset+8 {
+				break
+			}
+			committedOffset := int64(binary.BigEndian.Uint64(data[offset : offset+8]))
+			offset += 8
+
+			// Parse leader epoch (4 bytes) - ONLY in version 6+
+			var leaderEpoch int32 = -1
+			if apiVersion >= 6 {
+				if len(data) < offset+4 {
+					break
+				}
+				leaderEpoch = int32(binary.BigEndian.Uint32(data[offset : offset+4]))
+				offset += 4
+			}
+
+			// Parse metadata (string)
+			var metadata string = ""
+			if len(data) >= offset+2 {
+				metadataLength := int16(binary.BigEndian.Uint16(data[offset : offset+2]))
+				offset += 2
+				if metadataLength == -1 {
+					metadata = ""
+				} else if metadataLength >= 0 && len(data) >= offset+int(metadataLength) {
+					metadata = string(data[offset : offset+int(metadataLength)])
+					offset += int(metadataLength)
+				}
+			}
+
+			partitions = append(partitions, OffsetCommitPartition{
+				Index:       partitionIndex,
+				Offset:      committedOffset,
+				LeaderEpoch: leaderEpoch,
+				Metadata:    metadata,
+			})
+		}
+		topics = append(topics, OffsetCommitTopic{
+			Name:       topicName,
+			Partitions: partitions,
+		})
+	}
+
+	return &OffsetCommitRequest{
+		GroupID:         groupID,
+		GenerationID:    generationID,
+		MemberID:        memberID,
+		GroupInstanceID: groupInstanceID,
+		RetentionTime:   retentionTime,
+		Topics:          topics,
+	}, nil
+}
+
+func (h *Handler) parseOffsetFetchRequest(data []byte) (*OffsetFetchRequest, error) {
+	if len(data) < 4 {
+		return nil, fmt.Errorf("request too short")
+	}
+
+	offset := 0
+
+	// GroupID (string)
+	groupIDLength := int(binary.BigEndian.Uint16(data[offset:]))
+	offset += 2
+	if offset+groupIDLength > len(data) {
+		return nil, fmt.Errorf("invalid group ID length")
+	}
+	groupID := string(data[offset : offset+groupIDLength])
+	offset += groupIDLength
+
+	// Parse Topics array - classic encoding (INT32 count) for v0-v5
+	if len(data) < offset+4 {
+		return nil, fmt.Errorf("OffsetFetch request missing topics array")
+	}
+	topicsCount := binary.BigEndian.Uint32(data[offset : offset+4])
+	offset += 4
+
+	topics := make([]OffsetFetchTopic, 0, topicsCount)
+
+	for i := uint32(0); i < topicsCount && offset < len(data); i++ {
+		// Parse topic name (STRING: INT16 length + bytes)
+		if len(data) < offset+2 {
+			break
+		}
+		topicNameLength := binary.BigEndian.Uint16(data[offset : offset+2])
+		offset += 2
+
+		if len(data) < offset+int(topicNameLength) {
+			break
+		}
+		topicName := string(data[offset : offset+int(topicNameLength)])
+		offset += int(topicNameLength)
+
+		// Parse partitions array (ARRAY: INT32 count)
+		if len(data) < offset+4 {
+			break
+		}
+		partitionsCount := binary.BigEndian.Uint32(data[offset : offset+4])
+		offset += 4
+
+		partitions := make([]int32, 0, partitionsCount)
+
+		// If partitionsCount is 0, it means "fetch all partitions"
+		if partitionsCount == 0 {
+			partitions = nil // nil means all partitions
+		} else {
+			for j := uint32(0); j < partitionsCount && offset < len(data); j++ {
+				// Parse partition index (4 bytes)
+				if len(data) < offset+4 {
+					break
+				}
+				partitionIndex := int32(binary.BigEndian.Uint32(data[offset : offset+4]))
+				offset += 4
+
+				partitions = append(partitions, partitionIndex)
+			}
+		}
+
+		topics = append(topics, OffsetFetchTopic{
+			Name:       topicName,
+			Partitions: partitions,
+		})
+	}
+
+	// Parse RequireStable flag (1 byte) - for transactional consistency
+	var requireStable bool
+	if len(data) >= offset+1 {
+		requireStable = data[offset] != 0
+		offset += 1
+	}
+
+	return &OffsetFetchRequest{
+		GroupID:       groupID,
+		Topics:        topics,
+		RequireStable: requireStable,
+	}, nil
+}
+
+func (h *Handler) commitOffset(group *consumer.ConsumerGroup, topic string, partition int32, offset int64, metadata string) error {
+	// Initialize topic offsets if needed
+	if group.OffsetCommits == nil {
+		group.OffsetCommits = make(map[string]map[int32]consumer.OffsetCommit)
+	}
+
+	if group.OffsetCommits[topic] == nil {
+		group.OffsetCommits[topic] = make(map[int32]consumer.OffsetCommit)
+	}
+
+	// Store the offset commit
+	group.OffsetCommits[topic][partition] = consumer.OffsetCommit{
+		Offset:    offset,
+		Metadata:  metadata,
+		Timestamp: time.Now(),
+	}
+
+	return nil
+}
+
+func (h *Handler) fetchOffset(group *consumer.ConsumerGroup, topic string, partition int32) (int64, string, error) {
+	// Check if topic exists in offset commits
+	if group.OffsetCommits == nil {
+		return -1, "", nil // No committed offset
+	}
+
+	topicOffsets, exists := group.OffsetCommits[topic]
+	if !exists {
+		return -1, "", nil // No committed offset for topic
+	}
+
+	offsetCommit, exists := topicOffsets[partition]
+	if !exists {
+		return -1, "", nil // No committed offset for partition
+	}
+
+	return offsetCommit.Offset, offsetCommit.Metadata, nil
+}
+
+func (h *Handler) buildOffsetCommitResponse(response OffsetCommitResponse, apiVersion uint16) []byte {
+	estimatedSize := 16
+	for _, topic := range response.Topics {
+		estimatedSize += len(topic.Name) + 8 + len(topic.Partitions)*8
+	}
+
+	result := make([]byte, 0, estimatedSize)
+
+	// NOTE: Correlation ID is handled by writeResponseWithCorrelationID
+	// Do NOT include it in the response body
+
+	// Throttle time (4 bytes) - ONLY for version 3+, and it goes at the BEGINNING
+	if apiVersion >= 3 {
+		result = append(result, 0, 0, 0, 0) // throttle_time_ms = 0
+	}
+
+	// Topics array length (4 bytes)
+	topicsLengthBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(topicsLengthBytes, uint32(len(response.Topics)))
+	result = append(result, topicsLengthBytes...)
+
+	// Topics
+	for _, topic := range response.Topics {
+		// Topic name length (2 bytes)
+		nameLength := make([]byte, 2)
+		binary.BigEndian.PutUint16(nameLength, uint16(len(topic.Name)))
+		result = append(result, nameLength...)
+
+		// Topic name
+		result = append(result, []byte(topic.Name)...)
+
+		// Partitions array length (4 bytes)
+		partitionsLength := make([]byte, 4)
+		binary.BigEndian.PutUint32(partitionsLength, uint32(len(topic.Partitions)))
+		result = append(result, partitionsLength...)
+
+		// Partitions
+		for _, partition := range topic.Partitions {
+			// Partition index (4 bytes)
+			indexBytes := make([]byte, 4)
+			binary.BigEndian.PutUint32(indexBytes, uint32(partition.Index))
+			result = append(result, indexBytes...)
+
+			// Error code (2 bytes)
+			errorBytes := make([]byte, 2)
+			binary.BigEndian.PutUint16(errorBytes, uint16(partition.ErrorCode))
+			result = append(result, errorBytes...)
+		}
+	}
+
+	return result
+}
+
+func (h *Handler) buildOffsetFetchResponse(response OffsetFetchResponse, apiVersion uint16) []byte {
+	estimatedSize := 32
+	for _, topic := range response.Topics {
+		estimatedSize += len(topic.Name) + 16 + len(topic.Partitions)*32
+		for _, partition := range topic.Partitions {
+			estimatedSize += len(partition.Metadata)
+		}
+	}
+
+	result := make([]byte, 0, estimatedSize)
+
+	// NOTE: Correlation ID is handled by writeResponseWithCorrelationID
+	// Do NOT include it in the response body
+
+	// Throttle time (4 bytes) - for version 3+ this appears immediately after correlation ID
+	if apiVersion >= 3 {
+		result = append(result, 0, 0, 0, 0) // throttle_time_ms = 0
+	}
+
+	// Topics array length (4 bytes)
+	topicsLengthBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(topicsLengthBytes, uint32(len(response.Topics)))
+	result = append(result, topicsLengthBytes...)
+
+	// Topics
+	for _, topic := range response.Topics {
+		// Topic name length (2 bytes)
+		nameLength := make([]byte, 2)
+		binary.BigEndian.PutUint16(nameLength, uint16(len(topic.Name)))
+		result = append(result, nameLength...)
+
+		// Topic name
+		result = append(result, []byte(topic.Name)...)
+
+		// Partitions array length (4 bytes)
+		partitionsLength := make([]byte, 4)
+		binary.BigEndian.PutUint32(partitionsLength, uint32(len(topic.Partitions)))
+		result = append(result, partitionsLength...)
+
+		// Partitions
+		for _, partition := range topic.Partitions {
+			// Partition index (4 bytes)
+			indexBytes := make([]byte, 4)
+			binary.BigEndian.PutUint32(indexBytes, uint32(partition.Index))
+			result = append(result, indexBytes...)
+
+			// Committed offset (8 bytes)
+			offsetBytes := make([]byte, 8)
+			binary.BigEndian.PutUint64(offsetBytes, uint64(partition.Offset))
+			result = append(result, offsetBytes...)
+
+			// Leader epoch (4 bytes) - only included in version 5+
+			if apiVersion >= 5 {
+				epochBytes := make([]byte, 4)
+				binary.BigEndian.PutUint32(epochBytes, uint32(partition.LeaderEpoch))
+				result = append(result, epochBytes...)
+			}
+
+			// Metadata length (2 bytes)
+			metadataLength := make([]byte, 2)
+			binary.BigEndian.PutUint16(metadataLength, uint16(len(partition.Metadata)))
+			result = append(result, metadataLength...)
+
+			// Metadata
+			result = append(result, []byte(partition.Metadata)...)
+
+			// Error code (2 bytes)
+			errorBytes := make([]byte, 2)
+			binary.BigEndian.PutUint16(errorBytes, uint16(partition.ErrorCode))
+			result = append(result, errorBytes...)
+		}
+	}
+
+	// Group-level error code (2 bytes) - only included in version 2+
+	if apiVersion >= 2 {
+		groupErrorBytes := make([]byte, 2)
+		binary.BigEndian.PutUint16(groupErrorBytes, uint16(response.ErrorCode))
+		result = append(result, groupErrorBytes...)
+	}
+
+	return result
+}
+
+func (h *Handler) buildOffsetCommitErrorResponse(correlationID uint32, errorCode int16, apiVersion uint16) []byte {
+	response := OffsetCommitResponse{
+		CorrelationID: correlationID,
+		Topics: []OffsetCommitTopicResponse{
+			{
+				Name: "",
+				Partitions: []OffsetCommitPartitionResponse{
+					{Index: 0, ErrorCode: errorCode},
+				},
+			},
+		},
+	}
+
+	return h.buildOffsetCommitResponse(response, apiVersion)
+}
+
+func (h *Handler) buildOffsetFetchErrorResponse(correlationID uint32, errorCode int16) []byte {
+	response := OffsetFetchResponse{
+		CorrelationID: correlationID,
+		Topics:        []OffsetFetchTopicResponse{},
+		ErrorCode:     errorCode,
+	}
+
+	return h.buildOffsetFetchResponse(response, 0)
+}

weed/mq/kafka/protocol/offset_storage_adapter.go

@@ -0,0 +1,49 @@
+package protocol
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/consumer_offset"
+)
+
+// offsetStorageAdapter adapts consumer_offset.OffsetStorage to ConsumerOffsetStorage interface
+type offsetStorageAdapter struct {
+	storage consumer_offset.OffsetStorage
+}
+
+// newOffsetStorageAdapter creates a new adapter
+func newOffsetStorageAdapter(storage consumer_offset.OffsetStorage) ConsumerOffsetStorage {
+	return &offsetStorageAdapter{storage: storage}
+}
+
+func (a *offsetStorageAdapter) CommitOffset(group, topic string, partition int32, offset int64, metadata string) error {
+	return a.storage.CommitOffset(group, topic, partition, offset, metadata)
+}
+
+func (a *offsetStorageAdapter) FetchOffset(group, topic string, partition int32) (int64, string, error) {
+	return a.storage.FetchOffset(group, topic, partition)
+}
+
+func (a *offsetStorageAdapter) FetchAllOffsets(group string) (map[TopicPartition]OffsetMetadata, error) {
+	offsets, err := a.storage.FetchAllOffsets(group)
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert from consumer_offset types to protocol types
+	result := make(map[TopicPartition]OffsetMetadata, len(offsets))
+	for tp, om := range offsets {
+		result[TopicPartition{Topic: tp.Topic, Partition: tp.Partition}] = OffsetMetadata{
+			Offset:   om.Offset,
+			Metadata: om.Metadata,
+		}
+	}
+
+	return result, nil
+}
+
+func (a *offsetStorageAdapter) DeleteGroup(group string) error {
+	return a.storage.DeleteGroup(group)
+}
+
+func (a *offsetStorageAdapter) Close() error {
+	return a.storage.Close()
+}

weed/mq/kafka/protocol/record_batch_parser.go

@@ -0,0 +1,290 @@
+package protocol
+
+import (
+	"encoding/binary"
+	"fmt"
+	"hash/crc32"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/compression"
+)
+
+// RecordBatch represents a parsed Kafka record batch
+type RecordBatch struct {
+	BaseOffset           int64
+	BatchLength          int32
+	PartitionLeaderEpoch int32
+	Magic                int8
+	CRC32                uint32
+	Attributes           int16
+	LastOffsetDelta      int32
+	FirstTimestamp       int64
+	MaxTimestamp         int64
+	ProducerID           int64
+	ProducerEpoch        int16
+	BaseSequence         int32
+	RecordCount          int32
+	Records              []byte // Raw records data (may be compressed)
+}
+
+// RecordBatchParser handles parsing of Kafka record batches with compression support
+type RecordBatchParser struct {
+	// Add any configuration or state needed
+}
+
+// NewRecordBatchParser creates a new record batch parser
+func NewRecordBatchParser() *RecordBatchParser {
+	return &RecordBatchParser{}
+}
+
+// ParseRecordBatch parses a Kafka record batch from binary data
+func (p *RecordBatchParser) ParseRecordBatch(data []byte) (*RecordBatch, error) {
+	if len(data) < 61 { // Minimum record batch header size
+		return nil, fmt.Errorf("record batch too small: %d bytes, need at least 61", len(data))
+	}
+
+	batch := &RecordBatch{}
+	offset := 0
+
+	// Parse record batch header
+	batch.BaseOffset = int64(binary.BigEndian.Uint64(data[offset:]))
+	offset += 8
+
+	batch.BatchLength = int32(binary.BigEndian.Uint32(data[offset:]))
+	offset += 4
+
+	batch.PartitionLeaderEpoch = int32(binary.BigEndian.Uint32(data[offset:]))
+	offset += 4
+
+	batch.Magic = int8(data[offset])
+	offset += 1
+
+	// Validate magic byte
+	if batch.Magic != 2 {
+		return nil, fmt.Errorf("unsupported record batch magic byte: %d, expected 2", batch.Magic)
+	}
+
+	batch.CRC32 = binary.BigEndian.Uint32(data[offset:])
+	offset += 4
+
+	batch.Attributes = int16(binary.BigEndian.Uint16(data[offset:]))
+	offset += 2
+
+	batch.LastOffsetDelta = int32(binary.BigEndian.Uint32(data[offset:]))
+	offset += 4
+
+	batch.FirstTimestamp = int64(binary.BigEndian.Uint64(data[offset:]))
+	offset += 8
+
+	batch.MaxTimestamp = int64(binary.BigEndian.Uint64(data[offset:]))
+	offset += 8
+
+	batch.ProducerID = int64(binary.BigEndian.Uint64(data[offset:]))
+	offset += 8
+
+	batch.ProducerEpoch = int16(binary.BigEndian.Uint16(data[offset:]))
+	offset += 2
+
+	batch.BaseSequence = int32(binary.BigEndian.Uint32(data[offset:]))
+	offset += 4
+
+	batch.RecordCount = int32(binary.BigEndian.Uint32(data[offset:]))
+	offset += 4
+
+	// Validate record count
+	if batch.RecordCount < 0 || batch.RecordCount > 1000000 {
+		return nil, fmt.Errorf("invalid record count: %d", batch.RecordCount)
+	}
+
+	// Extract records data (rest of the batch)
+	if offset < len(data) {
+		batch.Records = data[offset:]
+	}
+
+	return batch, nil
+}
+
+// GetCompressionCodec extracts the compression codec from the batch attributes
+func (batch *RecordBatch) GetCompressionCodec() compression.CompressionCodec {
+	return compression.ExtractCompressionCodec(batch.Attributes)
+}
+
+// IsCompressed returns true if the record batch is compressed
+func (batch *RecordBatch) IsCompressed() bool {
+	return batch.GetCompressionCodec() != compression.None
+}
+
+// DecompressRecords decompresses the records data if compressed
+func (batch *RecordBatch) DecompressRecords() ([]byte, error) {
+	if !batch.IsCompressed() {
+		return batch.Records, nil
+	}
+
+	codec := batch.GetCompressionCodec()
+	decompressed, err := compression.Decompress(codec, batch.Records)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decompress records with %s: %w", codec, err)
+	}
+
+	return decompressed, nil
+}
+
+// ValidateCRC32 validates the CRC32 checksum of the record batch
+func (batch *RecordBatch) ValidateCRC32(originalData []byte) error {
+	if len(originalData) < 17 { // Need at least up to CRC field
+		return fmt.Errorf("data too small for CRC validation")
+	}
+
+	// CRC32 is calculated over the data starting after the CRC field
+	// Skip: BaseOffset(8) + BatchLength(4) + PartitionLeaderEpoch(4) + Magic(1) + CRC(4) = 21 bytes
+	// Kafka uses Castagnoli (CRC-32C) algorithm for record batch CRC
+	dataForCRC := originalData[21:]
+
+	calculatedCRC := crc32.Checksum(dataForCRC, crc32.MakeTable(crc32.Castagnoli))
+
+	if calculatedCRC != batch.CRC32 {
+		return fmt.Errorf("CRC32 mismatch: expected %x, got %x", batch.CRC32, calculatedCRC)
+	}
+
+	return nil
+}
+
+// ParseRecordBatchWithValidation parses and validates a record batch
+func (p *RecordBatchParser) ParseRecordBatchWithValidation(data []byte, validateCRC bool) (*RecordBatch, error) {
+	batch, err := p.ParseRecordBatch(data)
+	if err != nil {
+		return nil, err
+	}
+
+	if validateCRC {
+		if err := batch.ValidateCRC32(data); err != nil {
+			return nil, fmt.Errorf("CRC validation failed: %w", err)
+		}
+	}
+
+	return batch, nil
+}
+
+// ExtractRecords extracts and decompresses individual records from the batch
+func (batch *RecordBatch) ExtractRecords() ([]Record, error) {
+	decompressedData, err := batch.DecompressRecords()
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse individual records from decompressed data
+	// This is a simplified implementation - full implementation would parse varint-encoded records
+	records := make([]Record, 0, batch.RecordCount)
+
+	// For now, create placeholder records
+	// In a full implementation, this would parse the actual record format
+	for i := int32(0); i < batch.RecordCount; i++ {
+		record := Record{
+			Offset:    batch.BaseOffset + int64(i),
+			Key:       nil,                             // Would be parsed from record data
+			Value:     decompressedData,                // Simplified - would be individual record value
+			Headers:   nil,                             // Would be parsed from record data
+			Timestamp: batch.FirstTimestamp + int64(i), // Simplified
+		}
+		records = append(records, record)
+	}
+
+	return records, nil
+}
+
+// Record represents a single Kafka record
+type Record struct {
+	Offset    int64
+	Key       []byte
+	Value     []byte
+	Headers   map[string][]byte
+	Timestamp int64
+}
+
+// CompressRecordBatch compresses a record batch using the specified codec
+func CompressRecordBatch(codec compression.CompressionCodec, records []byte) ([]byte, int16, error) {
+	if codec == compression.None {
+		return records, 0, nil
+	}
+
+	compressed, err := compression.Compress(codec, records)
+	if err != nil {
+		return nil, 0, fmt.Errorf("failed to compress record batch: %w", err)
+	}
+
+	attributes := compression.SetCompressionCodec(0, codec)
+	return compressed, attributes, nil
+}
+
+// CreateRecordBatch creates a new record batch with the given parameters
+func CreateRecordBatch(baseOffset int64, records []byte, codec compression.CompressionCodec) ([]byte, error) {
+	// Compress records if needed
+	compressedRecords, attributes, err := CompressRecordBatch(codec, records)
+	if err != nil {
+		return nil, err
+	}
+
+	// Calculate batch length (everything after the batch length field)
+	recordsLength := len(compressedRecords)
+	batchLength := 4 + 1 + 4 + 2 + 4 + 8 + 8 + 8 + 2 + 4 + 4 + recordsLength // Header + records
+
+	// Build the record batch
+	batch := make([]byte, 0, 61+recordsLength)
+
+	// Base offset (8 bytes)
+	baseOffsetBytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(baseOffsetBytes, uint64(baseOffset))
+	batch = append(batch, baseOffsetBytes...)
+
+	// Batch length (4 bytes)
+	batchLengthBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(batchLengthBytes, uint32(batchLength))
+	batch = append(batch, batchLengthBytes...)
+
+	// Partition leader epoch (4 bytes) - use 0 for simplicity
+	batch = append(batch, 0, 0, 0, 0)
+
+	// Magic byte (1 byte) - version 2
+	batch = append(batch, 2)
+
+	// CRC32 placeholder (4 bytes) - will be calculated later
+	crcPos := len(batch)
+	batch = append(batch, 0, 0, 0, 0)
+
+	// Attributes (2 bytes)
+	attributesBytes := make([]byte, 2)
+	binary.BigEndian.PutUint16(attributesBytes, uint16(attributes))
+	batch = append(batch, attributesBytes...)
+
+	// Last offset delta (4 bytes) - assume single record for simplicity
+	batch = append(batch, 0, 0, 0, 0)
+
+	// First timestamp (8 bytes) - use current time
+	// For simplicity, use 0
+	batch = append(batch, 0, 0, 0, 0, 0, 0, 0, 0)
+
+	// Max timestamp (8 bytes)
+	batch = append(batch, 0, 0, 0, 0, 0, 0, 0, 0)
+
+	// Producer ID (8 bytes) - use -1 for non-transactional
+	batch = append(batch, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF)
+
+	// Producer epoch (2 bytes) - use -1
+	batch = append(batch, 0xFF, 0xFF)
+
+	// Base sequence (4 bytes) - use -1
+	batch = append(batch, 0xFF, 0xFF, 0xFF, 0xFF)
+
+	// Record count (4 bytes) - assume 1 for simplicity
+	batch = append(batch, 0, 0, 0, 1)
+
+	// Records data
+	batch = append(batch, compressedRecords...)
+
+	// Calculate and set CRC32
+	// Kafka uses Castagnoli (CRC-32C) algorithm for record batch CRC
+	dataForCRC := batch[21:] // Everything after CRC field
+	crc := crc32.Checksum(dataForCRC, crc32.MakeTable(crc32.Castagnoli))
+	binary.BigEndian.PutUint32(batch[crcPos:crcPos+4], crc)
+
+	return batch, nil
+}

weed/mq/kafka/protocol/record_batch_parser_test.go

@@ -0,0 +1,292 @@
+package protocol
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/compression"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestRecordBatchParser_ParseRecordBatch tests basic record batch parsing
+func TestRecordBatchParser_ParseRecordBatch(t *testing.T) {
+	parser := NewRecordBatchParser()
+
+	// Create a minimal valid record batch
+	recordData := []byte("test record data")
+	batch, err := CreateRecordBatch(100, recordData, compression.None)
+	require.NoError(t, err)
+
+	// Parse the batch
+	parsed, err := parser.ParseRecordBatch(batch)
+	require.NoError(t, err)
+
+	// Verify parsed fields
+	assert.Equal(t, int64(100), parsed.BaseOffset)
+	assert.Equal(t, int8(2), parsed.Magic)
+	assert.Equal(t, int32(1), parsed.RecordCount)
+	assert.Equal(t, compression.None, parsed.GetCompressionCodec())
+	assert.False(t, parsed.IsCompressed())
+}
+
+// TestRecordBatchParser_ParseRecordBatch_TooSmall tests parsing with insufficient data
+func TestRecordBatchParser_ParseRecordBatch_TooSmall(t *testing.T) {
+	parser := NewRecordBatchParser()
+
+	// Test with data that's too small
+	smallData := make([]byte, 30) // Less than 61 bytes minimum
+	_, err := parser.ParseRecordBatch(smallData)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "record batch too small")
+}
+
+// TestRecordBatchParser_ParseRecordBatch_InvalidMagic tests parsing with invalid magic byte
+func TestRecordBatchParser_ParseRecordBatch_InvalidMagic(t *testing.T) {
+	parser := NewRecordBatchParser()
+
+	// Create a batch with invalid magic byte
+	recordData := []byte("test record data")
+	batch, err := CreateRecordBatch(100, recordData, compression.None)
+	require.NoError(t, err)
+
+	// Corrupt the magic byte (at offset 16)
+	batch[16] = 1 // Invalid magic byte
+
+	// Parse should fail
+	_, err = parser.ParseRecordBatch(batch)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "unsupported record batch magic byte")
+}
+
+// TestRecordBatchParser_Compression tests compression support
+func TestRecordBatchParser_Compression(t *testing.T) {
+	parser := NewRecordBatchParser()
+	recordData := []byte("This is a test record that should compress well when repeated. " +
+		"This is a test record that should compress well when repeated. " +
+		"This is a test record that should compress well when repeated.")
+
+	codecs := []compression.CompressionCodec{
+		compression.None,
+		compression.Gzip,
+		compression.Snappy,
+		compression.Lz4,
+		compression.Zstd,
+	}
+
+	for _, codec := range codecs {
+		t.Run(codec.String(), func(t *testing.T) {
+			// Create compressed batch
+			batch, err := CreateRecordBatch(200, recordData, codec)
+			require.NoError(t, err)
+
+			// Parse the batch
+			parsed, err := parser.ParseRecordBatch(batch)
+			require.NoError(t, err)
+
+			// Verify compression codec
+			assert.Equal(t, codec, parsed.GetCompressionCodec())
+			assert.Equal(t, codec != compression.None, parsed.IsCompressed())
+
+			// Decompress and verify data
+			decompressed, err := parsed.DecompressRecords()
+			require.NoError(t, err)
+			assert.Equal(t, recordData, decompressed)
+		})
+	}
+}
+
+// TestRecordBatchParser_CRCValidation tests CRC32 validation
+func TestRecordBatchParser_CRCValidation(t *testing.T) {
+	parser := NewRecordBatchParser()
+	recordData := []byte("test record for CRC validation")
+
+	// Create a valid batch
+	batch, err := CreateRecordBatch(300, recordData, compression.None)
+	require.NoError(t, err)
+
+	t.Run("Valid CRC", func(t *testing.T) {
+		// Parse with CRC validation should succeed
+		parsed, err := parser.ParseRecordBatchWithValidation(batch, true)
+		require.NoError(t, err)
+		assert.Equal(t, int64(300), parsed.BaseOffset)
+	})
+
+	t.Run("Invalid CRC", func(t *testing.T) {
+		// Corrupt the CRC field
+		corruptedBatch := make([]byte, len(batch))
+		copy(corruptedBatch, batch)
+		corruptedBatch[17] = 0xFF // Corrupt CRC
+
+		// Parse with CRC validation should fail
+		_, err := parser.ParseRecordBatchWithValidation(corruptedBatch, true)
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "CRC validation failed")
+	})
+
+	t.Run("Skip CRC validation", func(t *testing.T) {
+		// Corrupt the CRC field
+		corruptedBatch := make([]byte, len(batch))
+		copy(corruptedBatch, batch)
+		corruptedBatch[17] = 0xFF // Corrupt CRC
+
+		// Parse without CRC validation should succeed
+		parsed, err := parser.ParseRecordBatchWithValidation(corruptedBatch, false)
+		require.NoError(t, err)
+		assert.Equal(t, int64(300), parsed.BaseOffset)
+	})
+}
+
+// TestRecordBatchParser_ExtractRecords tests record extraction
+func TestRecordBatchParser_ExtractRecords(t *testing.T) {
+	parser := NewRecordBatchParser()
+	recordData := []byte("test record data for extraction")
+
+	// Create a batch
+	batch, err := CreateRecordBatch(400, recordData, compression.Gzip)
+	require.NoError(t, err)
+
+	// Parse the batch
+	parsed, err := parser.ParseRecordBatch(batch)
+	require.NoError(t, err)
+
+	// Extract records
+	records, err := parsed.ExtractRecords()
+	require.NoError(t, err)
+
+	// Verify extracted records (simplified implementation returns 1 record)
+	assert.Len(t, records, 1)
+	assert.Equal(t, int64(400), records[0].Offset)
+	assert.Equal(t, recordData, records[0].Value)
+}
+
+// TestCompressRecordBatch tests the compression helper function
+func TestCompressRecordBatch(t *testing.T) {
+	recordData := []byte("test data for compression")
+
+	t.Run("No compression", func(t *testing.T) {
+		compressed, attributes, err := CompressRecordBatch(compression.None, recordData)
+		require.NoError(t, err)
+		assert.Equal(t, recordData, compressed)
+		assert.Equal(t, int16(0), attributes)
+	})
+
+	t.Run("Gzip compression", func(t *testing.T) {
+		compressed, attributes, err := CompressRecordBatch(compression.Gzip, recordData)
+		require.NoError(t, err)
+		assert.NotEqual(t, recordData, compressed)
+		assert.Equal(t, int16(1), attributes)
+
+		// Verify we can decompress
+		decompressed, err := compression.Decompress(compression.Gzip, compressed)
+		require.NoError(t, err)
+		assert.Equal(t, recordData, decompressed)
+	})
+}
+
+// TestCreateRecordBatch tests record batch creation
+func TestCreateRecordBatch(t *testing.T) {
+	recordData := []byte("test record data")
+	baseOffset := int64(500)
+
+	t.Run("Uncompressed batch", func(t *testing.T) {
+		batch, err := CreateRecordBatch(baseOffset, recordData, compression.None)
+		require.NoError(t, err)
+		assert.True(t, len(batch) >= 61) // Minimum header size
+
+		// Parse and verify
+		parser := NewRecordBatchParser()
+		parsed, err := parser.ParseRecordBatch(batch)
+		require.NoError(t, err)
+		assert.Equal(t, baseOffset, parsed.BaseOffset)
+		assert.Equal(t, compression.None, parsed.GetCompressionCodec())
+	})
+
+	t.Run("Compressed batch", func(t *testing.T) {
+		batch, err := CreateRecordBatch(baseOffset, recordData, compression.Snappy)
+		require.NoError(t, err)
+		assert.True(t, len(batch) >= 61) // Minimum header size
+
+		// Parse and verify
+		parser := NewRecordBatchParser()
+		parsed, err := parser.ParseRecordBatch(batch)
+		require.NoError(t, err)
+		assert.Equal(t, baseOffset, parsed.BaseOffset)
+		assert.Equal(t, compression.Snappy, parsed.GetCompressionCodec())
+		assert.True(t, parsed.IsCompressed())
+
+		// Verify decompression works
+		decompressed, err := parsed.DecompressRecords()
+		require.NoError(t, err)
+		assert.Equal(t, recordData, decompressed)
+	})
+}
+
+// TestRecordBatchParser_InvalidRecordCount tests handling of invalid record counts
+func TestRecordBatchParser_InvalidRecordCount(t *testing.T) {
+	parser := NewRecordBatchParser()
+
+	// Create a valid batch first
+	recordData := []byte("test record data")
+	batch, err := CreateRecordBatch(100, recordData, compression.None)
+	require.NoError(t, err)
+
+	// Corrupt the record count field (at offset 57-60)
+	// Set to a very large number
+	batch[57] = 0xFF
+	batch[58] = 0xFF
+	batch[59] = 0xFF
+	batch[60] = 0xFF
+
+	// Parse should fail
+	_, err = parser.ParseRecordBatch(batch)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "invalid record count")
+}
+
+// BenchmarkRecordBatchParser tests parsing performance
+func BenchmarkRecordBatchParser(b *testing.B) {
+	parser := NewRecordBatchParser()
+	recordData := make([]byte, 1024) // 1KB record
+	for i := range recordData {
+		recordData[i] = byte(i % 256)
+	}
+
+	codecs := []compression.CompressionCodec{
+		compression.None,
+		compression.Gzip,
+		compression.Snappy,
+		compression.Lz4,
+		compression.Zstd,
+	}
+
+	for _, codec := range codecs {
+		batch, err := CreateRecordBatch(0, recordData, codec)
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		b.Run("Parse_"+codec.String(), func(b *testing.B) {
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				_, err := parser.ParseRecordBatch(batch)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+
+		b.Run("Decompress_"+codec.String(), func(b *testing.B) {
+			parsed, err := parser.ParseRecordBatch(batch)
+			if err != nil {
+				b.Fatal(err)
+			}
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				_, err := parsed.DecompressRecords()
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

weed/mq/kafka/protocol/record_extraction_test.go

@@ -0,0 +1,158 @@
+package protocol
+
+import (
+	"encoding/binary"
+	"hash/crc32"
+	"testing"
+)
+
+// TestExtractAllRecords_RealKafkaFormat tests extracting records from a real Kafka v2 record batch
+func TestExtractAllRecords_RealKafkaFormat(t *testing.T) {
+	h := &Handler{} // Minimal handler for testing
+
+	// Create a proper Kafka v2 record batch with 1 record
+	// This mimics what Schema Registry or other Kafka clients would send
+
+	// Build record batch header (61 bytes)
+	batch := make([]byte, 0, 200)
+
+	// BaseOffset (8 bytes)
+	baseOffset := make([]byte, 8)
+	binary.BigEndian.PutUint64(baseOffset, 0)
+	batch = append(batch, baseOffset...)
+
+	// BatchLength (4 bytes) - will set after we know total size
+	batchLengthPos := len(batch)
+	batch = append(batch, 0, 0, 0, 0)
+
+	// PartitionLeaderEpoch (4 bytes)
+	batch = append(batch, 0, 0, 0, 0)
+
+	// Magic (1 byte) - must be 2 for v2
+	batch = append(batch, 2)
+
+	// CRC32 (4 bytes) - will calculate and set later
+	crcPos := len(batch)
+	batch = append(batch, 0, 0, 0, 0)
+
+	// Attributes (2 bytes) - no compression
+	batch = append(batch, 0, 0)
+
+	// LastOffsetDelta (4 bytes)
+	batch = append(batch, 0, 0, 0, 0)
+
+	// FirstTimestamp (8 bytes)
+	batch = append(batch, 0, 0, 0, 0, 0, 0, 0, 0)
+
+	// MaxTimestamp (8 bytes)
+	batch = append(batch, 0, 0, 0, 0, 0, 0, 0, 0)
+
+	// ProducerID (8 bytes)
+	batch = append(batch, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF)
+
+	// ProducerEpoch (2 bytes)
+	batch = append(batch, 0xFF, 0xFF)
+
+	// BaseSequence (4 bytes)
+	batch = append(batch, 0xFF, 0xFF, 0xFF, 0xFF)
+
+	// RecordCount (4 bytes)
+	batch = append(batch, 0, 0, 0, 1) // 1 record
+
+	// Now add the actual record (varint-encoded)
+	// Record format:
+	// - length (signed zigzag varint)
+	// - attributes (1 byte)
+	// - timestampDelta (signed zigzag varint)
+	// - offsetDelta (signed zigzag varint)
+	// - keyLength (signed zigzag varint, -1 for null)
+	// - key (bytes)
+	// - valueLength (signed zigzag varint, -1 for null)
+	// - value (bytes)
+	// - headersCount (signed zigzag varint)
+
+	record := make([]byte, 0, 50)
+
+	// attributes (1 byte)
+	record = append(record, 0)
+
+	// timestampDelta (signed zigzag varint - 0)
+	// 0 in zigzag is: (0 << 1) ^ (0 >> 63) = 0
+	record = append(record, 0)
+
+	// offsetDelta (signed zigzag varint - 0)
+	record = append(record, 0)
+
+	// keyLength (signed zigzag varint - -1 for null)
+	// -1 in zigzag is: (-1 << 1) ^ (-1 >> 63) = -2 ^ -1 = 1
+	record = append(record, 1)
+
+	// key (none, because null with length -1)
+
+	// valueLength (signed zigzag varint)
+	testValue := []byte(`{"type":"string"}`)
+	// Positive length N in zigzag is: (N << 1) = N*2
+	valueLen := len(testValue)
+	record = append(record, byte(valueLen<<1))
+
+	// value
+	record = append(record, testValue...)
+
+	// headersCount (signed zigzag varint - 0)
+	record = append(record, 0)
+
+	// Prepend record length as zigzag-encoded varint
+	recordLength := len(record)
+	recordWithLength := make([]byte, 0, recordLength+5)
+	// Zigzag encode the length: (n << 1) for positive n
+	zigzagLength := byte(recordLength << 1)
+	recordWithLength = append(recordWithLength, zigzagLength)
+	recordWithLength = append(recordWithLength, record...)
+
+	// Append record to batch
+	batch = append(batch, recordWithLength...)
+
+	// Calculate and set BatchLength (from PartitionLeaderEpoch to end)
+	batchLength := len(batch) - 12 // Exclude BaseOffset(8) + BatchLength(4)
+	binary.BigEndian.PutUint32(batch[batchLengthPos:batchLengthPos+4], uint32(batchLength))
+
+	// Calculate and set CRC32 (from Attributes to end)
+	// Kafka uses Castagnoli (CRC-32C) algorithm for record batch CRC
+	crcData := batch[21:] // From Attributes onwards
+	crc := crc32.Checksum(crcData, crc32.MakeTable(crc32.Castagnoli))
+	binary.BigEndian.PutUint32(batch[crcPos:crcPos+4], crc)
+
+	t.Logf("Created batch of %d bytes, record value: %s", len(batch), string(testValue))
+
+	// Now test extraction
+	results := h.extractAllRecords(batch)
+
+	if len(results) == 0 {
+		t.Fatalf("extractAllRecords returned 0 records, expected 1")
+	}
+
+	if len(results) != 1 {
+		t.Fatalf("extractAllRecords returned %d records, expected 1", len(results))
+	}
+
+	result := results[0]
+
+	// Key should be nil (we sent null key with varint -1)
+	if result.Key != nil {
+		t.Errorf("Expected nil key, got %v", result.Key)
+	}
+
+	// Value should match our test value
+	if string(result.Value) != string(testValue) {
+		t.Errorf("Value mismatch:\n  got:  %s\n  want: %s", string(result.Value), string(testValue))
+	}
+
+	t.Logf("Successfully extracted record with value: %s", string(result.Value))
+}
+
+// TestExtractAllRecords_CompressedBatch tests extracting records from a compressed batch
+func TestExtractAllRecords_CompressedBatch(t *testing.T) {
+	// This would test with actual compression, but for now we'll skip
+	// as we need to ensure uncompressed works first
+	t.Skip("Compressed batch test - implement after uncompressed works")
+}

weed/mq/kafka/protocol/response_cache.go

@@ -0,0 +1,80 @@
+package protocol
+
+import (
+	"sync"
+	"time"
+)
+
+// ResponseCache caches API responses to reduce CPU usage for repeated requests
+type ResponseCache struct {
+	mu    sync.RWMutex
+	cache map[string]*cacheEntry
+	ttl   time.Duration
+}
+
+type cacheEntry struct {
+	response  []byte
+	timestamp time.Time
+}
+
+// NewResponseCache creates a new response cache with the specified TTL
+func NewResponseCache(ttl time.Duration) *ResponseCache {
+	return &ResponseCache{
+		cache: make(map[string]*cacheEntry),
+		ttl:   ttl,
+	}
+}
+
+// Get retrieves a cached response if it exists and hasn't expired
+func (c *ResponseCache) Get(key string) ([]byte, bool) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	entry, exists := c.cache[key]
+	if !exists {
+		return nil, false
+	}
+
+	// Check if entry has expired
+	if time.Since(entry.timestamp) > c.ttl {
+		return nil, false
+	}
+
+	return entry.response, true
+}
+
+// Put stores a response in the cache
+func (c *ResponseCache) Put(key string, response []byte) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.cache[key] = &cacheEntry{
+		response:  response,
+		timestamp: time.Now(),
+	}
+}
+
+// Cleanup removes expired entries from the cache
+func (c *ResponseCache) Cleanup() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	now := time.Now()
+	for key, entry := range c.cache {
+		if now.Sub(entry.timestamp) > c.ttl {
+			delete(c.cache, key)
+		}
+	}
+}
+
+// StartCleanupLoop starts a background goroutine to periodically clean up expired entries
+func (c *ResponseCache) StartCleanupLoop(interval time.Duration) {
+	go func() {
+		ticker := time.NewTicker(interval)
+		defer ticker.Stop()
+
+		for range ticker.C {
+			c.Cleanup()
+		}
+	}()
+}

weed/mq/kafka/protocol/response_format_test.go

@@ -0,0 +1,313 @@
+package protocol
+
+import (
+	"encoding/binary"
+	"testing"
+)
+
+// TestResponseFormatsNoCorrelationID verifies that NO API response includes
+// the correlation ID in the response body (it should only be in the wire header)
+func TestResponseFormatsNoCorrelationID(t *testing.T) {
+	tests := []struct {
+		name        string
+		apiKey      uint16
+		apiVersion  uint16
+		buildFunc   func(correlationID uint32) ([]byte, error)
+		description string
+	}{
+		// Control Plane APIs
+		{
+			name:        "ApiVersions_v0",
+			apiKey:      18,
+			apiVersion:  0,
+			description: "ApiVersions v0 should not include correlation ID in body",
+		},
+		{
+			name:        "ApiVersions_v4",
+			apiKey:      18,
+			apiVersion:  4,
+			description: "ApiVersions v4 (flexible) should not include correlation ID in body",
+		},
+		{
+			name:        "Metadata_v0",
+			apiKey:      3,
+			apiVersion:  0,
+			description: "Metadata v0 should not include correlation ID in body",
+		},
+		{
+			name:        "Metadata_v7",
+			apiKey:      3,
+			apiVersion:  7,
+			description: "Metadata v7 should not include correlation ID in body",
+		},
+		{
+			name:        "FindCoordinator_v0",
+			apiKey:      10,
+			apiVersion:  0,
+			description: "FindCoordinator v0 should not include correlation ID in body",
+		},
+		{
+			name:        "FindCoordinator_v2",
+			apiKey:      10,
+			apiVersion:  2,
+			description: "FindCoordinator v2 should not include correlation ID in body",
+		},
+		{
+			name:        "DescribeConfigs_v0",
+			apiKey:      32,
+			apiVersion:  0,
+			description: "DescribeConfigs v0 should not include correlation ID in body",
+		},
+		{
+			name:        "DescribeConfigs_v4",
+			apiKey:      32,
+			apiVersion:  4,
+			description: "DescribeConfigs v4 (flexible) should not include correlation ID in body",
+		},
+		{
+			name:        "DescribeCluster_v0",
+			apiKey:      60,
+			apiVersion:  0,
+			description: "DescribeCluster v0 (flexible) should not include correlation ID in body",
+		},
+		{
+			name:        "InitProducerId_v0",
+			apiKey:      22,
+			apiVersion:  0,
+			description: "InitProducerId v0 should not include correlation ID in body",
+		},
+		{
+			name:        "InitProducerId_v4",
+			apiKey:      22,
+			apiVersion:  4,
+			description: "InitProducerId v4 (flexible) should not include correlation ID in body",
+		},
+
+		// Consumer Group Coordination APIs
+		{
+			name:        "JoinGroup_v0",
+			apiKey:      11,
+			apiVersion:  0,
+			description: "JoinGroup v0 should not include correlation ID in body",
+		},
+		{
+			name:        "SyncGroup_v0",
+			apiKey:      14,
+			apiVersion:  0,
+			description: "SyncGroup v0 should not include correlation ID in body",
+		},
+		{
+			name:        "Heartbeat_v0",
+			apiKey:      12,
+			apiVersion:  0,
+			description: "Heartbeat v0 should not include correlation ID in body",
+		},
+		{
+			name:        "LeaveGroup_v0",
+			apiKey:      13,
+			apiVersion:  0,
+			description: "LeaveGroup v0 should not include correlation ID in body",
+		},
+		{
+			name:        "OffsetFetch_v0",
+			apiKey:      9,
+			apiVersion:  0,
+			description: "OffsetFetch v0 should not include correlation ID in body",
+		},
+		{
+			name:        "OffsetCommit_v0",
+			apiKey:      8,
+			apiVersion:  0,
+			description: "OffsetCommit v0 should not include correlation ID in body",
+		},
+
+		// Data Plane APIs
+		{
+			name:        "Produce_v0",
+			apiKey:      0,
+			apiVersion:  0,
+			description: "Produce v0 should not include correlation ID in body",
+		},
+		{
+			name:        "Produce_v7",
+			apiKey:      0,
+			apiVersion:  7,
+			description: "Produce v7 should not include correlation ID in body",
+		},
+		{
+			name:        "Fetch_v0",
+			apiKey:      1,
+			apiVersion:  0,
+			description: "Fetch v0 should not include correlation ID in body",
+		},
+		{
+			name:        "Fetch_v7",
+			apiKey:      1,
+			apiVersion:  7,
+			description: "Fetch v7 should not include correlation ID in body",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Logf("Testing %s: %s", tt.name, tt.description)
+
+			// This test documents the EXPECTATION but can't automatically verify
+			// all responses without implementing mock handlers for each API.
+			// The key insight is: ALL responses should be checked manually
+			// or with integration tests.
+
+			t.Logf("✓ API Key %d Version %d: Correlation ID should be handled by writeResponseWithHeader",
+				tt.apiKey, tt.apiVersion)
+		})
+	}
+}
+
+// TestFlexibleResponseHeaderFormat verifies that flexible responses
+// include the 0x00 tagged fields byte in the header
+func TestFlexibleResponseHeaderFormat(t *testing.T) {
+	tests := []struct {
+		name       string
+		apiKey     uint16
+		apiVersion uint16
+		isFlexible bool
+	}{
+		// ApiVersions is special - never flexible header (AdminClient compatibility)
+		{"ApiVersions_v0", 18, 0, false},
+		{"ApiVersions_v3", 18, 3, false}, // Special case!
+		{"ApiVersions_v4", 18, 4, false}, // Special case!
+
+		// Metadata becomes flexible at v9+
+		{"Metadata_v0", 3, 0, false},
+		{"Metadata_v7", 3, 7, false},
+		{"Metadata_v9", 3, 9, true},
+
+		// Produce becomes flexible at v9+
+		{"Produce_v0", 0, 0, false},
+		{"Produce_v7", 0, 7, false},
+		{"Produce_v9", 0, 9, true},
+
+		// Fetch becomes flexible at v12+
+		{"Fetch_v0", 1, 0, false},
+		{"Fetch_v7", 1, 7, false},
+		{"Fetch_v12", 1, 12, true},
+
+		// FindCoordinator becomes flexible at v3+
+		{"FindCoordinator_v0", 10, 0, false},
+		{"FindCoordinator_v2", 10, 2, false},
+		{"FindCoordinator_v3", 10, 3, true},
+
+		// JoinGroup becomes flexible at v6+
+		{"JoinGroup_v0", 11, 0, false},
+		{"JoinGroup_v5", 11, 5, false},
+		{"JoinGroup_v6", 11, 6, true},
+
+		// SyncGroup becomes flexible at v4+
+		{"SyncGroup_v0", 14, 0, false},
+		{"SyncGroup_v3", 14, 3, false},
+		{"SyncGroup_v4", 14, 4, true},
+
+		// Heartbeat becomes flexible at v4+
+		{"Heartbeat_v0", 12, 0, false},
+		{"Heartbeat_v3", 12, 3, false},
+		{"Heartbeat_v4", 12, 4, true},
+
+		// LeaveGroup becomes flexible at v4+
+		{"LeaveGroup_v0", 13, 0, false},
+		{"LeaveGroup_v3", 13, 3, false},
+		{"LeaveGroup_v4", 13, 4, true},
+
+		// OffsetFetch becomes flexible at v6+
+		{"OffsetFetch_v0", 9, 0, false},
+		{"OffsetFetch_v5", 9, 5, false},
+		{"OffsetFetch_v6", 9, 6, true},
+
+		// OffsetCommit becomes flexible at v8+
+		{"OffsetCommit_v0", 8, 0, false},
+		{"OffsetCommit_v7", 8, 7, false},
+		{"OffsetCommit_v8", 8, 8, true},
+
+		// DescribeConfigs becomes flexible at v4+
+		{"DescribeConfigs_v0", 32, 0, false},
+		{"DescribeConfigs_v3", 32, 3, false},
+		{"DescribeConfigs_v4", 32, 4, true},
+
+		// InitProducerId becomes flexible at v2+
+		{"InitProducerId_v0", 22, 0, false},
+		{"InitProducerId_v1", 22, 1, false},
+		{"InitProducerId_v2", 22, 2, true},
+
+		// DescribeCluster is always flexible
+		{"DescribeCluster_v0", 60, 0, true},
+		{"DescribeCluster_v1", 60, 1, true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual := isFlexibleResponse(tt.apiKey, tt.apiVersion)
+			if actual != tt.isFlexible {
+				t.Errorf("%s: isFlexibleResponse(%d, %d) = %v, want %v",
+					tt.name, tt.apiKey, tt.apiVersion, actual, tt.isFlexible)
+			} else {
+				t.Logf("✓ %s: correctly identified as flexible=%v", tt.name, tt.isFlexible)
+			}
+		})
+	}
+}
+
+// TestCorrelationIDNotInResponseBody is a helper that can be used
+// to scan response bytes and detect if correlation ID appears in the body
+func TestCorrelationIDNotInResponseBody(t *testing.T) {
+	// Test helper function
+	hasCorrelationIDInBody := func(responseBody []byte, correlationID uint32) bool {
+		if len(responseBody) < 4 {
+			return false
+		}
+
+		// Check if the first 4 bytes match the correlation ID
+		actual := binary.BigEndian.Uint32(responseBody[0:4])
+		return actual == correlationID
+	}
+
+	t.Run("DetectCorrelationIDInBody", func(t *testing.T) {
+		correlationID := uint32(12345)
+
+		// Case 1: Response with correlation ID (BAD)
+		badResponse := make([]byte, 8)
+		binary.BigEndian.PutUint32(badResponse[0:4], correlationID)
+		badResponse[4] = 0x00 // some data
+
+		if !hasCorrelationIDInBody(badResponse, correlationID) {
+			t.Error("Failed to detect correlation ID in response body")
+		} else {
+			t.Log("✓ Successfully detected correlation ID in body (bad response)")
+		}
+
+		// Case 2: Response without correlation ID (GOOD)
+		goodResponse := make([]byte, 8)
+		goodResponse[0] = 0x00 // error code
+		goodResponse[1] = 0x00
+
+		if hasCorrelationIDInBody(goodResponse, correlationID) {
+			t.Error("False positive: detected correlation ID when it's not there")
+		} else {
+			t.Log("✓ Correctly identified response without correlation ID")
+		}
+	})
+}
+
+// TestWireProtocolFormat documents the expected wire format
+func TestWireProtocolFormat(t *testing.T) {
+	t.Log("Kafka Wire Protocol Format (KIP-482):")
+	t.Log("  Non-flexible responses:")
+	t.Log("    [Size: 4 bytes][Correlation ID: 4 bytes][Response Body]")
+	t.Log("")
+	t.Log("  Flexible responses (header version 1+):")
+	t.Log("    [Size: 4 bytes][Correlation ID: 4 bytes][Tagged Fields: 1+ bytes][Response Body]")
+	t.Log("")
+	t.Log("  Size field: includes correlation ID + tagged fields + body")
+	t.Log("  Tagged Fields: varint-encoded, 0x00 for empty")
+	t.Log("")
+	t.Log("CRITICAL: Response body should NEVER include correlation ID!")
+	t.Log("          It is written ONLY by writeResponseWithHeader")
+}

weed/mq/kafka/protocol/response_validation_example_test.go

@@ -0,0 +1,132 @@
+package protocol
+
+import (
+	"encoding/binary"
+	"testing"
+)
+
+// This file demonstrates what FIELD-LEVEL testing would look like
+// Currently these tests are NOT run automatically because they require
+// complex parsing logic for each API.
+
+// TestJoinGroupResponseStructure shows what we SHOULD test but currently don't
+func TestJoinGroupResponseStructure(t *testing.T) {
+	t.Skip("This is a demonstration test - shows what we SHOULD check")
+
+	// Hypothetical: build a JoinGroup response
+	// response := buildJoinGroupResponseV6(correlationID, generationID, protocolType, ...)
+
+	// What we SHOULD verify:
+	t.Log("Field-level checks we should perform:")
+	t.Log("  1. Error code (int16) - always present")
+	t.Log("  2. Generation ID (int32) - always present")
+	t.Log("  3. Protocol type (string/compact string) - nullable in some versions")
+	t.Log("  4. Protocol name (string/compact string) - always present")
+	t.Log("  5. Leader (string/compact string) - always present")
+	t.Log("  6. Member ID (string/compact string) - always present")
+	t.Log("  7. Members array - NON-NULLABLE, can be empty but must exist")
+	t.Log("     ^-- THIS is where the current bug is!")
+
+	// Example of what parsing would look like:
+	// offset := 0
+	// errorCode := binary.BigEndian.Uint16(response[offset:])
+	// offset += 2
+	// generationID := binary.BigEndian.Uint32(response[offset:])
+	// offset += 4
+	// ... parse protocol type ...
+	// ... parse protocol name ...
+	// ... parse leader ...
+	// ... parse member ID ...
+	// membersLength := parseCompactArray(response[offset:])
+	// if membersLength < 0 {
+	//     t.Error("Members array is null, but it should be non-nullable!")
+	// }
+}
+
+// TestProduceResponseStructure shows another example
+func TestProduceResponseStructure(t *testing.T) {
+	t.Skip("This is a demonstration test - shows what we SHOULD check")
+
+	t.Log("Produce response v7 structure:")
+	t.Log("  1. Topics array - must not be null")
+	t.Log("     - Topic name (string)")
+	t.Log("     - Partitions array - must not be null")
+	t.Log("       - Partition ID (int32)")
+	t.Log("       - Error code (int16)")
+	t.Log("       - Base offset (int64)")
+	t.Log("       - Log append time (int64)")
+	t.Log("       - Log start offset (int64)")
+	t.Log("  2. Throttle time (int32) - v1+")
+}
+
+// CompareWithReferenceImplementation shows ideal testing approach
+func TestCompareWithReferenceImplementation(t *testing.T) {
+	t.Skip("This would require a reference Kafka broker or client library")
+
+	// Ideal approach:
+	t.Log("1. Generate test data")
+	t.Log("2. Build response with our Gateway")
+	t.Log("3. Build response with kafka-go or Sarama library")
+	t.Log("4. Compare byte-by-byte")
+	t.Log("5. If different, highlight which fields differ")
+
+	// This would catch:
+	// - Wrong field order
+	// - Wrong field encoding
+	// - Missing fields
+	// - Null vs empty distinctions
+}
+
+// CurrentTestingApproach documents what we actually do
+func TestCurrentTestingApproach(t *testing.T) {
+	t.Log("Current testing strategy (as of Oct 2025):")
+	t.Log("")
+	t.Log("LEVEL 1: Static Code Analysis")
+	t.Log("  Tool: check_responses.sh")
+	t.Log("  Checks: Correlation ID patterns")
+	t.Log("  Coverage: Good for known issues")
+	t.Log("")
+	t.Log("LEVEL 2: Protocol Format Tests")
+	t.Log("  Tool: TestFlexibleResponseHeaderFormat")
+	t.Log("  Checks: Flexible vs non-flexible classification")
+	t.Log("  Coverage: Header format only")
+	t.Log("")
+	t.Log("LEVEL 3: Integration Testing")
+	t.Log("  Tool: Schema Registry, kafka-go, Sarama, Java client")
+	t.Log("  Checks: Real client compatibility")
+	t.Log("  Coverage: Complete but requires manual debugging")
+	t.Log("")
+	t.Log("MISSING: Field-level response body validation")
+	t.Log("  This is why JoinGroup issue wasn't caught by unit tests")
+}
+
+// Example of a REAL field-level test we could write
+func TestMetadataResponseHasBrokers(t *testing.T) {
+	t.Skip("Example of what a real field-level test would look like")
+
+	// Build a minimal metadata response
+	response := make([]byte, 0, 256)
+
+	// Brokers array (non-nullable)
+	brokerCount := uint32(1)
+	response = append(response,
+		byte(brokerCount>>24),
+		byte(brokerCount>>16),
+		byte(brokerCount>>8),
+		byte(brokerCount))
+
+	// Broker 1
+	response = append(response, 0, 0, 0, 1) // node_id = 1
+	// ... more fields ...
+
+	// Parse it back
+	offset := 0
+	parsedCount := binary.BigEndian.Uint32(response[offset : offset+4])
+
+	// Verify
+	if parsedCount == 0 {
+		t.Error("Metadata response has 0 brokers - should have at least 1")
+	}
+
+	t.Logf("✓ Metadata response correctly has %d broker(s)", parsedCount)
+}

weed/mq/kafka/protocol/syncgroup_assignment_test.go

@@ -0,0 +1,262 @@
+package protocol
+
+import (
+	"bytes"
+	"encoding/binary"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/consumer"
+)
+
+// TestSyncGroup_RaceCondition_BugDocumentation documents the original race condition bug
+// This test documents the bug where non-leader in Stable state would trigger server-side assignment
+func TestSyncGroup_RaceCondition_BugDocumentation(t *testing.T) {
+	// Original bug scenario:
+	// 1. Consumer 1 (leader) joins, gets all 15 partitions
+	// 2. Consumer 2 joins, triggers rebalance
+	// 3. Consumer 1 commits offsets during cleanup
+	// 4. Consumer 1 calls SyncGroup with client-side assignments, group moves to Stable
+	// 5. Consumer 2 calls SyncGroup (late arrival), group is already Stable
+	// 6. BUG: Consumer 2 falls into "else" branch, triggers server-side assignment
+	// 7. Consumer 2 gets 10 partitions via server-side assignment
+	// 8. Result: Some partitions (e.g., partition 2) assigned to BOTH consumers
+	// 9. Consumer 2 fetches offsets, gets offset 0 (no committed offsets yet)
+	// 10. Consumer 2 re-reads messages from offset 0 -> DUPLICATES (66.7%)!
+
+	// ORIGINAL BUGGY CODE (joingroup.go lines 887-905):
+	// } else if group.State == consumer.GroupStateCompletingRebalance || group.State == consumer.GroupStatePreparingRebalance {
+	//     // Non-leader member waiting for leader to provide assignments
+	//     glog.Infof("[SYNCGROUP] Non-leader %s waiting for leader assignments in group %s (state=%s)",
+	//         request.MemberID, request.GroupID, group.State)
+	// } else {
+	//     // BUG: This branch was triggered when non-leader arrived in Stable state!
+	//     glog.Warningf("[SYNCGROUP] Using server-side assignment for group %s (Leader=%s State=%s)",
+	//         request.GroupID, group.Leader, group.State)
+	//     topicPartitions := h.getTopicPartitions(group)
+	//     group.AssignPartitions(topicPartitions)  // <- Duplicate assignment!
+	// }
+
+	// FIXED CODE (joingroup.go lines 887-906):
+	// } else if request.MemberID != group.Leader && len(request.GroupAssignments) == 0 {
+	//     // Non-leader member requesting its assignment
+	//     // CRITICAL FIX: Non-leader members should ALWAYS wait for leader's client-side assignments
+	//     // This is the correct behavior for Sarama and other client-side assignment protocols
+	//     glog.Infof("[SYNCGROUP] Non-leader %s waiting for/retrieving assignment in group %s (state=%s)",
+	//         request.MemberID, request.GroupID, group.State)
+	//     // Assignment will be retrieved from member.Assignment below
+	// } else {
+	//     // This branch should only be reached for server-side assignment protocols
+	//     // (not Sarama's client-side assignment)
+	// }
+
+	t.Log("Original bug: Non-leader in Stable state would trigger server-side assignment")
+	t.Log("This caused duplicate partition assignments and message re-reads (66.7% duplicates)")
+	t.Log("Fix: Check if member is non-leader with empty assignments, regardless of group state")
+}
+
+// TestSyncGroup_FixVerification verifies the fix logic
+func TestSyncGroup_FixVerification(t *testing.T) {
+	testCases := []struct {
+		name           string
+		isLeader       bool
+		hasAssignments bool
+		shouldWait     bool
+		shouldAssign   bool
+		description    string
+	}{
+		{
+			name:           "Leader with assignments",
+			isLeader:       true,
+			hasAssignments: true,
+			shouldWait:     false,
+			shouldAssign:   false,
+			description:    "Leader provides client-side assignments, processes them",
+		},
+		{
+			name:           "Non-leader without assignments (PreparingRebalance)",
+			isLeader:       false,
+			hasAssignments: false,
+			shouldWait:     true,
+			shouldAssign:   false,
+			description:    "Non-leader waits for leader to provide assignments",
+		},
+		{
+			name:           "Non-leader without assignments (Stable) - THE BUG CASE",
+			isLeader:       false,
+			hasAssignments: false,
+			shouldWait:     true,
+			shouldAssign:   false,
+			description:    "Non-leader retrieves assignment from leader (already processed)",
+		},
+		{
+			name:           "Leader without assignments",
+			isLeader:       true,
+			hasAssignments: false,
+			shouldWait:     false,
+			shouldAssign:   true,
+			description:    "Edge case: server-side assignment (should not happen with Sarama)",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Simulate the fixed logic
+			memberID := "consumer-1"
+			leaderID := "consumer-1"
+			if !tc.isLeader {
+				memberID = "consumer-2"
+			}
+
+			groupAssignmentsCount := 0
+			if tc.hasAssignments {
+				groupAssignmentsCount = 2 // Leader provides assignments for 2 members
+			}
+
+			// THE FIX: Check if non-leader with no assignments
+			isNonLeaderWaiting := (memberID != leaderID) && (groupAssignmentsCount == 0)
+
+			if tc.shouldWait && !isNonLeaderWaiting {
+				t.Errorf("%s: Expected to wait, but logic says no", tc.description)
+			}
+			if !tc.shouldWait && isNonLeaderWaiting {
+				t.Errorf("%s: Expected not to wait, but logic says yes", tc.description)
+			}
+
+			t.Logf("✓ %s: isLeader=%v hasAssignments=%v shouldWait=%v",
+				tc.description, tc.isLeader, tc.hasAssignments, tc.shouldWait)
+		})
+	}
+}
+
+func TestParseSyncGroupRequestV5SkipsProtocolFields(t *testing.T) {
+	memberID := "consumer-schema-registry"
+	requestBody := buildSyncGroupV5Request("schema-registry", 12, memberID, "consumer", "range", []GroupAssignment{
+		{MemberID: memberID, Assignment: []byte(`{"version":1,"error":0}`)},
+	})
+
+	request, err := (&Handler{}).parseSyncGroupRequest(requestBody, 5)
+	if err != nil {
+		t.Fatalf("parseSyncGroupRequest failed: %v", err)
+	}
+
+	if request.GroupID != "schema-registry" {
+		t.Fatalf("GroupID = %q, want schema-registry", request.GroupID)
+	}
+	if request.GenerationID != 12 {
+		t.Fatalf("GenerationID = %d, want 12", request.GenerationID)
+	}
+	if request.MemberID != memberID {
+		t.Fatalf("MemberID = %q, want %q", request.MemberID, memberID)
+	}
+	if request.ProtocolType != "consumer" {
+		t.Fatalf("ProtocolType = %q, want consumer", request.ProtocolType)
+	}
+	if request.ProtocolName != "range" {
+		t.Fatalf("ProtocolName = %q, want range", request.ProtocolName)
+	}
+	if len(request.GroupAssignments) != 1 {
+		t.Fatalf("len(GroupAssignments) = %d, want 1", len(request.GroupAssignments))
+	}
+	if request.GroupAssignments[0].MemberID != memberID {
+		t.Fatalf("assignment member = %q, want %q", request.GroupAssignments[0].MemberID, memberID)
+	}
+	if string(request.GroupAssignments[0].Assignment) != `{"version":1,"error":0}` {
+		t.Fatalf("assignment payload = %q", string(request.GroupAssignments[0].Assignment))
+	}
+}
+
+func TestSyncGroupSchemaRegistryV5LeaderAssignmentDoesNotRebalance(t *testing.T) {
+	memberID := "consumer-schema-registry"
+	handler := &Handler{
+		groupCoordinator:  consumer.NewGroupCoordinator(),
+		defaultPartitions: 1,
+	}
+	t.Cleanup(handler.groupCoordinator.Close)
+
+	group := handler.groupCoordinator.GetOrCreateGroup("schema-registry")
+	group.Mu.Lock()
+	group.State = consumer.GroupStateCompletingRebalance
+	group.Generation = 7
+	group.Protocol = consumer.ProtocolNameRange
+	group.Leader = memberID
+	group.Members[memberID] = &consumer.GroupMember{
+		ID:    memberID,
+		State: consumer.MemberStatePending,
+		Metadata: []byte(`{
+			"host":"schema-registry",
+			"port":8081,
+			"scheme":"http",
+			"version":1,
+			"master_eligibility":true
+		}`),
+	}
+	group.Mu.Unlock()
+
+	requestBody := buildSyncGroupV5Request("schema-registry", 7, memberID, "consumer", "range", []GroupAssignment{
+		{MemberID: memberID, Assignment: []byte(`{"version":1,"error":0}`)},
+	})
+
+	response, err := handler.handleSyncGroup(99, 5, requestBody)
+	if err != nil {
+		t.Fatalf("handleSyncGroup failed: %v", err)
+	}
+	if code := syncGroupResponseErrorCode(t, response, 5); code != ErrorCodeNone {
+		t.Fatalf("SyncGroup error code = %d, want %d", code, ErrorCodeNone)
+	}
+	if !bytes.Contains(response, []byte(`"master":"`+memberID+`"`)) {
+		t.Fatalf("Schema Registry assignment response does not name leader %q: %q", memberID, string(response))
+	}
+
+	group.Mu.RLock()
+	defer group.Mu.RUnlock()
+	if group.Generation != 7 {
+		t.Fatalf("group generation = %d, want 7", group.Generation)
+	}
+	if group.State != consumer.GroupStateStable {
+		t.Fatalf("group state = %s, want Stable", group.State)
+	}
+	if group.Members[memberID].State != consumer.MemberStateStable {
+		t.Fatalf("member state = %s, want Stable", group.Members[memberID].State)
+	}
+}
+
+func buildSyncGroupV5Request(groupID string, generation int32, memberID, protocolType, protocolName string, assignments []GroupAssignment) []byte {
+	var data []byte
+	data = appendCompactString(data, groupID)
+
+	var generationBytes [4]byte
+	binary.BigEndian.PutUint32(generationBytes[:], uint32(generation))
+	data = append(data, generationBytes[:]...)
+
+	data = appendCompactString(data, memberID)
+	data = append(data, 0) // group_instance_id = null
+	data = appendCompactString(data, protocolType)
+	data = appendCompactString(data, protocolName)
+
+	data = append(data, CompactArrayLength(uint32(len(assignments)))...)
+	for _, assignment := range assignments {
+		data = appendCompactString(data, assignment.MemberID)
+		data = append(data, CompactStringLength(len(assignment.Assignment))...)
+		data = append(data, assignment.Assignment...)
+		data = append(data, 0) // assignment tagged fields
+	}
+	data = append(data, 0) // request tagged fields
+	return data
+}
+
+func appendCompactString(data []byte, value string) []byte {
+	data = append(data, CompactStringLength(len(value))...)
+	return append(data, value...)
+}
+
+func syncGroupResponseErrorCode(t *testing.T, response []byte, apiVersion uint16) int16 {
+	t.Helper()
+	offset := 0
+	if apiVersion >= 1 {
+		offset += 4
+	}
+	if len(response) < offset+2 {
+		t.Fatalf("SyncGroup response too short: %d bytes", len(response))
+	}
+	return int16(binary.BigEndian.Uint16(response[offset : offset+2]))
+}

weed/mq/kafka/protocol/topic_autocreate_test.go

@@ -0,0 +1,74 @@
+package protocol
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/integration"
+)
+
+// autoCreateMockHandler models the flaky-existence scenario: TopicExists reports
+// the topic missing (a transient broker/filer false-negative), while a create
+// attempt reports the topic already exists. It reuses FastMockHandler for the
+// rest of the interface and overrides only the two methods under test.
+type autoCreateMockHandler struct {
+	*FastMockHandler
+	exists    bool
+	createErr error
+}
+
+func (h *autoCreateMockHandler) TopicExists(name string) bool { return h.exists }
+
+func (h *autoCreateMockHandler) CreateTopic(name string, partitions int32) error {
+	return h.createErr
+}
+
+// TestEnsureTopicExistsTreatsAlreadyExistsAsSuccess reproduces the offset-resumption
+// flake: TopicExists returns a false-negative for a topic that is in fact present,
+// so the auto-create attempt fails with ErrTopicAlreadyExists. The topic must still
+// be reported as existing, otherwise the client sees UNKNOWN_TOPIC_OR_PARTITION.
+func TestEnsureTopicExistsTreatsAlreadyExistsAsSuccess(t *testing.T) {
+	handler := &Handler{
+		seaweedMQHandler: &autoCreateMockHandler{
+			FastMockHandler: &FastMockHandler{},
+			exists:          false,
+			createErr:       fmt.Errorf("%s: %w", "offset-resumption", integration.ErrTopicAlreadyExists),
+		},
+	}
+
+	if !handler.ensureTopicExists("offset-resumption", 1) {
+		t.Fatal("ensureTopicExists must report true when the create attempt finds the topic already exists")
+	}
+}
+
+// TestEnsureTopicExistsReportsGenuineFailure confirms a real create failure (not
+// "already exists") still results in the topic being treated as absent.
+func TestEnsureTopicExistsReportsGenuineFailure(t *testing.T) {
+	handler := &Handler{
+		seaweedMQHandler: &autoCreateMockHandler{
+			FastMockHandler: &FastMockHandler{},
+			exists:          false,
+			createErr:       fmt.Errorf("broker unavailable"),
+		},
+	}
+
+	if handler.ensureTopicExists("missing-topic", 1) {
+		t.Fatal("ensureTopicExists must report false when the topic is absent and creation genuinely fails")
+	}
+}
+
+// TestEnsureTopicExistsWhenPresent covers the common path where the topic is
+// already known to exist and no create attempt is needed.
+func TestEnsureTopicExistsWhenPresent(t *testing.T) {
+	handler := &Handler{
+		seaweedMQHandler: &autoCreateMockHandler{
+			FastMockHandler: &FastMockHandler{},
+			exists:          true,
+			createErr:       fmt.Errorf("CreateTopic should not be called when the topic exists"),
+		},
+	}
+
+	if !handler.ensureTopicExists("present-topic", 1) {
+		t.Fatal("ensureTopicExists must report true for an existing topic")
+	}
+}