snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/mq/offset/benchmark_test.go

@@ -0,0 +1,305 @@
+package offset
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+// BenchmarkOffsetAssignment benchmarks sequential offset assignment
+func BenchmarkOffsetAssignment(b *testing.B) {
+	storage := NewInMemoryOffsetStorage()
+
+	partition := &schema_pb.Partition{
+		RingSize:   1024,
+		RangeStart: 0,
+		RangeStop:  31,
+		UnixTimeNs: time.Now().UnixNano(),
+	}
+
+	manager, err := NewPartitionOffsetManager("test-namespace", "test-topic", partition, storage)
+	if err != nil {
+		b.Fatalf("Failed to create partition manager: %v", err)
+	}
+
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			manager.AssignOffset()
+		}
+	})
+}
+
+// BenchmarkBatchOffsetAssignment benchmarks batch offset assignment
+func BenchmarkBatchOffsetAssignment(b *testing.B) {
+	storage := NewInMemoryOffsetStorage()
+
+	partition := &schema_pb.Partition{
+		RingSize:   1024,
+		RangeStart: 0,
+		RangeStop:  31,
+		UnixTimeNs: time.Now().UnixNano(),
+	}
+
+	manager, err := NewPartitionOffsetManager("test-namespace", "test-topic", partition, storage)
+	if err != nil {
+		b.Fatalf("Failed to create partition manager: %v", err)
+	}
+
+	batchSizes := []int64{1, 10, 100, 1000}
+
+	for _, batchSize := range batchSizes {
+		b.Run(fmt.Sprintf("BatchSize%d", batchSize), func(b *testing.B) {
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				manager.AssignOffsets(batchSize)
+			}
+		})
+	}
+}
+
+// BenchmarkOffsetSubscription benchmarks subscription operations
+func BenchmarkOffsetSubscription(b *testing.B) {
+	storage := NewInMemoryOffsetStorage()
+	registry := NewPartitionOffsetRegistry(storage)
+	subscriber := NewOffsetSubscriber(registry)
+
+	partition := &schema_pb.Partition{
+		RingSize:   1024,
+		RangeStart: 0,
+		RangeStop:  31,
+		UnixTimeNs: time.Now().UnixNano(),
+	}
+
+	// Pre-assign offsets
+	registry.AssignOffsets("test-namespace", "test-topic", partition, 10000)
+
+	b.Run("CreateSubscription", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			subscriptionID := fmt.Sprintf("bench-sub-%d", i)
+			_, err := subscriber.CreateSubscription(
+				subscriptionID,
+				"test-namespace", "test-topic",
+				partition,
+				schema_pb.OffsetType_RESET_TO_EARLIEST,
+				0,
+			)
+			if err != nil {
+				b.Fatalf("Failed to create subscription: %v", err)
+			}
+			subscriber.CloseSubscription(subscriptionID)
+		}
+	})
+
+	// Create subscription for other benchmarks
+	sub, err := subscriber.CreateSubscription(
+		"bench-sub",
+		"test-namespace", "test-topic",
+		partition,
+		schema_pb.OffsetType_RESET_TO_EARLIEST,
+		0,
+	)
+	if err != nil {
+		b.Fatalf("Failed to create subscription: %v", err)
+	}
+
+	b.Run("GetOffsetRange", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			sub.GetOffsetRange(100)
+		}
+	})
+
+	b.Run("AdvanceOffset", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			sub.AdvanceOffset()
+		}
+	})
+
+	b.Run("GetLag", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			sub.GetLag()
+		}
+	})
+
+	b.Run("SeekToOffset", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			offset := int64(i % 9000) // Stay within bounds
+			sub.SeekToOffset(offset)
+		}
+	})
+}
+
+// BenchmarkSMQOffsetIntegration benchmarks the full integration layer
+func BenchmarkSMQOffsetIntegration(b *testing.B) {
+	storage := NewInMemoryOffsetStorage()
+	integration := NewSMQOffsetIntegration(storage)
+
+	partition := &schema_pb.Partition{
+		RingSize:   1024,
+		RangeStart: 0,
+		RangeStop:  31,
+		UnixTimeNs: time.Now().UnixNano(),
+	}
+
+	b.Run("PublishRecord", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			key := fmt.Sprintf("key-%d", i)
+			integration.PublishRecord("test-namespace", "test-topic", partition, []byte(key), &schema_pb.RecordValue{})
+		}
+	})
+
+	b.Run("PublishRecordBatch", func(b *testing.B) {
+		batchSizes := []int{1, 10, 100}
+
+		for _, batchSize := range batchSizes {
+			b.Run(fmt.Sprintf("BatchSize%d", batchSize), func(b *testing.B) {
+				b.ResetTimer()
+				for i := 0; i < b.N; i++ {
+					records := make([]PublishRecordRequest, batchSize)
+					for j := 0; j < batchSize; j++ {
+						records[j] = PublishRecordRequest{
+							Key:   []byte(fmt.Sprintf("batch-%d-key-%d", i, j)),
+							Value: &schema_pb.RecordValue{},
+						}
+					}
+					integration.PublishRecordBatch("test-namespace", "test-topic", partition, records)
+				}
+			})
+		}
+	})
+
+	// Pre-populate for subscription benchmarks
+	records := make([]PublishRecordRequest, 1000)
+	for i := 0; i < 1000; i++ {
+		records[i] = PublishRecordRequest{
+			Key:   []byte(fmt.Sprintf("pre-key-%d", i)),
+			Value: &schema_pb.RecordValue{},
+		}
+	}
+	integration.PublishRecordBatch("test-namespace", "test-topic", partition, records)
+
+	b.Run("CreateSubscription", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			subscriptionID := fmt.Sprintf("integration-sub-%d", i)
+			_, err := integration.CreateSubscription(
+				subscriptionID,
+				"test-namespace", "test-topic",
+				partition,
+				schema_pb.OffsetType_RESET_TO_EARLIEST,
+				0,
+			)
+			if err != nil {
+				b.Fatalf("Failed to create subscription: %v", err)
+			}
+			integration.CloseSubscription(subscriptionID)
+		}
+	})
+
+	b.Run("GetHighWaterMark", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			integration.GetHighWaterMark("test-namespace", "test-topic", partition)
+		}
+	})
+
+	b.Run("GetPartitionOffsetInfo", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			integration.GetPartitionOffsetInfo("test-namespace", "test-topic", partition)
+		}
+	})
+}
+
+// BenchmarkConcurrentOperations benchmarks concurrent offset operations
+func BenchmarkConcurrentOperations(b *testing.B) {
+	storage := NewInMemoryOffsetStorage()
+	integration := NewSMQOffsetIntegration(storage)
+
+	partition := &schema_pb.Partition{
+		RingSize:   1024,
+		RangeStart: 0,
+		RangeStop:  31,
+		UnixTimeNs: time.Now().UnixNano(),
+	}
+
+	b.Run("ConcurrentPublish", func(b *testing.B) {
+		b.ResetTimer()
+		b.RunParallel(func(pb *testing.PB) {
+			i := 0
+			for pb.Next() {
+				key := fmt.Sprintf("concurrent-key-%d", i)
+				integration.PublishRecord("test-namespace", "test-topic", partition, []byte(key), &schema_pb.RecordValue{})
+				i++
+			}
+		})
+	})
+
+	// Pre-populate for concurrent reads
+	for i := 0; i < 1000; i++ {
+		key := fmt.Sprintf("read-key-%d", i)
+		integration.PublishRecord("test-namespace", "test-topic", partition, []byte(key), &schema_pb.RecordValue{})
+	}
+
+	b.Run("ConcurrentRead", func(b *testing.B) {
+		b.ResetTimer()
+		b.RunParallel(func(pb *testing.PB) {
+			for pb.Next() {
+				integration.GetHighWaterMark("test-namespace", "test-topic", partition)
+			}
+		})
+	})
+
+	b.Run("ConcurrentMixed", func(b *testing.B) {
+		b.ResetTimer()
+		b.RunParallel(func(pb *testing.PB) {
+			i := 0
+			for pb.Next() {
+				if i%10 == 0 {
+					// 10% writes
+					key := fmt.Sprintf("mixed-key-%d", i)
+					integration.PublishRecord("test-namespace", "test-topic", partition, []byte(key), &schema_pb.RecordValue{})
+				} else {
+					// 90% reads
+					integration.GetHighWaterMark("test-namespace", "test-topic", partition)
+				}
+				i++
+			}
+		})
+	})
+}
+
+// BenchmarkMemoryUsage benchmarks memory usage patterns
+func BenchmarkMemoryUsage(b *testing.B) {
+	b.Run("InMemoryStorage", func(b *testing.B) {
+		storage := NewInMemoryOffsetStorage()
+		partition := &schema_pb.Partition{
+			RingSize:   1024,
+			RangeStart: 0,
+			RangeStop:  31,
+			UnixTimeNs: time.Now().UnixNano(),
+		}
+
+		manager, err := NewPartitionOffsetManager("test-namespace", "test-topic", partition, storage)
+		if err != nil {
+			b.Fatalf("Failed to create partition manager: %v", err)
+		}
+
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			manager.AssignOffset()
+			// Note: Checkpointing now happens automatically in background every 2 seconds
+		}
+
+		// Clean up background goroutine
+		manager.Close()
+	})
+}

weed/mq/offset/consumer_group_storage.go

@@ -0,0 +1,181 @@
+package offset
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/filer_client"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+// ConsumerGroupPosition represents a consumer's position in a partition
+// This can be either a timestamp or an offset
+type ConsumerGroupPosition struct {
+	Type        string `json:"type"`         // "offset" or "timestamp"
+	Value       int64  `json:"value"`        // The actual offset or timestamp value
+	OffsetType  string `json:"offset_type"`  // Optional: OffsetType enum name (e.g., "EXACT_OFFSET")
+	CommittedAt int64  `json:"committed_at"` // Unix timestamp in milliseconds when committed
+	Metadata    string `json:"metadata"`     // Optional: application-specific metadata
+}
+
+// ConsumerGroupOffsetStorage handles consumer group offset persistence
+// Each consumer group gets its own offset file in a dedicated consumers/ subfolder:
+// Path: /topics/{namespace}/{topic}/{version}/{partition}/consumers/{consumer_group}.offset
+type ConsumerGroupOffsetStorage interface {
+	// SaveConsumerGroupOffset saves the committed offset for a consumer group
+	SaveConsumerGroupOffset(t topic.Topic, p topic.Partition, consumerGroup string, offset int64) error
+
+	// SaveConsumerGroupPosition saves the committed position (offset or timestamp) for a consumer group
+	SaveConsumerGroupPosition(t topic.Topic, p topic.Partition, consumerGroup string, position *ConsumerGroupPosition) error
+
+	// LoadConsumerGroupOffset loads the committed offset for a consumer group (backward compatible)
+	LoadConsumerGroupOffset(t topic.Topic, p topic.Partition, consumerGroup string) (int64, error)
+
+	// LoadConsumerGroupPosition loads the committed position for a consumer group
+	LoadConsumerGroupPosition(t topic.Topic, p topic.Partition, consumerGroup string) (*ConsumerGroupPosition, error)
+
+	// ListConsumerGroups returns all consumer groups for a topic partition
+	ListConsumerGroups(t topic.Topic, p topic.Partition) ([]string, error)
+
+	// DeleteConsumerGroupOffset removes the offset file for a consumer group
+	DeleteConsumerGroupOffset(t topic.Topic, p topic.Partition, consumerGroup string) error
+}
+
+// FilerConsumerGroupOffsetStorage implements ConsumerGroupOffsetStorage using SeaweedFS filer
+type FilerConsumerGroupOffsetStorage struct {
+	filerClientAccessor *filer_client.FilerClientAccessor
+}
+
+// NewFilerConsumerGroupOffsetStorageWithAccessor creates storage using a shared filer client accessor
+func NewFilerConsumerGroupOffsetStorageWithAccessor(filerClientAccessor *filer_client.FilerClientAccessor) *FilerConsumerGroupOffsetStorage {
+	return &FilerConsumerGroupOffsetStorage{
+		filerClientAccessor: filerClientAccessor,
+	}
+}
+
+// SaveConsumerGroupOffset saves the committed offset for a consumer group
+// Stores as: /topics/{namespace}/{topic}/{version}/{partition}/consumers/{consumer_group}.offset
+// This is a convenience method that wraps SaveConsumerGroupPosition
+func (f *FilerConsumerGroupOffsetStorage) SaveConsumerGroupOffset(t topic.Topic, p topic.Partition, consumerGroup string, offset int64) error {
+	position := &ConsumerGroupPosition{
+		Type:        "offset",
+		Value:       offset,
+		OffsetType:  schema_pb.OffsetType_EXACT_OFFSET.String(),
+		CommittedAt: time.Now().UnixMilli(),
+	}
+	return f.SaveConsumerGroupPosition(t, p, consumerGroup, position)
+}
+
+// SaveConsumerGroupPosition saves the committed position (offset or timestamp) for a consumer group
+// Stores as JSON: /topics/{namespace}/{topic}/{version}/{partition}/consumers/{consumer_group}.offset
+func (f *FilerConsumerGroupOffsetStorage) SaveConsumerGroupPosition(t topic.Topic, p topic.Partition, consumerGroup string, position *ConsumerGroupPosition) error {
+	partitionDir := topic.PartitionDir(t, p)
+	consumersDir := fmt.Sprintf("%s/consumers", partitionDir)
+	offsetFileName := fmt.Sprintf("%s.offset", consumerGroup)
+
+	// Marshal position to JSON
+	jsonBytes, err := json.Marshal(position)
+	if err != nil {
+		return fmt.Errorf("failed to marshal position to JSON: %w", err)
+	}
+
+	return f.filerClientAccessor.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		return filer.SaveInsideFiler(context.Background(), client, consumersDir, offsetFileName, jsonBytes)
+	})
+}
+
+// LoadConsumerGroupOffset loads the committed offset for a consumer group
+// This method provides backward compatibility and returns just the offset value
+func (f *FilerConsumerGroupOffsetStorage) LoadConsumerGroupOffset(t topic.Topic, p topic.Partition, consumerGroup string) (int64, error) {
+	position, err := f.LoadConsumerGroupPosition(t, p, consumerGroup)
+	if err != nil {
+		return -1, err
+	}
+	return position.Value, nil
+}
+
+// LoadConsumerGroupPosition loads the committed position for a consumer group
+func (f *FilerConsumerGroupOffsetStorage) LoadConsumerGroupPosition(t topic.Topic, p topic.Partition, consumerGroup string) (*ConsumerGroupPosition, error) {
+	partitionDir := topic.PartitionDir(t, p)
+	consumersDir := fmt.Sprintf("%s/consumers", partitionDir)
+	offsetFileName := fmt.Sprintf("%s.offset", consumerGroup)
+
+	var position *ConsumerGroupPosition
+	err := f.filerClientAccessor.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		data, err := filer.ReadInsideFiler(context.Background(), client, consumersDir, offsetFileName)
+		if err != nil {
+			return err
+		}
+
+		// Parse JSON format
+		position = &ConsumerGroupPosition{}
+		if err := json.Unmarshal(data, position); err != nil {
+			return fmt.Errorf("invalid consumer group offset file format: %w", err)
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return position, nil
+}
+
+// ListConsumerGroups returns all consumer groups for a topic partition
+func (f *FilerConsumerGroupOffsetStorage) ListConsumerGroups(t topic.Topic, p topic.Partition) ([]string, error) {
+	partitionDir := topic.PartitionDir(t, p)
+	consumersDir := fmt.Sprintf("%s/consumers", partitionDir)
+	var consumerGroups []string
+
+	err := f.filerClientAccessor.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		// Use ListEntries to get directory contents
+		stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
+			Directory: consumersDir,
+		})
+		if err != nil {
+			return err
+		}
+
+		for {
+			resp, err := stream.Recv()
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				return err
+			}
+
+			entry := resp.Entry
+			if entry != nil && !entry.IsDirectory && entry.Name != "" {
+				// Check if this is a consumer group offset file (ends with .offset)
+				if len(entry.Name) > 7 && entry.Name[len(entry.Name)-7:] == ".offset" {
+					// Extract consumer group name (remove .offset suffix)
+					consumerGroup := entry.Name[:len(entry.Name)-7]
+					consumerGroups = append(consumerGroups, consumerGroup)
+				}
+			}
+		}
+		return nil
+	})
+
+	return consumerGroups, err
+}
+
+// DeleteConsumerGroupOffset removes the offset file for a consumer group
+func (f *FilerConsumerGroupOffsetStorage) DeleteConsumerGroupOffset(t topic.Topic, p topic.Partition, consumerGroup string) error {
+	partitionDir := topic.PartitionDir(t, p)
+	consumersDir := fmt.Sprintf("%s/consumers", partitionDir)
+	offsetFileName := fmt.Sprintf("%s.offset", consumerGroup)
+
+	return f.filerClientAccessor.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		return filer_pb.DoRemove(context.Background(), client, consumersDir, offsetFileName, false, false, false, false, nil)
+	})
+}

weed/mq/offset/consumer_group_storage_test.go

@@ -0,0 +1,128 @@
+package offset
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+func TestConsumerGroupPosition_JSON(t *testing.T) {
+	tests := []struct {
+		name     string
+		position *ConsumerGroupPosition
+	}{
+		{
+			name: "offset-based position",
+			position: &ConsumerGroupPosition{
+				Type:        "offset",
+				Value:       12345,
+				OffsetType:  schema_pb.OffsetType_EXACT_OFFSET.String(),
+				CommittedAt: time.Now().UnixMilli(),
+				Metadata:    "test metadata",
+			},
+		},
+		{
+			name: "timestamp-based position",
+			position: &ConsumerGroupPosition{
+				Type:        "timestamp",
+				Value:       time.Now().UnixNano(),
+				OffsetType:  schema_pb.OffsetType_EXACT_TS_NS.String(),
+				CommittedAt: time.Now().UnixMilli(),
+				Metadata:    "checkpoint at 2024-10-05",
+			},
+		},
+		{
+			name: "minimal position",
+			position: &ConsumerGroupPosition{
+				Type:  "offset",
+				Value: 42,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Marshal to JSON
+			jsonBytes, err := json.Marshal(tt.position)
+			if err != nil {
+				t.Fatalf("Failed to marshal: %v", err)
+			}
+
+			t.Logf("JSON: %s", string(jsonBytes))
+
+			// Unmarshal from JSON
+			var decoded ConsumerGroupPosition
+			if err := json.Unmarshal(jsonBytes, &decoded); err != nil {
+				t.Fatalf("Failed to unmarshal: %v", err)
+			}
+
+			// Verify fields
+			if decoded.Type != tt.position.Type {
+				t.Errorf("Type mismatch: got %s, want %s", decoded.Type, tt.position.Type)
+			}
+			if decoded.Value != tt.position.Value {
+				t.Errorf("Value mismatch: got %d, want %d", decoded.Value, tt.position.Value)
+			}
+			if decoded.OffsetType != tt.position.OffsetType {
+				t.Errorf("OffsetType mismatch: got %s, want %s", decoded.OffsetType, tt.position.OffsetType)
+			}
+			if decoded.Metadata != tt.position.Metadata {
+				t.Errorf("Metadata mismatch: got %s, want %s", decoded.Metadata, tt.position.Metadata)
+			}
+		})
+	}
+}
+
+func TestConsumerGroupPosition_JSONExamples(t *testing.T) {
+	// Test JSON format examples
+	jsonExamples := []string{
+		`{"type":"offset","value":12345}`,
+		`{"type":"timestamp","value":1696521600000000000}`,
+		`{"type":"offset","value":42,"offset_type":"EXACT_OFFSET","committed_at":1696521600000,"metadata":"test"}`,
+	}
+
+	for i, jsonStr := range jsonExamples {
+		var position ConsumerGroupPosition
+		if err := json.Unmarshal([]byte(jsonStr), &position); err != nil {
+			t.Errorf("Example %d: Failed to parse JSON: %v", i, err)
+			continue
+		}
+
+		t.Logf("Example %d: Type=%s, Value=%d", i, position.Type, position.Value)
+
+		// Verify required fields
+		if position.Type == "" {
+			t.Errorf("Example %d: Type is empty", i)
+		}
+		if position.Value == 0 {
+			t.Errorf("Example %d: Value is zero", i)
+		}
+	}
+}
+
+func TestConsumerGroupPosition_TypeValidation(t *testing.T) {
+	validTypes := []string{"offset", "timestamp"}
+
+	for _, typ := range validTypes {
+		position := &ConsumerGroupPosition{
+			Type:  typ,
+			Value: 100,
+		}
+
+		jsonBytes, err := json.Marshal(position)
+		if err != nil {
+			t.Fatalf("Failed to marshal position with type '%s': %v", typ, err)
+		}
+
+		var decoded ConsumerGroupPosition
+		if err := json.Unmarshal(jsonBytes, &decoded); err != nil {
+			t.Fatalf("Failed to unmarshal position with type '%s': %v", typ, err)
+		}
+
+		if decoded.Type != typ {
+			t.Errorf("Type mismatch: got '%s', want '%s'", decoded.Type, typ)
+		}
+	}
+}

weed/mq/offset/filer_storage.go

@@ -0,0 +1,95 @@
+package offset
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/filer_client"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// FilerOffsetStorage implements OffsetStorage using SeaweedFS filer
+// Stores offset data as files in the same directory structure as SMQ
+// Path: /topics/{namespace}/{topic}/{version}/{partition}/checkpoint.offset
+// The namespace and topic are derived from the actual partition information
+type FilerOffsetStorage struct {
+	filerClientAccessor *filer_client.FilerClientAccessor
+}
+
+// NewFilerOffsetStorageWithAccessor creates a new filer-based offset storage using existing filer client accessor
+func NewFilerOffsetStorageWithAccessor(filerClientAccessor *filer_client.FilerClientAccessor) *FilerOffsetStorage {
+	return &FilerOffsetStorage{
+		filerClientAccessor: filerClientAccessor,
+	}
+}
+
+// SaveCheckpoint saves the checkpoint for a partition
+// Stores as: /topics/{namespace}/{topic}/{version}/{partition}/checkpoint.offset
+func (f *FilerOffsetStorage) SaveCheckpoint(namespace, topicName string, partition *schema_pb.Partition, offset int64) error {
+	partitionDir := f.getPartitionDir(namespace, topicName, partition)
+	fileName := "checkpoint.offset"
+
+	// Use SMQ's 8-byte offset format
+	offsetBytes := make([]byte, 8)
+	util.Uint64toBytes(offsetBytes, uint64(offset))
+
+	return f.filerClientAccessor.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		return filer.SaveInsideFiler(context.Background(), client, partitionDir, fileName, offsetBytes)
+	})
+}
+
+// LoadCheckpoint loads the checkpoint for a partition
+func (f *FilerOffsetStorage) LoadCheckpoint(namespace, topicName string, partition *schema_pb.Partition) (int64, error) {
+	partitionDir := f.getPartitionDir(namespace, topicName, partition)
+	fileName := "checkpoint.offset"
+
+	var offset int64 = -1
+	err := f.filerClientAccessor.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		data, err := filer.ReadInsideFiler(context.Background(), client, partitionDir, fileName)
+		if err != nil {
+			return err
+		}
+		if len(data) != 8 {
+			return fmt.Errorf("invalid checkpoint file format: expected 8 bytes, got %d", len(data))
+		}
+		offset = int64(util.BytesToUint64(data))
+		return nil
+	})
+
+	if err != nil {
+		return -1, err
+	}
+
+	return offset, nil
+}
+
+// GetHighestOffset returns the highest offset stored for a partition
+// For filer storage, this is the same as the checkpoint since we don't store individual records
+func (f *FilerOffsetStorage) GetHighestOffset(namespace, topicName string, partition *schema_pb.Partition) (int64, error) {
+	return f.LoadCheckpoint(namespace, topicName, partition)
+}
+
+// Reset clears all data for testing
+func (f *FilerOffsetStorage) Reset() error {
+	// For testing, we could delete all offset files, but this is dangerous
+	// Instead, just return success - individual tests should clean up their own data
+	return nil
+}
+
+// Helper methods
+
+// getPartitionDir returns the directory path for a partition following SMQ convention
+// Format: /topics/{namespace}/{topic}/{version}/{partition}
+func (f *FilerOffsetStorage) getPartitionDir(namespace, topicName string, partition *schema_pb.Partition) string {
+	// Generate version from UnixTimeNs
+	version := time.Unix(0, partition.UnixTimeNs).UTC().Format("v2006-01-02-15-04-05")
+
+	// Generate partition range string
+	partitionRange := fmt.Sprintf("%04d-%04d", partition.RangeStart, partition.RangeStop)
+
+	return fmt.Sprintf("%s/%s/%s/%s/%s", filer.TopicsDir, namespace, topicName, version, partitionRange)
+}

weed/mq/offset/integration.go

@@ -0,0 +1,387 @@
+package offset
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_agent_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+// SMQOffsetIntegration provides integration between offset management and SMQ broker
+type SMQOffsetIntegration struct {
+	mu               sync.RWMutex
+	registry         *PartitionOffsetRegistry
+	offsetAssigner   *OffsetAssigner
+	offsetSubscriber *OffsetSubscriber
+	offsetSeeker     *OffsetSeeker
+}
+
+// NewSMQOffsetIntegration creates a new SMQ offset integration
+func NewSMQOffsetIntegration(storage OffsetStorage) *SMQOffsetIntegration {
+	registry := NewPartitionOffsetRegistry(storage)
+	assigner := &OffsetAssigner{registry: registry}
+
+	return &SMQOffsetIntegration{
+		registry:         registry,
+		offsetAssigner:   assigner,
+		offsetSubscriber: NewOffsetSubscriber(registry),
+		offsetSeeker:     NewOffsetSeeker(registry),
+	}
+}
+
+// Close stops all background checkpoint goroutines and performs final checkpoints
+func (integration *SMQOffsetIntegration) Close() error {
+	return integration.registry.Close()
+}
+
+// PublishRecord publishes a record and assigns it an offset
+func (integration *SMQOffsetIntegration) PublishRecord(
+	namespace, topicName string,
+	partition *schema_pb.Partition,
+	key []byte,
+	value *schema_pb.RecordValue,
+) (*mq_agent_pb.PublishRecordResponse, error) {
+
+	// Assign offset for this record
+	result := integration.offsetAssigner.AssignSingleOffset(namespace, topicName, partition)
+	if result.Error != nil {
+		return &mq_agent_pb.PublishRecordResponse{
+			Error: fmt.Sprintf("Failed to assign offset: %v", result.Error),
+		}, nil
+	}
+
+	assignment := result.Assignment
+
+	// Note: Removed in-memory mapping storage to prevent memory leaks
+	// Record-to-offset mappings are now handled by persistent storage layer
+
+	// Return response with offset information
+	return &mq_agent_pb.PublishRecordResponse{
+		AckSequence: assignment.Offset, // Use offset as ack sequence for now
+		BaseOffset:  assignment.Offset,
+		LastOffset:  assignment.Offset,
+		Error:       "",
+	}, nil
+}
+
+// PublishRecordBatch publishes a batch of records and assigns them offsets
+func (integration *SMQOffsetIntegration) PublishRecordBatch(
+	namespace, topicName string,
+	partition *schema_pb.Partition,
+	records []PublishRecordRequest,
+) (*mq_agent_pb.PublishRecordResponse, error) {
+
+	if len(records) == 0 {
+		return &mq_agent_pb.PublishRecordResponse{
+			Error: "Empty record batch",
+		}, nil
+	}
+
+	// Assign batch of offsets
+	result := integration.offsetAssigner.AssignBatchOffsets(namespace, topicName, partition, int64(len(records)))
+	if result.Error != nil {
+		return &mq_agent_pb.PublishRecordResponse{
+			Error: fmt.Sprintf("Failed to assign batch offsets: %v", result.Error),
+		}, nil
+	}
+
+	batch := result.Batch
+
+	// Note: Removed in-memory mapping storage to prevent memory leaks
+	// Batch record-to-offset mappings are now handled by persistent storage layer
+
+	return &mq_agent_pb.PublishRecordResponse{
+		AckSequence: batch.LastOffset, // Use last offset as ack sequence
+		BaseOffset:  batch.BaseOffset,
+		LastOffset:  batch.LastOffset,
+		Error:       "",
+	}, nil
+}
+
+// CreateSubscription creates an offset-based subscription
+func (integration *SMQOffsetIntegration) CreateSubscription(
+	subscriptionID string,
+	namespace, topicName string,
+	partition *schema_pb.Partition,
+	offsetType schema_pb.OffsetType,
+	startOffset int64,
+) (*OffsetSubscription, error) {
+
+	return integration.offsetSubscriber.CreateSubscription(
+		subscriptionID,
+		namespace, topicName,
+		partition,
+		offsetType,
+		startOffset,
+	)
+}
+
+// SubscribeRecords subscribes to records starting from a specific offset
+func (integration *SMQOffsetIntegration) SubscribeRecords(
+	subscription *OffsetSubscription,
+	maxRecords int64,
+) ([]*mq_agent_pb.SubscribeRecordResponse, error) {
+
+	if !subscription.IsActive {
+		return nil, fmt.Errorf("subscription is not active")
+	}
+
+	// Get the range of offsets to read
+	offsetRange, err := subscription.GetOffsetRange(maxRecords)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get offset range: %w", err)
+	}
+
+	if offsetRange.Count == 0 {
+		// No records available
+		return []*mq_agent_pb.SubscribeRecordResponse{}, nil
+	}
+
+	// TODO: This is where we would integrate with SMQ's actual storage layer
+	// For now, return mock responses with offset information
+	responses := make([]*mq_agent_pb.SubscribeRecordResponse, offsetRange.Count)
+
+	for i := int64(0); i < offsetRange.Count; i++ {
+		offset := offsetRange.StartOffset + i
+
+		responses[i] = &mq_agent_pb.SubscribeRecordResponse{
+			Key:           []byte(fmt.Sprintf("key-%d", offset)),
+			Value:         &schema_pb.RecordValue{}, // Mock value
+			TsNs:          offset * 1000000,         // Mock timestamp based on offset
+			Offset:        offset,
+			IsEndOfStream: false,
+			IsEndOfTopic:  false,
+			Error:         "",
+		}
+	}
+
+	// Advance the subscription
+	subscription.AdvanceOffsetBy(offsetRange.Count)
+
+	return responses, nil
+}
+
+// GetHighWaterMark returns the high water mark for a partition
+func (integration *SMQOffsetIntegration) GetHighWaterMark(namespace, topicName string, partition *schema_pb.Partition) (int64, error) {
+	return integration.offsetAssigner.GetHighWaterMark(namespace, topicName, partition)
+}
+
+// SeekSubscription seeks a subscription to a specific offset
+func (integration *SMQOffsetIntegration) SeekSubscription(
+	subscriptionID string,
+	offset int64,
+) error {
+
+	subscription, err := integration.offsetSubscriber.GetSubscription(subscriptionID)
+	if err != nil {
+		return fmt.Errorf("subscription not found: %w", err)
+	}
+
+	return subscription.SeekToOffset(offset)
+}
+
+// GetSubscriptionLag returns the lag for a subscription
+func (integration *SMQOffsetIntegration) GetSubscriptionLag(subscriptionID string) (int64, error) {
+	subscription, err := integration.offsetSubscriber.GetSubscription(subscriptionID)
+	if err != nil {
+		return 0, fmt.Errorf("subscription not found: %w", err)
+	}
+
+	return subscription.GetLag()
+}
+
+// CloseSubscription closes a subscription
+func (integration *SMQOffsetIntegration) CloseSubscription(subscriptionID string) error {
+	return integration.offsetSubscriber.CloseSubscription(subscriptionID)
+}
+
+// ValidateOffsetRange validates an offset range for a partition
+func (integration *SMQOffsetIntegration) ValidateOffsetRange(
+	namespace, topicName string,
+	partition *schema_pb.Partition,
+	startOffset, endOffset int64,
+) error {
+
+	return integration.offsetSeeker.ValidateOffsetRange(namespace, topicName, partition, startOffset, endOffset)
+}
+
+// GetAvailableOffsetRange returns the available offset range for a partition
+func (integration *SMQOffsetIntegration) GetAvailableOffsetRange(namespace, topicName string, partition *schema_pb.Partition) (*OffsetRange, error) {
+	return integration.offsetSeeker.GetAvailableOffsetRange(namespace, topicName, partition)
+}
+
+// PublishRecordRequest represents a record to be published
+type PublishRecordRequest struct {
+	Key   []byte
+	Value *schema_pb.RecordValue
+}
+
+// OffsetMetrics provides metrics about offset usage
+type OffsetMetrics struct {
+	PartitionCount      int64
+	TotalOffsets        int64
+	ActiveSubscriptions int64
+	AverageLatency      float64
+}
+
+// GetOffsetMetrics returns metrics about offset usage
+func (integration *SMQOffsetIntegration) GetOffsetMetrics() *OffsetMetrics {
+	integration.mu.RLock()
+	defer integration.mu.RUnlock()
+
+	// Count active subscriptions
+	activeSubscriptions := int64(0)
+	for _, subscription := range integration.offsetSubscriber.subscriptions {
+		if subscription.IsActive {
+			activeSubscriptions++
+		}
+	}
+
+	// Calculate total offsets from all partition managers instead of in-memory map
+	var totalOffsets int64
+	for _, manager := range integration.offsetAssigner.registry.managers {
+		totalOffsets += manager.GetHighWaterMark()
+	}
+
+	return &OffsetMetrics{
+		PartitionCount:      int64(len(integration.offsetAssigner.registry.managers)),
+		TotalOffsets:        totalOffsets, // Now calculated from storage, not memory maps
+		ActiveSubscriptions: activeSubscriptions,
+		AverageLatency:      0.0, // TODO: Implement latency tracking
+	}
+}
+
+// OffsetInfo provides detailed information about an offset
+type OffsetInfo struct {
+	Offset    int64
+	Timestamp int64
+	Partition *schema_pb.Partition
+	Exists    bool
+}
+
+// GetOffsetInfo returns detailed information about a specific offset
+func (integration *SMQOffsetIntegration) GetOffsetInfo(
+	namespace, topicName string,
+	partition *schema_pb.Partition,
+	offset int64,
+) (*OffsetInfo, error) {
+
+	hwm, err := integration.GetHighWaterMark(namespace, topicName, partition)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get high water mark: %w", err)
+	}
+
+	exists := offset >= 0 && offset < hwm
+
+	// TODO: Get actual timestamp from storage
+	timestamp := int64(0)
+	// Note: Timestamp lookup from in-memory map removed to prevent memory leaks
+	// For now, use a placeholder timestamp. In production, this should come from
+	// persistent storage if timestamp tracking is needed.
+	if exists {
+		timestamp = time.Now().UnixNano() // Placeholder - should come from storage
+	}
+
+	return &OffsetInfo{
+		Offset:    offset,
+		Timestamp: timestamp,
+		Partition: partition,
+		Exists:    exists,
+	}, nil
+}
+
+// PartitionOffsetInfo provides offset information for a partition
+type PartitionOffsetInfo struct {
+	Partition           *schema_pb.Partition
+	EarliestOffset      int64
+	LatestOffset        int64
+	HighWaterMark       int64
+	RecordCount         int64
+	ActiveSubscriptions int64
+}
+
+// GetPartitionOffsetInfo returns comprehensive offset information for a partition
+func (integration *SMQOffsetIntegration) GetPartitionOffsetInfo(namespace, topicName string, partition *schema_pb.Partition) (*PartitionOffsetInfo, error) {
+	hwm, err := integration.GetHighWaterMark(namespace, topicName, partition)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get high water mark: %w", err)
+	}
+
+	earliestOffset := int64(0)
+	latestOffset := hwm - 1
+	if hwm == 0 {
+		latestOffset = -1 // No records
+	}
+
+	// Count active subscriptions for this partition
+	activeSubscriptions := int64(0)
+	integration.mu.RLock()
+	for _, subscription := range integration.offsetSubscriber.subscriptions {
+		if subscription.IsActive && partitionKey(subscription.Partition) == partitionKey(partition) {
+			activeSubscriptions++
+		}
+	}
+	integration.mu.RUnlock()
+
+	return &PartitionOffsetInfo{
+		Partition:           partition,
+		EarliestOffset:      earliestOffset,
+		LatestOffset:        latestOffset,
+		HighWaterMark:       hwm,
+		RecordCount:         hwm,
+		ActiveSubscriptions: activeSubscriptions,
+	}, nil
+}
+
+// GetSubscription retrieves an existing subscription
+func (integration *SMQOffsetIntegration) GetSubscription(subscriptionID string) (*OffsetSubscription, error) {
+	return integration.offsetSubscriber.GetSubscription(subscriptionID)
+}
+
+// ListActiveSubscriptions returns all active subscriptions
+func (integration *SMQOffsetIntegration) ListActiveSubscriptions() ([]*OffsetSubscription, error) {
+	integration.mu.RLock()
+	defer integration.mu.RUnlock()
+
+	result := make([]*OffsetSubscription, 0)
+	for _, subscription := range integration.offsetSubscriber.subscriptions {
+		if subscription.IsActive {
+			result = append(result, subscription)
+		}
+	}
+
+	return result, nil
+}
+
+// AssignSingleOffset assigns a single offset for a partition
+func (integration *SMQOffsetIntegration) AssignSingleOffset(namespace, topicName string, partition *schema_pb.Partition) *AssignmentResult {
+	return integration.offsetAssigner.AssignSingleOffset(namespace, topicName, partition)
+}
+
+// AssignBatchOffsets assigns a batch of offsets for a partition
+func (integration *SMQOffsetIntegration) AssignBatchOffsets(namespace, topicName string, partition *schema_pb.Partition, count int64) *AssignmentResult {
+	return integration.offsetAssigner.AssignBatchOffsets(namespace, topicName, partition, count)
+}
+
+// Reset resets the integration layer state (for testing)
+func (integration *SMQOffsetIntegration) Reset() {
+	integration.mu.Lock()
+	defer integration.mu.Unlock()
+
+	// Note: No in-memory maps to clear (removed to prevent memory leaks)
+
+	// Close all subscriptions
+	for _, subscription := range integration.offsetSubscriber.subscriptions {
+		subscription.IsActive = false
+	}
+	integration.offsetSubscriber.subscriptions = make(map[string]*OffsetSubscription)
+
+	// Reset the registries by creating new ones with the same storage
+	// This ensures that partition managers start fresh
+	registry := NewPartitionOffsetRegistry(integration.offsetAssigner.registry.storage)
+	integration.offsetAssigner.registry = registry
+	integration.offsetSubscriber.offsetRegistry = registry
+	integration.offsetSeeker.offsetRegistry = registry
+}

weed/mq/offset/manager.go

@@ -0,0 +1,378 @@
+package offset
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+// PartitionOffsetManager manages sequential offset assignment for a single partition
+type PartitionOffsetManager struct {
+	mu         sync.RWMutex
+	namespace  string
+	topicName  string
+	partition  *schema_pb.Partition
+	nextOffset int64
+
+	// Checkpointing for recovery
+	lastCheckpoint         int64
+	lastCheckpointedOffset int64
+	storage                OffsetStorage
+
+	// Background checkpointing
+	stopCheckpoint chan struct{}
+}
+
+// OffsetStorage interface for persisting offset state
+type OffsetStorage interface {
+	// SaveCheckpoint persists the current offset state for recovery
+	// Takes topic information along with partition to determine the correct storage location
+	SaveCheckpoint(namespace, topicName string, partition *schema_pb.Partition, offset int64) error
+
+	// LoadCheckpoint retrieves the last saved offset state
+	LoadCheckpoint(namespace, topicName string, partition *schema_pb.Partition) (int64, error)
+
+	// GetHighestOffset scans storage to find the highest assigned offset
+	GetHighestOffset(namespace, topicName string, partition *schema_pb.Partition) (int64, error)
+}
+
+// NewPartitionOffsetManager creates a new offset manager for a partition
+func NewPartitionOffsetManager(namespace, topicName string, partition *schema_pb.Partition, storage OffsetStorage) (*PartitionOffsetManager, error) {
+	manager := &PartitionOffsetManager{
+		namespace:      namespace,
+		topicName:      topicName,
+		partition:      partition,
+		storage:        storage,
+		stopCheckpoint: make(chan struct{}),
+	}
+
+	// Recover offset state
+	if err := manager.recover(); err != nil {
+		return nil, fmt.Errorf("failed to recover offset state: %w", err)
+	}
+
+	// Start background checkpoint goroutine
+	go manager.runPeriodicCheckpoint()
+
+	return manager, nil
+}
+
+// Close stops the background checkpoint goroutine and performs a final checkpoint
+func (m *PartitionOffsetManager) Close() error {
+	close(m.stopCheckpoint)
+
+	// Perform final checkpoint
+	m.mu.RLock()
+	currentOffset := m.nextOffset - 1 // Last assigned offset
+	lastCheckpointed := m.lastCheckpointedOffset
+	m.mu.RUnlock()
+
+	if currentOffset >= 0 && currentOffset > lastCheckpointed {
+		return m.storage.SaveCheckpoint(m.namespace, m.topicName, m.partition, currentOffset)
+	}
+	return nil
+}
+
+// AssignOffset assigns the next sequential offset
+func (m *PartitionOffsetManager) AssignOffset() int64 {
+	m.mu.Lock()
+	offset := m.nextOffset
+	m.nextOffset++
+	m.mu.Unlock()
+
+	return offset
+}
+
+// AssignOffsets assigns a batch of sequential offsets
+func (m *PartitionOffsetManager) AssignOffsets(count int64) (baseOffset int64, lastOffset int64) {
+	m.mu.Lock()
+	baseOffset = m.nextOffset
+	lastOffset = m.nextOffset + count - 1
+	m.nextOffset += count
+	m.mu.Unlock()
+
+	return baseOffset, lastOffset
+}
+
+// GetNextOffset returns the next offset that will be assigned
+func (m *PartitionOffsetManager) GetNextOffset() int64 {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.nextOffset
+}
+
+// GetHighWaterMark returns the high water mark (next offset)
+func (m *PartitionOffsetManager) GetHighWaterMark() int64 {
+	return m.GetNextOffset()
+}
+
+// recover restores offset state from storage
+func (m *PartitionOffsetManager) recover() error {
+	var checkpointOffset int64 = -1
+	var highestOffset int64 = -1
+
+	// Try to load checkpoint
+	if offset, err := m.storage.LoadCheckpoint(m.namespace, m.topicName, m.partition); err == nil && offset >= 0 {
+		checkpointOffset = offset
+	}
+
+	// Try to scan storage for highest offset
+	if offset, err := m.storage.GetHighestOffset(m.namespace, m.topicName, m.partition); err == nil && offset >= 0 {
+		highestOffset = offset
+	}
+
+	// Use the higher of checkpoint or storage scan
+	if checkpointOffset >= 0 && highestOffset >= 0 {
+		if highestOffset > checkpointOffset {
+			m.nextOffset = highestOffset + 1
+			m.lastCheckpoint = highestOffset
+			m.lastCheckpointedOffset = highestOffset
+		} else {
+			m.nextOffset = checkpointOffset + 1
+			m.lastCheckpoint = checkpointOffset
+			m.lastCheckpointedOffset = checkpointOffset
+		}
+	} else if checkpointOffset >= 0 {
+		m.nextOffset = checkpointOffset + 1
+		m.lastCheckpoint = checkpointOffset
+		m.lastCheckpointedOffset = checkpointOffset
+	} else if highestOffset >= 0 {
+		m.nextOffset = highestOffset + 1
+		m.lastCheckpoint = highestOffset
+		m.lastCheckpointedOffset = highestOffset
+	} else {
+		// No data exists, start from 0
+		m.nextOffset = 0
+		m.lastCheckpoint = -1
+		m.lastCheckpointedOffset = -1
+	}
+
+	return nil
+}
+
+// runPeriodicCheckpoint runs in the background and checkpoints every 2 seconds if the offset changed
+func (m *PartitionOffsetManager) runPeriodicCheckpoint() {
+	ticker := time.NewTicker(2 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			m.performCheckpointIfChanged()
+		case <-m.stopCheckpoint:
+			return
+		}
+	}
+}
+
+// performCheckpointIfChanged saves checkpoint only if offset has changed since last checkpoint
+func (m *PartitionOffsetManager) performCheckpointIfChanged() {
+	m.mu.RLock()
+	currentOffset := m.nextOffset - 1 // Last assigned offset
+	lastCheckpointed := m.lastCheckpointedOffset
+	m.mu.RUnlock()
+
+	// Skip if no messages have been assigned, or no change since last checkpoint
+	if currentOffset < 0 || currentOffset == lastCheckpointed {
+		return
+	}
+
+	// Perform checkpoint
+	if err := m.storage.SaveCheckpoint(m.namespace, m.topicName, m.partition, currentOffset); err != nil {
+		// Log error but don't fail - checkpointing is for optimization
+		fmt.Printf("Failed to checkpoint offset %d for %s/%s: %v\n", currentOffset, m.namespace, m.topicName, err)
+		return
+	}
+
+	// Update last checkpointed offset
+	m.mu.Lock()
+	m.lastCheckpointedOffset = currentOffset
+	m.lastCheckpoint = currentOffset
+	m.mu.Unlock()
+}
+
+// PartitionOffsetRegistry manages offset managers for multiple partitions
+type PartitionOffsetRegistry struct {
+	mu       sync.RWMutex
+	managers map[string]*PartitionOffsetManager
+	storage  OffsetStorage
+}
+
+// NewPartitionOffsetRegistry creates a new registry
+func NewPartitionOffsetRegistry(storage OffsetStorage) *PartitionOffsetRegistry {
+	return &PartitionOffsetRegistry{
+		managers: make(map[string]*PartitionOffsetManager),
+		storage:  storage,
+	}
+}
+
+// GetManager returns the offset manager for a partition, creating it if needed
+func (r *PartitionOffsetRegistry) GetManager(namespace, topicName string, partition *schema_pb.Partition) (*PartitionOffsetManager, error) {
+	// CRITICAL FIX: Use TopicPartitionKey to ensure each topic has its own offset manager
+	key := TopicPartitionKey(namespace, topicName, partition)
+
+	r.mu.RLock()
+	manager, exists := r.managers[key]
+	r.mu.RUnlock()
+
+	if exists {
+		return manager, nil
+	}
+
+	// Create new manager
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	// Double-check after acquiring write lock
+	if manager, exists := r.managers[key]; exists {
+		return manager, nil
+	}
+
+	manager, err := NewPartitionOffsetManager(namespace, topicName, partition, r.storage)
+	if err != nil {
+		return nil, err
+	}
+
+	r.managers[key] = manager
+	return manager, nil
+}
+
+// AssignOffset assigns an offset for the given partition
+func (r *PartitionOffsetRegistry) AssignOffset(namespace, topicName string, partition *schema_pb.Partition) (int64, error) {
+	manager, err := r.GetManager(namespace, topicName, partition)
+	if err != nil {
+		return 0, err
+	}
+
+	assignedOffset := manager.AssignOffset()
+
+	return assignedOffset, nil
+}
+
+// AssignOffsets assigns a batch of offsets for the given partition
+func (r *PartitionOffsetRegistry) AssignOffsets(namespace, topicName string, partition *schema_pb.Partition, count int64) (baseOffset, lastOffset int64, err error) {
+	manager, err := r.GetManager(namespace, topicName, partition)
+	if err != nil {
+		return 0, 0, err
+	}
+
+	baseOffset, lastOffset = manager.AssignOffsets(count)
+	return baseOffset, lastOffset, nil
+}
+
+// GetHighWaterMark returns the high water mark for a partition
+func (r *PartitionOffsetRegistry) GetHighWaterMark(namespace, topicName string, partition *schema_pb.Partition) (int64, error) {
+	manager, err := r.GetManager(namespace, topicName, partition)
+	if err != nil {
+		return 0, err
+	}
+
+	return manager.GetHighWaterMark(), nil
+}
+
+// Close stops all partition managers and performs final checkpoints
+func (r *PartitionOffsetRegistry) Close() error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	var firstErr error
+	for _, manager := range r.managers {
+		if err := manager.Close(); err != nil && firstErr == nil {
+			firstErr = err
+		}
+	}
+
+	return firstErr
+}
+
+// TopicPartitionKey generates a unique key for a topic-partition combination
+// This is the canonical key format used across the offset management system
+func TopicPartitionKey(namespace, topicName string, partition *schema_pb.Partition) string {
+	return fmt.Sprintf("%s/%s/ring:%d:range:%d-%d",
+		namespace, topicName,
+		partition.RingSize, partition.RangeStart, partition.RangeStop)
+}
+
+// PartitionKey generates a unique key for a partition (without topic context)
+// Note: UnixTimeNs is intentionally excluded from the key because it represents
+// partition creation time, not partition identity. Using it would cause offset
+// tracking to reset whenever a partition is recreated or looked up again.
+// DEPRECATED: Use TopicPartitionKey for production code to avoid key collisions
+func PartitionKey(partition *schema_pb.Partition) string {
+	return fmt.Sprintf("ring:%d:range:%d-%d",
+		partition.RingSize, partition.RangeStart, partition.RangeStop)
+}
+
+// partitionKey is the internal lowercase version for backward compatibility within this package
+func partitionKey(partition *schema_pb.Partition) string {
+	return PartitionKey(partition)
+}
+
+// OffsetAssignment represents an assigned offset with metadata
+type OffsetAssignment struct {
+	Offset    int64
+	Timestamp int64
+	Partition *schema_pb.Partition
+}
+
+// BatchOffsetAssignment represents a batch of assigned offsets
+type BatchOffsetAssignment struct {
+	BaseOffset int64
+	LastOffset int64
+	Count      int64
+	Timestamp  int64
+	Partition  *schema_pb.Partition
+}
+
+// AssignmentResult contains the result of offset assignment
+type AssignmentResult struct {
+	Assignment *OffsetAssignment
+	Batch      *BatchOffsetAssignment
+	Error      error
+}
+
+// OffsetAssigner provides high-level offset assignment operations
+type OffsetAssigner struct {
+	registry *PartitionOffsetRegistry
+}
+
+// AssignSingleOffset assigns a single offset with timestamp
+func (a *OffsetAssigner) AssignSingleOffset(namespace, topicName string, partition *schema_pb.Partition) *AssignmentResult {
+	offset, err := a.registry.AssignOffset(namespace, topicName, partition)
+	if err != nil {
+		return &AssignmentResult{Error: err}
+	}
+
+	return &AssignmentResult{
+		Assignment: &OffsetAssignment{
+			Offset:    offset,
+			Timestamp: time.Now().UnixNano(),
+			Partition: partition,
+		},
+	}
+}
+
+// AssignBatchOffsets assigns a batch of offsets with timestamp
+func (a *OffsetAssigner) AssignBatchOffsets(namespace, topicName string, partition *schema_pb.Partition, count int64) *AssignmentResult {
+	baseOffset, lastOffset, err := a.registry.AssignOffsets(namespace, topicName, partition, count)
+	if err != nil {
+		return &AssignmentResult{Error: err}
+	}
+
+	return &AssignmentResult{
+		Batch: &BatchOffsetAssignment{
+			BaseOffset: baseOffset,
+			LastOffset: lastOffset,
+			Count:      count,
+			Timestamp:  time.Now().UnixNano(),
+			Partition:  partition,
+		},
+	}
+}
+
+// GetHighWaterMark returns the high water mark for a partition
+func (a *OffsetAssigner) GetHighWaterMark(namespace, topicName string, partition *schema_pb.Partition) (int64, error) {
+	return a.registry.GetHighWaterMark(namespace, topicName, partition)
+}

weed/mq/offset/memory_storage_test.go

@@ -0,0 +1,228 @@
+package offset
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+// recordEntry holds a record with timestamp for TTL cleanup
+type recordEntry struct {
+	exists    bool
+	timestamp time.Time
+}
+
+// InMemoryOffsetStorage provides an in-memory implementation of OffsetStorage for testing ONLY
+// WARNING: This should NEVER be used in production - use FilerOffsetStorage or SQLOffsetStorage instead
+type InMemoryOffsetStorage struct {
+	mu          sync.RWMutex
+	checkpoints map[string]int64                  // partition key -> offset
+	records     map[string]map[int64]*recordEntry // partition key -> offset -> entry with timestamp
+
+	// Memory leak protection
+	maxRecordsPerPartition int           // Maximum records to keep per partition
+	recordTTL              time.Duration // TTL for record entries
+	lastCleanup            time.Time     // Last cleanup time
+	cleanupInterval        time.Duration // How often to run cleanup
+}
+
+// NewInMemoryOffsetStorage creates a new in-memory storage with memory leak protection
+// FOR TESTING ONLY - do not use in production
+func NewInMemoryOffsetStorage() *InMemoryOffsetStorage {
+	return &InMemoryOffsetStorage{
+		checkpoints:            make(map[string]int64),
+		records:                make(map[string]map[int64]*recordEntry),
+		maxRecordsPerPartition: 10000,           // Limit to 10K records per partition
+		recordTTL:              1 * time.Hour,   // Records expire after 1 hour
+		cleanupInterval:        5 * time.Minute, // Cleanup every 5 minutes
+		lastCleanup:            time.Now(),
+	}
+}
+
+// SaveCheckpoint saves the checkpoint for a partition
+func (s *InMemoryOffsetStorage) SaveCheckpoint(namespace, topicName string, partition *schema_pb.Partition, offset int64) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// Use TopicPartitionKey for consistency with other storage implementations
+	key := TopicPartitionKey(namespace, topicName, partition)
+	s.checkpoints[key] = offset
+	return nil
+}
+
+// LoadCheckpoint loads the checkpoint for a partition
+func (s *InMemoryOffsetStorage) LoadCheckpoint(namespace, topicName string, partition *schema_pb.Partition) (int64, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	// Use TopicPartitionKey to match SaveCheckpoint
+	key := TopicPartitionKey(namespace, topicName, partition)
+	offset, exists := s.checkpoints[key]
+	if !exists {
+		return -1, fmt.Errorf("no checkpoint found")
+	}
+
+	return offset, nil
+}
+
+// GetHighestOffset finds the highest offset in storage for a partition
+func (s *InMemoryOffsetStorage) GetHighestOffset(namespace, topicName string, partition *schema_pb.Partition) (int64, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	// Use TopicPartitionKey to match SaveCheckpoint
+	key := TopicPartitionKey(namespace, topicName, partition)
+	offsets, exists := s.records[key]
+	if !exists || len(offsets) == 0 {
+		return -1, fmt.Errorf("no records found")
+	}
+
+	var highest int64 = -1
+	for offset, entry := range offsets {
+		if entry.exists && offset > highest {
+			highest = offset
+		}
+	}
+
+	return highest, nil
+}
+
+// AddRecord simulates storing a record with an offset (for testing)
+func (s *InMemoryOffsetStorage) AddRecord(namespace, topicName string, partition *schema_pb.Partition, offset int64) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// Use TopicPartitionKey to match GetHighestOffset
+	key := TopicPartitionKey(namespace, topicName, partition)
+	if s.records[key] == nil {
+		s.records[key] = make(map[int64]*recordEntry)
+	}
+
+	// Add record with current timestamp
+	s.records[key][offset] = &recordEntry{
+		exists:    true,
+		timestamp: time.Now(),
+	}
+
+	// Trigger cleanup if needed (memory leak protection)
+	s.cleanupIfNeeded()
+}
+
+// GetRecordCount returns the number of records for a partition (for testing)
+func (s *InMemoryOffsetStorage) GetRecordCount(namespace, topicName string, partition *schema_pb.Partition) int {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	// Use TopicPartitionKey to match GetHighestOffset
+	key := TopicPartitionKey(namespace, topicName, partition)
+	if offsets, exists := s.records[key]; exists {
+		count := 0
+		for _, entry := range offsets {
+			if entry.exists {
+				count++
+			}
+		}
+		return count
+	}
+	return 0
+}
+
+// Clear removes all data (for testing)
+func (s *InMemoryOffsetStorage) Clear() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.checkpoints = make(map[string]int64)
+	s.records = make(map[string]map[int64]*recordEntry)
+	s.lastCleanup = time.Now()
+}
+
+// Reset removes all data (implements resettable interface for shutdown)
+func (s *InMemoryOffsetStorage) Reset() error {
+	s.Clear()
+	return nil
+}
+
+// cleanupIfNeeded performs memory leak protection cleanup
+// This method assumes the caller already holds the write lock
+func (s *InMemoryOffsetStorage) cleanupIfNeeded() {
+	now := time.Now()
+
+	// Only cleanup if enough time has passed
+	if now.Sub(s.lastCleanup) < s.cleanupInterval {
+		return
+	}
+
+	s.lastCleanup = now
+	cutoff := now.Add(-s.recordTTL)
+
+	// Clean up expired records and enforce size limits
+	for partitionKey, offsets := range s.records {
+		// Remove expired records
+		for offset, entry := range offsets {
+			if entry.timestamp.Before(cutoff) {
+				delete(offsets, offset)
+			}
+		}
+
+		// Enforce size limit per partition
+		if len(offsets) > s.maxRecordsPerPartition {
+			// Keep only the most recent records
+			type offsetTime struct {
+				offset int64
+				time   time.Time
+			}
+
+			var entries []offsetTime
+			for offset, entry := range offsets {
+				entries = append(entries, offsetTime{offset: offset, time: entry.timestamp})
+			}
+
+			// Sort by timestamp (newest first)
+			for i := 0; i < len(entries)-1; i++ {
+				for j := i + 1; j < len(entries); j++ {
+					if entries[i].time.Before(entries[j].time) {
+						entries[i], entries[j] = entries[j], entries[i]
+					}
+				}
+			}
+
+			// Keep only the newest maxRecordsPerPartition entries
+			newOffsets := make(map[int64]*recordEntry)
+			for i := 0; i < s.maxRecordsPerPartition && i < len(entries); i++ {
+				offset := entries[i].offset
+				newOffsets[offset] = offsets[offset]
+			}
+
+			s.records[partitionKey] = newOffsets
+		}
+
+		// Remove empty partition maps
+		if len(offsets) == 0 {
+			delete(s.records, partitionKey)
+		}
+	}
+}
+
+// GetMemoryStats returns memory usage statistics for monitoring
+func (s *InMemoryOffsetStorage) GetMemoryStats() map[string]interface{} {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	totalRecords := 0
+	partitionCount := len(s.records)
+
+	for _, offsets := range s.records {
+		totalRecords += len(offsets)
+	}
+
+	return map[string]interface{}{
+		"total_partitions":          partitionCount,
+		"total_records":             totalRecords,
+		"max_records_per_partition": s.maxRecordsPerPartition,
+		"record_ttl_hours":          s.recordTTL.Hours(),
+		"last_cleanup":              s.lastCleanup,
+	}
+}

weed/mq/offset/storage.go

@@ -0,0 +1,5 @@
+package offset
+
+// Note: OffsetStorage interface is defined in manager.go
+// Production implementations: FilerOffsetStorage (filer_storage.go), SQLOffsetStorage (sql_storage.go)
+// Test implementation: InMemoryOffsetStorage (storage_test.go)

weed/mq/offset/subscriber.go

@@ -0,0 +1,355 @@
+package offset
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+// OffsetSubscriber handles offset-based subscription logic
+type OffsetSubscriber struct {
+	mu             sync.RWMutex
+	offsetRegistry *PartitionOffsetRegistry
+	subscriptions  map[string]*OffsetSubscription
+}
+
+// OffsetSubscription represents an active offset-based subscription
+type OffsetSubscription struct {
+	ID             string
+	Namespace      string
+	TopicName      string
+	Partition      *schema_pb.Partition
+	StartOffset    int64
+	CurrentOffset  int64
+	OffsetType     schema_pb.OffsetType
+	IsActive       bool
+	offsetRegistry *PartitionOffsetRegistry
+}
+
+// NewOffsetSubscriber creates a new offset-based subscriber
+func NewOffsetSubscriber(offsetRegistry *PartitionOffsetRegistry) *OffsetSubscriber {
+	return &OffsetSubscriber{
+		offsetRegistry: offsetRegistry,
+		subscriptions:  make(map[string]*OffsetSubscription),
+	}
+}
+
+// CreateSubscription creates a new offset-based subscription
+func (s *OffsetSubscriber) CreateSubscription(
+	subscriptionID string,
+	namespace, topicName string,
+	partition *schema_pb.Partition,
+	offsetType schema_pb.OffsetType,
+	startOffset int64,
+) (*OffsetSubscription, error) {
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// Check if subscription already exists
+	if _, exists := s.subscriptions[subscriptionID]; exists {
+		return nil, fmt.Errorf("subscription %s already exists", subscriptionID)
+	}
+
+	// Resolve the actual start offset based on type
+	actualStartOffset, err := s.resolveStartOffset(namespace, topicName, partition, offsetType, startOffset)
+	if err != nil {
+		return nil, fmt.Errorf("failed to resolve start offset: %w", err)
+	}
+
+	subscription := &OffsetSubscription{
+		ID:             subscriptionID,
+		Namespace:      namespace,
+		TopicName:      topicName,
+		Partition:      partition,
+		StartOffset:    actualStartOffset,
+		CurrentOffset:  actualStartOffset,
+		OffsetType:     offsetType,
+		IsActive:       true,
+		offsetRegistry: s.offsetRegistry,
+	}
+
+	s.subscriptions[subscriptionID] = subscription
+	return subscription, nil
+}
+
+// GetSubscription retrieves an existing subscription
+func (s *OffsetSubscriber) GetSubscription(subscriptionID string) (*OffsetSubscription, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	subscription, exists := s.subscriptions[subscriptionID]
+	if !exists {
+		return nil, fmt.Errorf("subscription %s not found", subscriptionID)
+	}
+
+	return subscription, nil
+}
+
+// CloseSubscription closes and removes a subscription
+func (s *OffsetSubscriber) CloseSubscription(subscriptionID string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	subscription, exists := s.subscriptions[subscriptionID]
+	if !exists {
+		return fmt.Errorf("subscription %s not found", subscriptionID)
+	}
+
+	subscription.IsActive = false
+	delete(s.subscriptions, subscriptionID)
+	return nil
+}
+
+// resolveStartOffset resolves the actual start offset based on OffsetType
+func (s *OffsetSubscriber) resolveStartOffset(
+	namespace, topicName string,
+	partition *schema_pb.Partition,
+	offsetType schema_pb.OffsetType,
+	requestedOffset int64,
+) (int64, error) {
+
+	switch offsetType {
+	case schema_pb.OffsetType_EXACT_OFFSET:
+		// Validate that the requested offset exists
+		return s.validateAndGetOffset(namespace, topicName, partition, requestedOffset)
+
+	case schema_pb.OffsetType_RESET_TO_OFFSET:
+		// Use the requested offset, even if it doesn't exist yet
+		return requestedOffset, nil
+
+	case schema_pb.OffsetType_RESET_TO_EARLIEST:
+		// Start from offset 0
+		return 0, nil
+
+	case schema_pb.OffsetType_RESET_TO_LATEST:
+		// Start from the current high water mark
+		hwm, err := s.offsetRegistry.GetHighWaterMark(namespace, topicName, partition)
+		if err != nil {
+			return 0, err
+		}
+		return hwm, nil
+
+	case schema_pb.OffsetType_RESUME_OR_EARLIEST:
+		// Try to resume from a saved position, fallback to earliest
+		// For now, just use earliest (consumer group position tracking will be added later)
+		return 0, nil
+
+	case schema_pb.OffsetType_RESUME_OR_LATEST:
+		// Try to resume from a saved position, fallback to latest
+		// For now, just use latest
+		hwm, err := s.offsetRegistry.GetHighWaterMark(namespace, topicName, partition)
+		if err != nil {
+			return 0, err
+		}
+		return hwm, nil
+
+	default:
+		return 0, fmt.Errorf("unsupported offset type: %v", offsetType)
+	}
+}
+
+// validateAndGetOffset validates that an offset exists and returns it
+func (s *OffsetSubscriber) validateAndGetOffset(namespace, topicName string, partition *schema_pb.Partition, offset int64) (int64, error) {
+	if offset < 0 {
+		return 0, fmt.Errorf("offset cannot be negative: %d", offset)
+	}
+
+	// Get the current high water mark
+	hwm, err := s.offsetRegistry.GetHighWaterMark(namespace, topicName, partition)
+	if err != nil {
+		return 0, fmt.Errorf("failed to get high water mark: %w", err)
+	}
+
+	// Check if offset is within valid range
+	if offset >= hwm {
+		return 0, fmt.Errorf("offset %d is beyond high water mark %d", offset, hwm)
+	}
+
+	return offset, nil
+}
+
+// SeekToOffset seeks a subscription to a specific offset
+func (sub *OffsetSubscription) SeekToOffset(offset int64) error {
+	if !sub.IsActive {
+		return fmt.Errorf("subscription is not active")
+	}
+
+	// Validate the offset
+	if offset < 0 {
+		return fmt.Errorf("offset cannot be negative: %d", offset)
+	}
+
+	hwm, err := sub.offsetRegistry.GetHighWaterMark(sub.Namespace, sub.TopicName, sub.Partition)
+	if err != nil {
+		return fmt.Errorf("failed to get high water mark: %w", err)
+	}
+
+	if offset > hwm {
+		return fmt.Errorf("offset %d is beyond high water mark %d", offset, hwm)
+	}
+
+	sub.CurrentOffset = offset
+	return nil
+}
+
+// GetNextOffset returns the next offset to read
+func (sub *OffsetSubscription) GetNextOffset() int64 {
+	return sub.CurrentOffset
+}
+
+// AdvanceOffset advances the subscription to the next offset
+func (sub *OffsetSubscription) AdvanceOffset() {
+	sub.CurrentOffset++
+}
+
+// GetLag returns the lag between current position and high water mark
+func (sub *OffsetSubscription) GetLag() (int64, error) {
+	if !sub.IsActive {
+		return 0, fmt.Errorf("subscription is not active")
+	}
+
+	hwm, err := sub.offsetRegistry.GetHighWaterMark(sub.Namespace, sub.TopicName, sub.Partition)
+	if err != nil {
+		return 0, fmt.Errorf("failed to get high water mark: %w", err)
+	}
+
+	lag := hwm - sub.CurrentOffset
+	if lag < 0 {
+		lag = 0
+	}
+
+	return lag, nil
+}
+
+// IsAtEnd checks if the subscription has reached the end of available data
+func (sub *OffsetSubscription) IsAtEnd() (bool, error) {
+	if !sub.IsActive {
+		return true, fmt.Errorf("subscription is not active")
+	}
+
+	hwm, err := sub.offsetRegistry.GetHighWaterMark(sub.Namespace, sub.TopicName, sub.Partition)
+	if err != nil {
+		return false, fmt.Errorf("failed to get high water mark: %w", err)
+	}
+
+	return sub.CurrentOffset >= hwm, nil
+}
+
+// OffsetRange represents a range of offsets
+type OffsetRange struct {
+	StartOffset int64
+	EndOffset   int64
+	Count       int64
+}
+
+// GetOffsetRange returns a range of offsets for batch reading
+func (sub *OffsetSubscription) GetOffsetRange(maxCount int64) (*OffsetRange, error) {
+	if !sub.IsActive {
+		return nil, fmt.Errorf("subscription is not active")
+	}
+
+	hwm, err := sub.offsetRegistry.GetHighWaterMark(sub.Namespace, sub.TopicName, sub.Partition)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get high water mark: %w", err)
+	}
+
+	startOffset := sub.CurrentOffset
+	endOffset := startOffset + maxCount - 1
+
+	// Don't go beyond high water mark
+	if endOffset >= hwm {
+		endOffset = hwm - 1
+	}
+
+	// If start is already at or beyond HWM, return empty range
+	if startOffset >= hwm {
+		return &OffsetRange{
+			StartOffset: startOffset,
+			EndOffset:   startOffset - 1, // Empty range
+			Count:       0,
+		}, nil
+	}
+
+	count := endOffset - startOffset + 1
+	return &OffsetRange{
+		StartOffset: startOffset,
+		EndOffset:   endOffset,
+		Count:       count,
+	}, nil
+}
+
+// AdvanceOffsetBy advances the subscription by a specific number of offsets
+func (sub *OffsetSubscription) AdvanceOffsetBy(count int64) {
+	sub.CurrentOffset += count
+}
+
+// OffsetSeeker provides utilities for offset-based seeking
+type OffsetSeeker struct {
+	offsetRegistry *PartitionOffsetRegistry
+}
+
+// NewOffsetSeeker creates a new offset seeker
+func NewOffsetSeeker(offsetRegistry *PartitionOffsetRegistry) *OffsetSeeker {
+	return &OffsetSeeker{
+		offsetRegistry: offsetRegistry,
+	}
+}
+
+// SeekToTimestamp finds the offset closest to a given timestamp
+// This bridges offset-based and timestamp-based seeking
+func (seeker *OffsetSeeker) SeekToTimestamp(partition *schema_pb.Partition, timestamp int64) (int64, error) {
+	// TODO: This requires integration with the storage layer to map timestamps to offsets
+	// For now, return an error indicating this feature needs implementation
+	return 0, fmt.Errorf("timestamp-to-offset mapping not implemented yet")
+}
+
+// ValidateOffsetRange validates that an offset range is valid
+func (seeker *OffsetSeeker) ValidateOffsetRange(namespace, topicName string, partition *schema_pb.Partition, startOffset, endOffset int64) error {
+	if startOffset < 0 {
+		return fmt.Errorf("start offset cannot be negative: %d", startOffset)
+	}
+
+	if endOffset < startOffset {
+		return fmt.Errorf("end offset %d cannot be less than start offset %d", endOffset, startOffset)
+	}
+
+	hwm, err := seeker.offsetRegistry.GetHighWaterMark(namespace, topicName, partition)
+	if err != nil {
+		return fmt.Errorf("failed to get high water mark: %w", err)
+	}
+
+	if startOffset >= hwm {
+		return fmt.Errorf("start offset %d is beyond high water mark %d", startOffset, hwm)
+	}
+
+	if endOffset >= hwm {
+		return fmt.Errorf("end offset %d is beyond high water mark %d", endOffset, hwm)
+	}
+
+	return nil
+}
+
+// GetAvailableOffsetRange returns the range of available offsets for a partition
+func (seeker *OffsetSeeker) GetAvailableOffsetRange(namespace, topicName string, partition *schema_pb.Partition) (*OffsetRange, error) {
+	hwm, err := seeker.offsetRegistry.GetHighWaterMark(namespace, topicName, partition)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get high water mark: %w", err)
+	}
+
+	if hwm == 0 {
+		// No data available
+		return &OffsetRange{
+			StartOffset: 0,
+			EndOffset:   -1,
+			Count:       0,
+		}, nil
+	}
+
+	return &OffsetRange{
+		StartOffset: 0,
+		EndOffset:   hwm - 1,
+		Count:       hwm,
+	}, nil
+}