snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/mq/topic/local_manager.go

@@ -0,0 +1,229 @@
+package topic
+
+import (
+	"context"
+	"time"
+
+	cmap "github.com/orcaman/concurrent-map/v2"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/shirou/gopsutil/v4/cpu"
+)
+
+// LocalTopicManager manages topics on local broker
+type LocalTopicManager struct {
+	topics       cmap.ConcurrentMap[string, *LocalTopic]
+	cleanupDone  chan struct{} // Signal cleanup goroutine to stop
+	cleanupTimer *time.Ticker
+}
+
+// NewLocalTopicManager creates a new LocalTopicManager
+func NewLocalTopicManager() *LocalTopicManager {
+	return &LocalTopicManager{
+		topics:      cmap.New[*LocalTopic](),
+		cleanupDone: make(chan struct{}),
+	}
+}
+
+// StartIdlePartitionCleanup starts a background goroutine that periodically
+// cleans up idle partitions (partitions with no publishers and no subscribers)
+func (manager *LocalTopicManager) StartIdlePartitionCleanup(ctx context.Context, checkInterval, idleTimeout time.Duration) {
+	manager.cleanupTimer = time.NewTicker(checkInterval)
+
+	go func() {
+		defer close(manager.cleanupDone)
+		defer manager.cleanupTimer.Stop()
+
+		glog.V(1).Infof("Idle partition cleanup started: check every %v, cleanup after %v idle", checkInterval, idleTimeout)
+
+		for {
+			select {
+			case <-ctx.Done():
+				glog.V(1).Info("Idle partition cleanup stopped")
+				return
+			case <-manager.cleanupTimer.C:
+				manager.cleanupIdlePartitions(idleTimeout)
+			}
+		}
+	}()
+}
+
+// cleanupIdlePartitions removes idle partitions from memory
+func (manager *LocalTopicManager) cleanupIdlePartitions(idleTimeout time.Duration) {
+	cleanedCount := 0
+	var emptyTopics []string
+
+	// Iterate through all topics
+	manager.topics.IterCb(func(topicKey string, localTopic *LocalTopic) {
+		localTopic.partitionLock.Lock()
+		defer localTopic.partitionLock.Unlock()
+
+		// Check each partition
+		for i := len(localTopic.Partitions) - 1; i >= 0; i-- {
+			partition := localTopic.Partitions[i]
+
+			if partition.ShouldCleanup(idleTimeout) {
+				glog.V(1).Infof("Cleaning up idle partition %s (idle for %v, publishers=%d, subscribers=%d)",
+					partition.Partition.String(),
+					partition.GetIdleDuration(),
+					partition.Publishers.Size(),
+					partition.Subscribers.Size())
+
+				// Shutdown the partition (closes LogBuffer, etc.)
+				partition.Shutdown()
+
+				// Remove from slice
+				localTopic.Partitions = append(localTopic.Partitions[:i], localTopic.Partitions[i+1:]...)
+				cleanedCount++
+			}
+		}
+
+		if len(localTopic.Partitions) == 0 {
+			emptyTopics = append(emptyTopics, topicKey)
+		}
+	})
+
+	// Remove emptied topics only after the iteration: IterCb holds the shard's
+	// read lock while running the callback, so calling Remove (which takes the
+	// same shard's write lock) inside it self-deadlocks and permanently wedges
+	// the shard — every later reader of that shard blocks behind the stuck
+	// writer, hanging ListTopicsInMemory/TopicExistsInMemory forever.
+	for _, topicKey := range emptyTopics {
+		manager.topics.RemoveCb(topicKey, func(key string, localTopic *LocalTopic, exists bool) bool {
+			if !exists || localTopic == nil {
+				return false
+			}
+			// Re-check emptiness under the shard's write lock: a publisher or
+			// subscriber may have added a partition since the scan above.
+			localTopic.partitionLock.RLock()
+			defer localTopic.partitionLock.RUnlock()
+			if len(localTopic.Partitions) > 0 {
+				return false
+			}
+			glog.V(1).Infof("Removing empty topic %s", key)
+			return true
+		})
+	}
+
+	if cleanedCount > 0 {
+		glog.V(0).Infof("Cleaned up %d idle partition(s)", cleanedCount)
+	}
+}
+
+// WaitForCleanupShutdown waits for the cleanup goroutine to finish
+func (manager *LocalTopicManager) WaitForCleanupShutdown() {
+	<-manager.cleanupDone
+	glog.V(1).Info("Idle partition cleanup shutdown complete")
+}
+
+// AddLocalPartition adds a topic to the local topic manager
+func (manager *LocalTopicManager) AddLocalPartition(topic Topic, localPartition *LocalPartition) {
+	localTopic, ok := manager.topics.Get(topic.String())
+	if !ok {
+		localTopic = NewLocalTopic(topic)
+	}
+	if !manager.topics.SetIfAbsent(topic.String(), localTopic) {
+		localTopic, _ = manager.topics.Get(topic.String())
+	}
+	localTopic.addPartition(localPartition)
+}
+
+// GetLocalPartition gets a topic from the local topic manager
+func (manager *LocalTopicManager) GetLocalPartition(topic Topic, partition Partition) *LocalPartition {
+	localTopic, ok := manager.topics.Get(topic.String())
+	if !ok {
+		return nil
+	}
+	result := localTopic.findPartition(partition)
+	return result
+}
+
+// RemoveTopic removes a topic from the local topic manager
+func (manager *LocalTopicManager) RemoveTopic(topic Topic) {
+	manager.topics.Remove(topic.String())
+}
+
+func (manager *LocalTopicManager) RemoveLocalPartition(topic Topic, partition Partition) (removed bool) {
+	localTopic, ok := manager.topics.Get(topic.String())
+	if !ok {
+		return false
+	}
+	return localTopic.removePartition(partition)
+}
+
+func (manager *LocalTopicManager) ClosePublishers(topic Topic, unixTsNs int64) (removed bool) {
+	localTopic, ok := manager.topics.Get(topic.String())
+	if !ok {
+		return false
+	}
+	return localTopic.closePartitionPublishers(unixTsNs)
+}
+
+func (manager *LocalTopicManager) CloseSubscribers(topic Topic, unixTsNs int64) (removed bool) {
+	localTopic, ok := manager.topics.Get(topic.String())
+	if !ok {
+		return false
+	}
+	return localTopic.closePartitionSubscribers(unixTsNs)
+}
+
+// ListTopicsInMemory returns all topics currently tracked in memory
+func (manager *LocalTopicManager) ListTopicsInMemory() []Topic {
+	var topics []Topic
+	for item := range manager.topics.IterBuffered() {
+		topics = append(topics, item.Val.Topic)
+	}
+	return topics
+}
+
+// TopicExistsInMemory checks if a topic exists in memory (not flushed data)
+func (manager *LocalTopicManager) TopicExistsInMemory(topic Topic) bool {
+	_, exists := manager.topics.Get(topic.String())
+	return exists
+}
+
+func (manager *LocalTopicManager) CollectStats(duration time.Duration) *mq_pb.BrokerStats {
+	stats := &mq_pb.BrokerStats{
+		Stats: make(map[string]*mq_pb.TopicPartitionStats),
+	}
+
+	// collect current broker's cpu usage
+	// this needs to be in front, so the following stats can be more accurate
+	usages, err := cpu.Percent(duration, false)
+	if err == nil && len(usages) > 0 {
+		stats.CpuUsagePercent = int32(usages[0])
+	}
+
+	// collect current broker's topics and partitions
+	manager.topics.IterCb(func(topic string, localTopic *LocalTopic) {
+		for _, localPartition := range localTopic.Partitions {
+			topicPartition := &TopicPartition{
+				Topic:     Topic{Namespace: localTopic.Namespace, Name: localTopic.Name},
+				Partition: localPartition.Partition,
+			}
+			stats.Stats[topicPartition.TopicPartitionId()] = &mq_pb.TopicPartitionStats{
+				Topic: &schema_pb.Topic{
+					Namespace: string(localTopic.Namespace),
+					Name:      localTopic.Name,
+				},
+				Partition:       localPartition.Partition.ToPbPartition(),
+				PublisherCount:  int32(localPartition.Publishers.Size()),
+				SubscriberCount: int32(localPartition.Subscribers.Size()),
+				Follower:        localPartition.Follower,
+			}
+			// fmt.Printf("collect topic %+v partition %+v\n", topicPartition, localPartition.Partition)
+		}
+	})
+
+	return stats
+
+}
+
+func (manager *LocalTopicManager) WaitUntilNoPublishers(topic Topic) {
+	localTopic, ok := manager.topics.Get(topic.String())
+	if !ok {
+		return
+	}
+	localTopic.WaitUntilNoPublishers()
+}

weed/mq/topic/local_manager_test.go

@@ -0,0 +1,75 @@
+package topic
+
+import (
+	"testing"
+	"time"
+)
+
+// cleanupIdlePartitions used to call manager.topics.Remove from inside
+// manager.topics.IterCb. IterCb holds the shard's read lock while running the
+// callback, and Remove takes the same shard's write lock, so removing an
+// emptied topic deadlocked the cleanup goroutine and permanently wedged the
+// shard: every later ListTopicsInMemory / TopicExistsInMemory blocked behind
+// the stuck writer. Observed in CI as the Kafka gateway's Metadata requests
+// timing out forever once any topic went idle mid-test.
+func TestCleanupIdlePartitionsRemovesEmptyTopicWithoutDeadlock(t *testing.T) {
+	manager := NewLocalTopicManager()
+
+	tp := NewTopic("test", "idle-topic")
+	localTopic := NewLocalTopic(tp)
+	localPartition := NewLocalPartition(Partition{RingSize: 2520, RangeStart: 0, RangeStop: 2520}, 1, nil, nil)
+	localTopic.Partitions = append(localTopic.Partitions, localPartition)
+	manager.topics.Set(tp.String(), localTopic)
+
+	// Make the partition idle beyond any timeout.
+	localPartition.lastActivityTime.Store(time.Now().Add(-time.Hour).UnixNano())
+
+	done := make(chan struct{})
+	go func() {
+		manager.cleanupIdlePartitions(time.Minute)
+		close(done)
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(10 * time.Second):
+		t.Fatal("cleanupIdlePartitions deadlocked while removing an emptied topic")
+	}
+
+	if manager.topics.Has(tp.String()) {
+		t.Errorf("emptied topic %s should have been removed", tp.String())
+	}
+
+	// The map must still be fully usable afterwards — with the deadlock the
+	// wedged shard made this call block forever.
+	listed := make(chan []Topic, 1)
+	go func() { listed <- manager.ListTopicsInMemory() }()
+	select {
+	case topics := <-listed:
+		if len(topics) != 0 {
+			t.Errorf("expected no topics in memory, got %v", topics)
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatal("ListTopicsInMemory blocked after cleanup — shard still wedged")
+	}
+}
+
+// A topic that gains a partition between the cleanup scan and the removal must
+// be kept: removal re-checks emptiness under the shard write lock.
+func TestCleanupIdlePartitionsKeepsActiveTopic(t *testing.T) {
+	manager := NewLocalTopicManager()
+
+	tp := NewTopic("test", "active-topic")
+	localPartition := NewLocalPartition(Partition{RingSize: 2520, RangeStart: 0, RangeStop: 2520}, 1, nil, nil)
+	manager.AddLocalPartition(tp, localPartition)
+
+	// Fresh activity: nothing should be cleaned up.
+	manager.cleanupIdlePartitions(time.Minute)
+
+	if !manager.topics.Has(tp.String()) {
+		t.Errorf("active topic %s should not have been removed", tp.String())
+	}
+	if manager.GetLocalPartition(tp, localPartition.Partition) == nil {
+		t.Errorf("active partition should still be registered")
+	}
+}

weed/mq/topic/local_partition.go

@@ -0,0 +1,415 @@
+package topic
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+type LocalPartition struct {
+	ListenersWaits int64
+	AckTsNs        int64
+
+	// notifying clients
+	ListenersLock sync.Mutex
+	ListenersCond *sync.Cond
+
+	Partition
+	LogBuffer   *log_buffer.LogBuffer
+	Publishers  *LocalPartitionPublishers
+	Subscribers *LocalPartitionSubscribers
+
+	publishFolloweMeStream mq_pb.SeaweedMessaging_PublishFollowMeClient
+	followerGrpcConnection *grpc.ClientConn
+	Follower               string
+
+	// Track last activity for idle cleanup
+	lastActivityTime atomic.Int64 // Unix nano timestamp
+}
+
+var TIME_FORMAT = "2006-01-02-15-04-05"
+var PartitionGenerationFormat = "v2006-01-02-15-04-05"
+
+func NewLocalPartition(partition Partition, logFlushInterval int, logFlushFn log_buffer.LogFlushFuncType, readFromDiskFn log_buffer.LogReadFromDiskFuncType) *LocalPartition {
+	lp := &LocalPartition{
+		Partition:   partition,
+		Publishers:  NewLocalPartitionPublishers(),
+		Subscribers: NewLocalPartitionSubscribers(),
+	}
+	lp.ListenersCond = sync.NewCond(&lp.ListenersLock)
+	lp.lastActivityTime.Store(time.Now().UnixNano()) // Initialize with current time
+
+	// Ensure a minimum flush interval to prevent busy-loop when set to 0
+	// A flush interval of 0 would cause time.Sleep(0) creating a CPU-consuming busy loop
+	flushInterval := time.Duration(logFlushInterval) * time.Second
+	if flushInterval == 0 {
+		flushInterval = 1 * time.Second // Minimum 1 second to avoid busy-loop, allow near-immediate flushing
+	}
+
+	lp.LogBuffer = log_buffer.NewLogBuffer(fmt.Sprintf("%d/%04d-%04d", partition.UnixTimeNs, partition.RangeStart, partition.RangeStop),
+		flushInterval, logFlushFn, readFromDiskFn, func() {
+			if atomic.LoadInt64(&lp.ListenersWaits) > 0 {
+				lp.ListenersCond.Broadcast()
+			}
+		})
+	return lp
+}
+
+func (p *LocalPartition) Publish(message *mq_pb.DataMessage) error {
+	if err := p.LogBuffer.AddToBuffer(message); err != nil {
+		return fmt.Errorf("failed to add message to log buffer: %w", err)
+	}
+	p.UpdateActivity() // Track publish activity for idle cleanup
+
+	// maybe send to the follower
+	if p.publishFolloweMeStream != nil {
+		// println("recv", string(message.Key), message.TsNs)
+		if followErr := p.publishFolloweMeStream.Send(&mq_pb.PublishFollowMeRequest{
+			Message: &mq_pb.PublishFollowMeRequest_Data{
+				Data: message,
+			},
+		}); followErr != nil {
+			return fmt.Errorf("send to follower %s: %v", p.Follower, followErr)
+		}
+	} else {
+		atomic.StoreInt64(&p.AckTsNs, message.TsNs)
+	}
+
+	return nil
+}
+
+func (p *LocalPartition) Subscribe(clientName string, startPosition log_buffer.MessagePosition,
+	onNoMessageFn func() bool, eachMessageFn log_buffer.EachLogEntryFuncType) error {
+	var processedPosition log_buffer.MessagePosition
+	var readPersistedLogErr error
+	var readInMemoryLogErr error
+	var isDone bool
+
+	p.UpdateActivity() // Track subscribe activity for idle cleanup
+
+	// CRITICAL FIX: Use offset-based functions if startPosition is offset-based
+	// This allows reading historical data by offset, not just by timestamp
+	if startPosition.IsOffsetBased {
+		// Wrap eachMessageFn to match the signature expected by LoopProcessLogDataWithOffset
+		// Also update activity when messages are processed
+		eachMessageWithOffsetFn := func(logEntry *filer_pb.LogEntry, offset int64) (bool, error) {
+			p.UpdateActivity() // Track message read activity
+			return eachMessageFn(logEntry)
+		}
+
+		// Wrap eachMessageFn for disk reads to also update activity
+		eachMessageWithActivityFn := func(logEntry *filer_pb.LogEntry) (bool, error) {
+			p.UpdateActivity() // Track disk read activity for idle cleanup
+			return eachMessageFn(logEntry)
+		}
+
+		// Always attempt initial disk read for historical data
+		// This is fast if no data on disk, and ensures we don't miss old data
+		// The memory read loop below handles new data with instant notifications
+		glog.V(2).Infof("%s reading historical data from disk starting at offset %d", clientName, startPosition.Offset)
+		processedPosition, isDone, readPersistedLogErr = p.LogBuffer.ReadFromDiskFn(startPosition, 0, eachMessageWithActivityFn)
+		if readPersistedLogErr != nil {
+			glog.V(2).Infof("%s read %v persisted log: %v", clientName, p.Partition, readPersistedLogErr)
+			return readPersistedLogErr
+		}
+		if isDone {
+			return nil
+		}
+
+		// Update position after reading from disk
+		if processedPosition.Time.UnixNano() != 0 || processedPosition.IsOffsetBased {
+			startPosition = processedPosition
+		}
+
+		// Step 2: Enter the main loop - read from in-memory buffer, occasionally checking disk
+		for {
+			// Read from in-memory buffer (this is the hot path - handles streaming data)
+			glog.V(4).Infof("SUBSCRIBE: Reading from in-memory buffer for %s at offset %d", clientName, startPosition.Offset)
+			processedPosition, isDone, readInMemoryLogErr = p.LogBuffer.LoopProcessLogDataWithOffset(clientName, startPosition, 0, onNoMessageFn, eachMessageWithOffsetFn)
+
+			if isDone {
+				return nil
+			}
+
+			// Update position
+			// CRITICAL FIX: For offset-based reads, Time is zero, so check Offset instead
+			if processedPosition.Time.UnixNano() != 0 || processedPosition.IsOffsetBased {
+				startPosition = processedPosition
+			}
+
+			// If we get ResumeFromDiskError, it means data was flushed to disk
+			// Read from disk ONCE to catch up, then continue with in-memory buffer
+			if readInMemoryLogErr == log_buffer.ResumeFromDiskError {
+				glog.V(4).Infof("SUBSCRIBE: ResumeFromDiskError - reading flushed data from disk for %s at offset %d", clientName, startPosition.Offset)
+				processedPosition, isDone, readPersistedLogErr = p.LogBuffer.ReadFromDiskFn(startPosition, 0, eachMessageWithActivityFn)
+				if readPersistedLogErr != nil {
+					glog.V(2).Infof("%s read %v persisted log after flush: %v", clientName, p.Partition, readPersistedLogErr)
+					return readPersistedLogErr
+				}
+				if isDone {
+					return nil
+				}
+
+				// Update position and continue the loop (back to in-memory buffer)
+				// CRITICAL FIX: For offset-based reads, Time is zero, so check Offset instead
+				if processedPosition.Time.UnixNano() != 0 || processedPosition.IsOffsetBased {
+					startPosition = processedPosition
+				}
+				// Loop continues - back to reading from in-memory buffer
+				continue
+			}
+
+			// Any other error is a real error
+			if readInMemoryLogErr != nil {
+				glog.V(2).Infof("%s read %v in memory log: %v", clientName, p.Partition, readInMemoryLogErr)
+				return readInMemoryLogErr
+			}
+
+			// If we get here with no error and not done, something is wrong
+			glog.V(1).Infof("SUBSCRIBE: Unexpected state for %s - no error but not done, continuing", clientName)
+		}
+	}
+
+	// Original timestamp-based subscription logic
+	// Wrap eachMessageFn for disk reads to also update activity
+	eachMessageWithActivityFn := func(logEntry *filer_pb.LogEntry) (bool, error) {
+		p.UpdateActivity() // Track disk read activity for idle cleanup
+		return eachMessageFn(logEntry)
+	}
+
+	for {
+		processedPosition, isDone, readPersistedLogErr = p.LogBuffer.ReadFromDiskFn(startPosition, 0, eachMessageWithActivityFn)
+		if readPersistedLogErr != nil {
+			glog.V(0).Infof("%s read %v persisted log: %v", clientName, p.Partition, readPersistedLogErr)
+			return readPersistedLogErr
+		}
+		if isDone {
+			return nil
+		}
+
+		// CRITICAL FIX: For offset-based reads, Time is zero, so check Offset instead
+		if processedPosition.Time.UnixNano() != 0 || processedPosition.IsOffsetBased {
+			startPosition = processedPosition
+		}
+		processedPosition, isDone, readInMemoryLogErr = p.LogBuffer.LoopProcessLogData(clientName, startPosition, 0, onNoMessageFn, eachMessageFn)
+		if isDone {
+			return nil
+		}
+		// CRITICAL FIX: For offset-based reads, Time is zero, so check Offset instead
+		if processedPosition.Time.UnixNano() != 0 || processedPosition.IsOffsetBased {
+			startPosition = processedPosition
+		}
+
+		if readInMemoryLogErr == log_buffer.ResumeFromDiskError {
+			continue
+		}
+		if readInMemoryLogErr != nil {
+			glog.V(0).Infof("%s read %v in memory log: %v", clientName, p.Partition, readInMemoryLogErr)
+			return readInMemoryLogErr
+		}
+	}
+}
+
+func (p *LocalPartition) GetEarliestMessageTimeInMemory() time.Time {
+	return p.LogBuffer.GetEarliestTime()
+}
+
+func (p *LocalPartition) HasData() bool {
+	return !p.LogBuffer.GetEarliestTime().IsZero()
+}
+
+func (p *LocalPartition) GetEarliestInMemoryMessagePosition() log_buffer.MessagePosition {
+	return p.LogBuffer.GetEarliestPosition()
+}
+
+func (p *LocalPartition) closePublishers() {
+	p.Publishers.SignalShutdown()
+}
+func (p *LocalPartition) closeSubscribers() {
+	p.Subscribers.SignalShutdown()
+}
+
+func (p *LocalPartition) WaitUntilNoPublishers() {
+	for {
+		if p.Publishers.Size() == 0 {
+			return
+		}
+		time.Sleep(113 * time.Millisecond)
+	}
+}
+
+func (p *LocalPartition) MaybeConnectToFollowers(initMessage *mq_pb.PublishMessageRequest_InitMessage, grpcDialOption grpc.DialOption) (err error) {
+	if p.publishFolloweMeStream != nil {
+		return nil
+	}
+	if initMessage.FollowerBroker == "" {
+		return nil
+	}
+
+	p.Follower = initMessage.FollowerBroker
+	ctx := context.Background()
+	p.followerGrpcConnection, err = pb.GrpcDial(ctx, p.Follower, true, grpcDialOption)
+	if err != nil {
+		return fmt.Errorf("fail to dial %s: %v", p.Follower, err)
+	}
+	followerClient := mq_pb.NewSeaweedMessagingClient(p.followerGrpcConnection)
+	p.publishFolloweMeStream, err = followerClient.PublishFollowMe(ctx)
+	if err != nil {
+		return fmt.Errorf("fail to create publish client: %w", err)
+	}
+	if err = p.publishFolloweMeStream.Send(&mq_pb.PublishFollowMeRequest{
+		Message: &mq_pb.PublishFollowMeRequest_Init{
+			Init: &mq_pb.PublishFollowMeRequest_InitMessage{
+				Topic:     initMessage.Topic,
+				Partition: initMessage.Partition,
+			},
+		},
+	}); err != nil {
+		return err
+	}
+
+	// start receiving ack from follower
+	go func() {
+		defer func() {
+			// println("stop receiving ack from follower")
+		}()
+
+		for {
+			ack, err := p.publishFolloweMeStream.Recv()
+			if err != nil {
+				e, _ := status.FromError(err)
+				if e.Code() == codes.Canceled {
+					glog.V(0).Infof("local partition %v follower %v stopped", p.Partition, p.Follower)
+					return
+				}
+				glog.Errorf("Receiving local partition %v  follower %s ack: %v", p.Partition, p.Follower, err)
+				return
+			}
+			atomic.StoreInt64(&p.AckTsNs, ack.AckTsNs)
+			// println("recv ack", ack.AckTsNs)
+		}
+	}()
+	return nil
+}
+
+func (p *LocalPartition) MaybeShutdownLocalPartition() (hasShutdown bool) {
+
+	if p.Publishers.Size() == 0 && p.Subscribers.Size() == 0 {
+		p.LogBuffer.ShutdownLogBuffer()
+		for !p.LogBuffer.IsAllFlushed() {
+			time.Sleep(113 * time.Millisecond)
+		}
+		if p.publishFolloweMeStream != nil {
+			// send close to the follower
+			if followErr := p.publishFolloweMeStream.Send(&mq_pb.PublishFollowMeRequest{
+				Message: &mq_pb.PublishFollowMeRequest_Close{
+					Close: &mq_pb.PublishFollowMeRequest_CloseMessage{},
+				},
+			}); followErr != nil {
+				glog.Errorf("Error closing follower stream: %v", followErr)
+			}
+			glog.V(4).Infof("closing grpcConnection to follower")
+			p.followerGrpcConnection.Close()
+			p.publishFolloweMeStream = nil
+			p.Follower = ""
+		}
+
+		hasShutdown = true
+	}
+
+	glog.V(0).Infof("local partition %v Publisher:%d Subscriber:%d follower:%s shutdown %v", p.Partition, p.Publishers.Size(), p.Subscribers.Size(), p.Follower, hasShutdown)
+	return
+}
+
+// MaybeShutdownLocalPartitionForTopic is a topic-aware version that considers system topic retention
+func (p *LocalPartition) MaybeShutdownLocalPartitionForTopic(topicName string) (hasShutdown bool) {
+	// For system topics like _schemas, be more conservative about shutdown
+	if isSystemTopic(topicName) {
+		glog.V(0).Infof("System topic %s - skipping aggressive shutdown for partition %v (Publishers:%d Subscribers:%d)",
+			topicName, p.Partition, p.Publishers.Size(), p.Subscribers.Size())
+		return false
+	}
+
+	// For regular topics, use the standard shutdown logic
+	return p.MaybeShutdownLocalPartition()
+}
+
+// isSystemTopic checks if a topic should have special retention behavior
+func isSystemTopic(topicName string) bool {
+	systemTopics := []string{
+		"_schemas",            // Schema Registry topic
+		"__consumer_offsets",  // Kafka consumer offsets topic
+		"__transaction_state", // Kafka transaction state topic
+	}
+
+	for _, systemTopic := range systemTopics {
+		if topicName == systemTopic {
+			return true
+		}
+	}
+
+	// Also check for topics with system prefixes
+	return strings.HasPrefix(topicName, "_") || strings.HasPrefix(topicName, "__")
+}
+
+func (p *LocalPartition) Shutdown() {
+	p.closePublishers()
+	p.closeSubscribers()
+	p.LogBuffer.ShutdownLogBuffer()
+	glog.V(0).Infof("local partition %v shutting down", p.Partition)
+}
+
+func (p *LocalPartition) NotifyLogFlushed(flushTsNs int64) {
+	if p.publishFolloweMeStream != nil {
+		if followErr := p.publishFolloweMeStream.Send(&mq_pb.PublishFollowMeRequest{
+			Message: &mq_pb.PublishFollowMeRequest_Flush{
+				Flush: &mq_pb.PublishFollowMeRequest_FlushMessage{
+					TsNs: flushTsNs,
+				},
+			},
+		}); followErr != nil {
+			glog.Errorf("send follower %s flush message: %v", p.Follower, followErr)
+		}
+		// println("notifying", p.Follower, "flushed at", flushTsNs)
+	}
+}
+
+// UpdateActivity updates the last activity timestamp for this partition
+// Should be called whenever a publisher publishes or a subscriber reads
+func (p *LocalPartition) UpdateActivity() {
+	p.lastActivityTime.Store(time.Now().UnixNano())
+}
+
+// IsIdle returns true if the partition has no publishers and no subscribers
+func (p *LocalPartition) IsIdle() bool {
+	return p.Publishers.Size() == 0 && p.Subscribers.Size() == 0
+}
+
+// GetIdleDuration returns how long the partition has been idle
+func (p *LocalPartition) GetIdleDuration() time.Duration {
+	lastActivity := p.lastActivityTime.Load()
+	return time.Since(time.Unix(0, lastActivity))
+}
+
+// ShouldCleanup returns true if the partition should be cleaned up
+// A partition should be cleaned up if:
+// 1. It has no publishers and no subscribers
+// 2. It has been idle for longer than the idle timeout
+func (p *LocalPartition) ShouldCleanup(idleTimeout time.Duration) bool {
+	if !p.IsIdle() {
+		return false
+	}
+	return p.GetIdleDuration() > idleTimeout
+}

weed/mq/topic/local_partition_offset.go

@@ -0,0 +1,92 @@
+package topic
+
+import (
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// OffsetAssignmentFunc is a function type for assigning offsets to messages
+type OffsetAssignmentFunc func() (int64, error)
+
+// PublishWithOffset publishes a message with offset assignment
+// This method is used by the Kafka gateway integration for sequential offset assignment
+func (p *LocalPartition) PublishWithOffset(message *mq_pb.DataMessage, assignOffsetFn OffsetAssignmentFunc) (int64, error) {
+	// Assign offset for this message
+	offset, err := assignOffsetFn()
+	if err != nil {
+		return 0, fmt.Errorf("failed to assign offset: %w", err)
+	}
+
+	// Add message to buffer with offset
+	err = p.addToBufferWithOffset(message, offset)
+	if err != nil {
+		return 0, fmt.Errorf("failed to add message to buffer: %w", err)
+	}
+
+	// Track publish activity for idle cleanup (consistent with Publish method)
+	p.UpdateActivity()
+
+	// Send to follower if needed (same logic as original Publish)
+	if p.publishFolloweMeStream != nil {
+		if followErr := p.publishFolloweMeStream.Send(&mq_pb.PublishFollowMeRequest{
+			Message: &mq_pb.PublishFollowMeRequest_Data{
+				Data: message,
+			},
+		}); followErr != nil {
+			return 0, fmt.Errorf("send to follower %s: %v", p.Follower, followErr)
+		}
+	} else {
+		atomic.StoreInt64(&p.AckTsNs, message.TsNs)
+	}
+
+	return offset, nil
+}
+
+// addToBufferWithOffset adds a message to the log buffer with a pre-assigned offset
+func (p *LocalPartition) addToBufferWithOffset(message *mq_pb.DataMessage, offset int64) error {
+	// Ensure we have a timestamp
+	processingTsNs := message.TsNs
+	if processingTsNs == 0 {
+		processingTsNs = time.Now().UnixNano()
+	}
+
+	// Build a LogEntry that preserves the assigned sequential offset
+	logEntry := &filer_pb.LogEntry{
+		TsNs:             processingTsNs,
+		PartitionKeyHash: util.HashToInt32(message.Key),
+		Data:             message.Value,
+		Key:              message.Key,
+		Offset:           offset,
+	}
+
+	// Add the entry to the buffer in a way that preserves offset on disk and in-memory
+	if err := p.LogBuffer.AddLogEntryToBuffer(logEntry); err != nil {
+		return fmt.Errorf("failed to add log entry to buffer: %w", err)
+	}
+
+	return nil
+}
+
+// GetOffsetInfo returns offset information for this partition
+// Used for debugging and monitoring partition offset state
+func (p *LocalPartition) GetOffsetInfo() map[string]interface{} {
+	return map[string]interface{}{
+		"partition_ring_size":   p.RingSize,
+		"partition_range_start": p.RangeStart,
+		"partition_range_stop":  p.RangeStop,
+		"partition_unix_time":   p.UnixTimeNs,
+		"buffer_name":           p.LogBuffer.GetName(),
+		"buffer_offset":         p.LogBuffer.GetOffset(),
+	}
+}
+
+// OffsetAwarePublisher wraps a LocalPartition with offset assignment capability
+type OffsetAwarePublisher struct {
+	partition      *LocalPartition
+	assignOffsetFn OffsetAssignmentFunc
+}

weed/mq/topic/local_partition_publishers.go

@@ -0,0 +1,115 @@
+package topic
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+type LocalPartitionPublishers struct {
+	publishers     map[string]*LocalPublisher
+	publishersLock sync.RWMutex
+}
+type LocalPublisher struct {
+	connectTimeNs       int64 // accessed atomically
+	lastSeenTimeNs      int64 // accessed atomically
+	lastPublishedOffset int64 // accessed atomically - offset of last message published
+	lastAckedOffset     int64 // accessed atomically - offset of last message acknowledged by broker
+}
+
+func NewLocalPublisher() *LocalPublisher {
+	now := time.Now().UnixNano()
+	publisher := &LocalPublisher{}
+	atomic.StoreInt64(&publisher.connectTimeNs, now)
+	atomic.StoreInt64(&publisher.lastSeenTimeNs, now)
+	atomic.StoreInt64(&publisher.lastPublishedOffset, 0)
+	atomic.StoreInt64(&publisher.lastAckedOffset, 0)
+	return publisher
+}
+func (p *LocalPublisher) SignalShutdown() {
+}
+
+// UpdateLastSeen updates the last activity time for this publisher
+func (p *LocalPublisher) UpdateLastSeen() {
+	atomic.StoreInt64(&p.lastSeenTimeNs, time.Now().UnixNano())
+}
+
+// UpdatePublishedOffset updates the offset of the last message published by this publisher
+func (p *LocalPublisher) UpdatePublishedOffset(offset int64) {
+	atomic.StoreInt64(&p.lastPublishedOffset, offset)
+	atomic.StoreInt64(&p.lastSeenTimeNs, time.Now().UnixNano())
+}
+
+// UpdateAckedOffset updates the offset of the last message acknowledged by the broker for this publisher
+func (p *LocalPublisher) UpdateAckedOffset(offset int64) {
+	atomic.StoreInt64(&p.lastAckedOffset, offset)
+	atomic.StoreInt64(&p.lastSeenTimeNs, time.Now().UnixNano())
+}
+
+// GetTimestamps returns the connect and last seen timestamps safely
+func (p *LocalPublisher) GetTimestamps() (connectTimeNs, lastSeenTimeNs int64) {
+	return atomic.LoadInt64(&p.connectTimeNs), atomic.LoadInt64(&p.lastSeenTimeNs)
+}
+
+// GetOffsets returns the published and acknowledged offsets safely
+func (p *LocalPublisher) GetOffsets() (lastPublishedOffset, lastAckedOffset int64) {
+	return atomic.LoadInt64(&p.lastPublishedOffset), atomic.LoadInt64(&p.lastAckedOffset)
+}
+
+func NewLocalPartitionPublishers() *LocalPartitionPublishers {
+	return &LocalPartitionPublishers{
+		publishers: make(map[string]*LocalPublisher),
+	}
+}
+
+func (p *LocalPartitionPublishers) AddPublisher(clientName string, publisher *LocalPublisher) {
+	p.publishersLock.Lock()
+	defer p.publishersLock.Unlock()
+
+	p.publishers[clientName] = publisher
+}
+
+func (p *LocalPartitionPublishers) RemovePublisher(clientName string) {
+	p.publishersLock.Lock()
+	defer p.publishersLock.Unlock()
+
+	delete(p.publishers, clientName)
+}
+
+func (p *LocalPartitionPublishers) SignalShutdown() {
+	p.publishersLock.RLock()
+	defer p.publishersLock.RUnlock()
+
+	for _, publisher := range p.publishers {
+		publisher.SignalShutdown()
+	}
+}
+
+func (p *LocalPartitionPublishers) Size() int {
+	p.publishersLock.RLock()
+	defer p.publishersLock.RUnlock()
+
+	return len(p.publishers)
+}
+
+// GetPublisherNames returns the names of all publishers
+func (p *LocalPartitionPublishers) GetPublisherNames() []string {
+	p.publishersLock.RLock()
+	defer p.publishersLock.RUnlock()
+
+	names := make([]string, 0, len(p.publishers))
+	for name := range p.publishers {
+		names = append(names, name)
+	}
+	return names
+}
+
+// ForEachPublisher iterates over all publishers
+func (p *LocalPartitionPublishers) ForEachPublisher(fn func(name string, publisher *LocalPublisher)) {
+	p.publishersLock.RLock()
+	defer p.publishersLock.RUnlock()
+
+	for name, publisher := range p.publishers {
+		fn(name, publisher)
+	}
+}

weed/mq/topic/local_partition_subscribe_test.go

@@ -0,0 +1,566 @@
+package topic
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+)
+
+// MockLogBuffer provides a controllable log buffer for testing
+type MockLogBuffer struct {
+	// In-memory data
+	memoryEntries     []*filer_pb.LogEntry
+	memoryStartTime   time.Time
+	memoryStopTime    time.Time
+	memoryStartOffset int64
+	memoryStopOffset  int64
+
+	// Disk data
+	diskEntries     []*filer_pb.LogEntry
+	diskStartTime   time.Time
+	diskStopTime    time.Time
+	diskStartOffset int64
+	diskStopOffset  int64
+
+	// Behavior control
+	diskReadDelay   time.Duration
+	memoryReadDelay time.Duration
+	diskReadError   error
+	memoryReadError error
+}
+
+// MockReadFromDiskFn simulates reading from disk
+func (m *MockLogBuffer) MockReadFromDiskFn(startPosition log_buffer.MessagePosition, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (log_buffer.MessagePosition, bool, error) {
+	if m.diskReadDelay > 0 {
+		time.Sleep(m.diskReadDelay)
+	}
+
+	if m.diskReadError != nil {
+		return startPosition, false, m.diskReadError
+	}
+
+	isOffsetBased := startPosition.IsOffsetBased
+	lastPosition := startPosition
+	isDone := false
+
+	for _, entry := range m.diskEntries {
+		// Filter based on mode
+		if isOffsetBased {
+			if entry.Offset < startPosition.Offset {
+				continue
+			}
+		} else {
+			entryTime := time.Unix(0, entry.TsNs)
+			if entryTime.Before(startPosition.Time) {
+				continue
+			}
+		}
+
+		// Apply stopTsNs filter
+		if stopTsNs > 0 && entry.TsNs > stopTsNs {
+			isDone = true
+			break
+		}
+
+		// Call handler
+		done, err := eachLogEntryFn(entry)
+		if err != nil {
+			return lastPosition, false, err
+		}
+		if done {
+			isDone = true
+			break
+		}
+
+		// Update position
+		if isOffsetBased {
+			lastPosition = log_buffer.NewMessagePosition(entry.TsNs, entry.Offset+1)
+		} else {
+			lastPosition = log_buffer.NewMessagePosition(entry.TsNs, entry.Offset)
+		}
+	}
+
+	return lastPosition, isDone, nil
+}
+
+// MockLoopProcessLogDataWithOffset simulates reading from memory with offset
+func (m *MockLogBuffer) MockLoopProcessLogDataWithOffset(readerName string, startPosition log_buffer.MessagePosition, stopTsNs int64, waitForDataFn func() bool, eachLogDataFn log_buffer.EachLogEntryWithOffsetFuncType) (log_buffer.MessagePosition, bool, error) {
+	if m.memoryReadDelay > 0 {
+		time.Sleep(m.memoryReadDelay)
+	}
+
+	if m.memoryReadError != nil {
+		return startPosition, false, m.memoryReadError
+	}
+
+	lastPosition := startPosition
+	isDone := false
+
+	// Check if requested offset is in memory
+	if startPosition.Offset < m.memoryStartOffset {
+		// Data is on disk
+		return startPosition, false, log_buffer.ResumeFromDiskError
+	}
+
+	for _, entry := range m.memoryEntries {
+		// Filter by offset
+		if entry.Offset < startPosition.Offset {
+			continue
+		}
+
+		// Apply stopTsNs filter
+		if stopTsNs > 0 && entry.TsNs > stopTsNs {
+			isDone = true
+			break
+		}
+
+		// Call handler
+		done, err := eachLogDataFn(entry, entry.Offset)
+		if err != nil {
+			return lastPosition, false, err
+		}
+		if done {
+			isDone = true
+			break
+		}
+
+		// Update position
+		lastPosition = log_buffer.NewMessagePosition(entry.TsNs, entry.Offset+1)
+	}
+
+	return lastPosition, isDone, nil
+}
+
+// Helper to create test entries
+func createTestEntry(offset int64, timestamp time.Time, key, value string) *filer_pb.LogEntry {
+	return &filer_pb.LogEntry{
+		TsNs:   timestamp.UnixNano(),
+		Offset: offset,
+		Key:    []byte(key),
+		Data:   []byte(value),
+	}
+}
+
+// TestOffsetBasedSubscribe_AllDataInMemory tests reading when all data is in memory
+func TestOffsetBasedSubscribe_AllDataInMemory(t *testing.T) {
+	baseTime := time.Now()
+
+	mock := &MockLogBuffer{
+		memoryEntries: []*filer_pb.LogEntry{
+			createTestEntry(0, baseTime, "key0", "value0"),
+			createTestEntry(1, baseTime.Add(1*time.Second), "key1", "value1"),
+			createTestEntry(2, baseTime.Add(2*time.Second), "key2", "value2"),
+			createTestEntry(3, baseTime.Add(3*time.Second), "key3", "value3"),
+		},
+		memoryStartOffset: 0,
+		memoryStopOffset:  3,
+		diskEntries:       []*filer_pb.LogEntry{}, // No disk data
+	}
+
+	// Test reading from offset 0
+	t.Run("ReadFromOffset0", func(t *testing.T) {
+		var receivedOffsets []int64
+		startPos := log_buffer.NewMessagePositionFromOffset(0)
+
+		eachLogFn := func(entry *filer_pb.LogEntry) (bool, error) {
+			receivedOffsets = append(receivedOffsets, entry.Offset)
+			return false, nil
+		}
+
+		// Simulate the Subscribe logic
+		// 1. Try disk read first
+		pos, done, err := mock.MockReadFromDiskFn(startPos, 0, eachLogFn)
+		if err != nil {
+			t.Fatalf("Disk read failed: %v", err)
+		}
+		if done {
+			t.Fatal("Should not be done after disk read")
+		}
+
+		// 2. Read from memory
+		eachLogWithOffsetFn := func(entry *filer_pb.LogEntry, offset int64) (bool, error) {
+			return eachLogFn(entry)
+		}
+
+		_, _, err = mock.MockLoopProcessLogDataWithOffset("test", pos, 0, func() bool { return true }, eachLogWithOffsetFn)
+		if err != nil && err != log_buffer.ResumeFromDiskError {
+			t.Fatalf("Memory read failed: %v", err)
+		}
+
+		// Verify we got all offsets in order
+		expected := []int64{0, 1, 2, 3}
+		if len(receivedOffsets) != len(expected) {
+			t.Errorf("Expected %d offsets, got %d", len(expected), len(receivedOffsets))
+		}
+		for i, offset := range receivedOffsets {
+			if offset != expected[i] {
+				t.Errorf("Offset[%d]: expected %d, got %d", i, expected[i], offset)
+			}
+		}
+	})
+
+	// Test reading from offset 2
+	t.Run("ReadFromOffset2", func(t *testing.T) {
+		var receivedOffsets []int64
+		startPos := log_buffer.NewMessagePositionFromOffset(2)
+
+		eachLogFn := func(entry *filer_pb.LogEntry) (bool, error) {
+			receivedOffsets = append(receivedOffsets, entry.Offset)
+			return false, nil
+		}
+
+		eachLogWithOffsetFn := func(entry *filer_pb.LogEntry, offset int64) (bool, error) {
+			return eachLogFn(entry)
+		}
+
+		// Should skip disk and go straight to memory
+		pos, _, err := mock.MockReadFromDiskFn(startPos, 0, eachLogFn)
+		if err != nil {
+			t.Fatalf("Disk read failed: %v", err)
+		}
+
+		_, _, err = mock.MockLoopProcessLogDataWithOffset("test", pos, 0, func() bool { return true }, eachLogWithOffsetFn)
+		if err != nil && err != log_buffer.ResumeFromDiskError {
+			t.Fatalf("Memory read failed: %v", err)
+		}
+
+		// Verify we got offsets 2, 3
+		expected := []int64{2, 3}
+		if len(receivedOffsets) != len(expected) {
+			t.Errorf("Expected %d offsets, got %d", len(expected), len(receivedOffsets))
+		}
+		for i, offset := range receivedOffsets {
+			if offset != expected[i] {
+				t.Errorf("Offset[%d]: expected %d, got %d", i, expected[i], offset)
+			}
+		}
+	})
+}
+
+// TestOffsetBasedSubscribe_DataOnDisk tests reading when data is on disk
+func TestOffsetBasedSubscribe_DataOnDisk(t *testing.T) {
+	baseTime := time.Now()
+
+	mock := &MockLogBuffer{
+		// Offsets 0-9 on disk
+		diskEntries: []*filer_pb.LogEntry{
+			createTestEntry(0, baseTime, "key0", "value0"),
+			createTestEntry(1, baseTime.Add(1*time.Second), "key1", "value1"),
+			createTestEntry(2, baseTime.Add(2*time.Second), "key2", "value2"),
+			createTestEntry(3, baseTime.Add(3*time.Second), "key3", "value3"),
+			createTestEntry(4, baseTime.Add(4*time.Second), "key4", "value4"),
+			createTestEntry(5, baseTime.Add(5*time.Second), "key5", "value5"),
+			createTestEntry(6, baseTime.Add(6*time.Second), "key6", "value6"),
+			createTestEntry(7, baseTime.Add(7*time.Second), "key7", "value7"),
+			createTestEntry(8, baseTime.Add(8*time.Second), "key8", "value8"),
+			createTestEntry(9, baseTime.Add(9*time.Second), "key9", "value9"),
+		},
+		diskStartOffset: 0,
+		diskStopOffset:  9,
+		// Offsets 10-12 in memory
+		memoryEntries: []*filer_pb.LogEntry{
+			createTestEntry(10, baseTime.Add(10*time.Second), "key10", "value10"),
+			createTestEntry(11, baseTime.Add(11*time.Second), "key11", "value11"),
+			createTestEntry(12, baseTime.Add(12*time.Second), "key12", "value12"),
+		},
+		memoryStartOffset: 10,
+		memoryStopOffset:  12,
+	}
+
+	// Test reading from offset 0 (on disk)
+	t.Run("ReadFromOffset0_OnDisk", func(t *testing.T) {
+		var receivedOffsets []int64
+		startPos := log_buffer.NewMessagePositionFromOffset(0)
+
+		eachLogFn := func(entry *filer_pb.LogEntry) (bool, error) {
+			receivedOffsets = append(receivedOffsets, entry.Offset)
+			return false, nil
+		}
+
+		eachLogWithOffsetFn := func(entry *filer_pb.LogEntry, offset int64) (bool, error) {
+			return eachLogFn(entry)
+		}
+
+		// 1. Read from disk (should get 0-9)
+		pos, done, err := mock.MockReadFromDiskFn(startPos, 0, eachLogFn)
+		if err != nil {
+			t.Fatalf("Disk read failed: %v", err)
+		}
+		if done {
+			t.Fatal("Should not be done after disk read")
+		}
+
+		// 2. Read from memory (should get 10-12)
+		_, _, err = mock.MockLoopProcessLogDataWithOffset("test", pos, 0, func() bool { return true }, eachLogWithOffsetFn)
+		if err != nil && err != log_buffer.ResumeFromDiskError {
+			t.Fatalf("Memory read failed: %v", err)
+		}
+
+		// Verify we got all offsets 0-12 in order
+		expected := []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}
+		if len(receivedOffsets) != len(expected) {
+			t.Errorf("Expected %d offsets, got %d: %v", len(expected), len(receivedOffsets), receivedOffsets)
+		}
+		for i, offset := range receivedOffsets {
+			if i < len(expected) && offset != expected[i] {
+				t.Errorf("Offset[%d]: expected %d, got %d", i, expected[i], offset)
+			}
+		}
+	})
+
+	// Test reading from offset 5 (on disk, middle)
+	t.Run("ReadFromOffset5_OnDisk", func(t *testing.T) {
+		var receivedOffsets []int64
+		startPos := log_buffer.NewMessagePositionFromOffset(5)
+
+		eachLogFn := func(entry *filer_pb.LogEntry) (bool, error) {
+			receivedOffsets = append(receivedOffsets, entry.Offset)
+			return false, nil
+		}
+
+		eachLogWithOffsetFn := func(entry *filer_pb.LogEntry, offset int64) (bool, error) {
+			return eachLogFn(entry)
+		}
+
+		// 1. Read from disk (should get 5-9)
+		pos, _, err := mock.MockReadFromDiskFn(startPos, 0, eachLogFn)
+		if err != nil {
+			t.Fatalf("Disk read failed: %v", err)
+		}
+
+		// 2. Read from memory (should get 10-12)
+		_, _, err = mock.MockLoopProcessLogDataWithOffset("test", pos, 0, func() bool { return true }, eachLogWithOffsetFn)
+		if err != nil && err != log_buffer.ResumeFromDiskError {
+			t.Fatalf("Memory read failed: %v", err)
+		}
+
+		// Verify we got offsets 5-12
+		expected := []int64{5, 6, 7, 8, 9, 10, 11, 12}
+		if len(receivedOffsets) != len(expected) {
+			t.Errorf("Expected %d offsets, got %d: %v", len(expected), len(receivedOffsets), receivedOffsets)
+		}
+		for i, offset := range receivedOffsets {
+			if i < len(expected) && offset != expected[i] {
+				t.Errorf("Offset[%d]: expected %d, got %d", i, expected[i], offset)
+			}
+		}
+	})
+
+	// Test reading from offset 11 (in memory)
+	t.Run("ReadFromOffset11_InMemory", func(t *testing.T) {
+		var receivedOffsets []int64
+		startPos := log_buffer.NewMessagePositionFromOffset(11)
+
+		eachLogFn := func(entry *filer_pb.LogEntry) (bool, error) {
+			receivedOffsets = append(receivedOffsets, entry.Offset)
+			return false, nil
+		}
+
+		eachLogWithOffsetFn := func(entry *filer_pb.LogEntry, offset int64) (bool, error) {
+			return eachLogFn(entry)
+		}
+
+		// 1. Try disk read (should get nothing)
+		pos, _, err := mock.MockReadFromDiskFn(startPos, 0, eachLogFn)
+		if err != nil {
+			t.Fatalf("Disk read failed: %v", err)
+		}
+
+		// 2. Read from memory (should get 11-12)
+		_, _, err = mock.MockLoopProcessLogDataWithOffset("test", pos, 0, func() bool { return true }, eachLogWithOffsetFn)
+		if err != nil && err != log_buffer.ResumeFromDiskError {
+			t.Fatalf("Memory read failed: %v", err)
+		}
+
+		// Verify we got offsets 11-12
+		expected := []int64{11, 12}
+		if len(receivedOffsets) != len(expected) {
+			t.Errorf("Expected %d offsets, got %d: %v", len(expected), len(receivedOffsets), receivedOffsets)
+		}
+		for i, offset := range receivedOffsets {
+			if i < len(expected) && offset != expected[i] {
+				t.Errorf("Offset[%d]: expected %d, got %d", i, expected[i], offset)
+			}
+		}
+	})
+}
+
+// TestTimestampBasedSubscribe tests timestamp-based reading
+func TestTimestampBasedSubscribe(t *testing.T) {
+	baseTime := time.Now()
+
+	mock := &MockLogBuffer{
+		diskEntries: []*filer_pb.LogEntry{
+			createTestEntry(0, baseTime, "key0", "value0"),
+			createTestEntry(1, baseTime.Add(10*time.Second), "key1", "value1"),
+			createTestEntry(2, baseTime.Add(20*time.Second), "key2", "value2"),
+		},
+		memoryEntries: []*filer_pb.LogEntry{
+			createTestEntry(3, baseTime.Add(30*time.Second), "key3", "value3"),
+			createTestEntry(4, baseTime.Add(40*time.Second), "key4", "value4"),
+		},
+	}
+
+	// Test reading from beginning
+	t.Run("ReadFromBeginning", func(t *testing.T) {
+		var receivedOffsets []int64
+		startPos := log_buffer.NewMessagePosition(baseTime.UnixNano(), -1) // Timestamp-based
+
+		eachLogFn := func(entry *filer_pb.LogEntry) (bool, error) {
+			receivedOffsets = append(receivedOffsets, entry.Offset)
+			return false, nil
+		}
+
+		// Read from disk
+		_, _, err := mock.MockReadFromDiskFn(startPos, 0, eachLogFn)
+		if err != nil {
+			t.Fatalf("Disk read failed: %v", err)
+		}
+
+		// In real scenario, would then read from memory using LoopProcessLogData
+		// For this test, just verify disk gave us 0-2
+		expected := []int64{0, 1, 2}
+		if len(receivedOffsets) != len(expected) {
+			t.Errorf("Expected %d offsets, got %d", len(expected), len(receivedOffsets))
+		}
+	})
+
+	// Test reading from middle timestamp
+	t.Run("ReadFromMiddleTimestamp", func(t *testing.T) {
+		var receivedOffsets []int64
+		startPos := log_buffer.NewMessagePosition(baseTime.Add(15*time.Second).UnixNano(), -1)
+
+		eachLogFn := func(entry *filer_pb.LogEntry) (bool, error) {
+			receivedOffsets = append(receivedOffsets, entry.Offset)
+			return false, nil
+		}
+
+		// Read from disk
+		_, _, err := mock.MockReadFromDiskFn(startPos, 0, eachLogFn)
+		if err != nil {
+			t.Fatalf("Disk read failed: %v", err)
+		}
+
+		// Should get offset 2 only (timestamp at 20s >= 15s, offset 1 at 10s is excluded)
+		expected := []int64{2}
+		if len(receivedOffsets) != len(expected) {
+			t.Errorf("Expected %d offsets, got %d: %v", len(expected), len(receivedOffsets), receivedOffsets)
+		}
+	})
+}
+
+// TestConcurrentSubscribers tests multiple concurrent subscribers
+func TestConcurrentSubscribers(t *testing.T) {
+	baseTime := time.Now()
+
+	mock := &MockLogBuffer{
+		diskEntries: []*filer_pb.LogEntry{
+			createTestEntry(0, baseTime, "key0", "value0"),
+			createTestEntry(1, baseTime.Add(1*time.Second), "key1", "value1"),
+			createTestEntry(2, baseTime.Add(2*time.Second), "key2", "value2"),
+		},
+		memoryEntries: []*filer_pb.LogEntry{
+			createTestEntry(3, baseTime.Add(3*time.Second), "key3", "value3"),
+			createTestEntry(4, baseTime.Add(4*time.Second), "key4", "value4"),
+		},
+		memoryStartOffset: 3,
+		memoryStopOffset:  4,
+	}
+
+	var wg sync.WaitGroup
+	results := make(map[string][]int64)
+	var mu sync.Mutex
+
+	// Spawn 3 concurrent subscribers
+	for i := 0; i < 3; i++ {
+		wg.Add(1)
+		subscriberName := fmt.Sprintf("subscriber-%d", i)
+
+		go func(name string) {
+			defer wg.Done()
+
+			var receivedOffsets []int64
+			startPos := log_buffer.NewMessagePositionFromOffset(0)
+
+			eachLogFn := func(entry *filer_pb.LogEntry) (bool, error) {
+				receivedOffsets = append(receivedOffsets, entry.Offset)
+				return false, nil
+			}
+
+			eachLogWithOffsetFn := func(entry *filer_pb.LogEntry, offset int64) (bool, error) {
+				return eachLogFn(entry)
+			}
+
+			// Read from disk
+			pos, _, _ := mock.MockReadFromDiskFn(startPos, 0, eachLogFn)
+
+			// Read from memory
+			mock.MockLoopProcessLogDataWithOffset(name, pos, 0, func() bool { return true }, eachLogWithOffsetFn)
+
+			mu.Lock()
+			results[name] = receivedOffsets
+			mu.Unlock()
+		}(subscriberName)
+	}
+
+	wg.Wait()
+
+	// Verify all subscribers got the same data
+	expected := []int64{0, 1, 2, 3, 4}
+	for name, offsets := range results {
+		if len(offsets) != len(expected) {
+			t.Errorf("%s: Expected %d offsets, got %d", name, len(expected), len(offsets))
+			continue
+		}
+		for i, offset := range offsets {
+			if offset != expected[i] {
+				t.Errorf("%s: Offset[%d]: expected %d, got %d", name, i, expected[i], offset)
+			}
+		}
+	}
+}
+
+// TestResumeFromDiskError tests handling of ResumeFromDiskError
+func TestResumeFromDiskError(t *testing.T) {
+	baseTime := time.Now()
+
+	mock := &MockLogBuffer{
+		diskEntries: []*filer_pb.LogEntry{
+			createTestEntry(0, baseTime, "key0", "value0"),
+			createTestEntry(1, baseTime.Add(1*time.Second), "key1", "value1"),
+		},
+		memoryEntries: []*filer_pb.LogEntry{
+			createTestEntry(10, baseTime.Add(10*time.Second), "key10", "value10"),
+		},
+		memoryStartOffset: 10,
+		memoryStopOffset:  10,
+	}
+
+	// Try to read offset 5, which is between disk (0-1) and memory (10)
+	// This should trigger ResumeFromDiskError from memory read
+	startPos := log_buffer.NewMessagePositionFromOffset(5)
+
+	eachLogFn := func(entry *filer_pb.LogEntry) (bool, error) {
+		return false, nil
+	}
+
+	eachLogWithOffsetFn := func(entry *filer_pb.LogEntry, offset int64) (bool, error) {
+		return eachLogFn(entry)
+	}
+
+	// Disk read should return no data (offset 5 > disk end)
+	_, _, err := mock.MockReadFromDiskFn(startPos, 0, eachLogFn)
+	if err != nil {
+		t.Fatalf("Unexpected disk read error: %v", err)
+	}
+
+	// Memory read should return ResumeFromDiskError (offset 5 < memory start)
+	_, _, err = mock.MockLoopProcessLogDataWithOffset("test", startPos, 0, func() bool { return true }, eachLogWithOffsetFn)
+	if err != log_buffer.ResumeFromDiskError {
+		t.Errorf("Expected ResumeFromDiskError, got: %v", err)
+	}
+}

weed/mq/topic/local_partition_subscribers.go

@@ -0,0 +1,124 @@
+package topic
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+type LocalPartitionSubscribers struct {
+	Subscribers     map[string]*LocalSubscriber
+	SubscribersLock sync.RWMutex
+}
+type LocalSubscriber struct {
+	connectTimeNs      int64 // accessed atomically
+	lastSeenTimeNs     int64 // accessed atomically
+	lastReceivedOffset int64 // accessed atomically - offset of last message received
+	lastAckedOffset    int64 // accessed atomically - offset of last message acknowledged
+	stopCh             chan struct{}
+}
+
+func NewLocalSubscriber() *LocalSubscriber {
+	now := time.Now().UnixNano()
+	subscriber := &LocalSubscriber{
+		stopCh: make(chan struct{}, 1),
+	}
+	atomic.StoreInt64(&subscriber.connectTimeNs, now)
+	atomic.StoreInt64(&subscriber.lastSeenTimeNs, now)
+	atomic.StoreInt64(&subscriber.lastReceivedOffset, 0)
+	atomic.StoreInt64(&subscriber.lastAckedOffset, 0)
+	return subscriber
+}
+func (p *LocalSubscriber) SignalShutdown() {
+	close(p.stopCh)
+}
+
+// UpdateLastSeen updates the last activity time for this subscriber
+func (p *LocalSubscriber) UpdateLastSeen() {
+	atomic.StoreInt64(&p.lastSeenTimeNs, time.Now().UnixNano())
+}
+
+// UpdateReceivedOffset updates the offset of the last message received by this subscriber
+func (p *LocalSubscriber) UpdateReceivedOffset(offset int64) {
+	atomic.StoreInt64(&p.lastReceivedOffset, offset)
+	atomic.StoreInt64(&p.lastSeenTimeNs, time.Now().UnixNano())
+}
+
+// UpdateAckedOffset updates the offset of the last message acknowledged by this subscriber
+func (p *LocalSubscriber) UpdateAckedOffset(offset int64) {
+	atomic.StoreInt64(&p.lastAckedOffset, offset)
+	atomic.StoreInt64(&p.lastSeenTimeNs, time.Now().UnixNano())
+}
+
+// GetTimestamps returns the connect and last seen timestamps safely
+func (p *LocalSubscriber) GetTimestamps() (connectTimeNs, lastSeenTimeNs int64) {
+	return atomic.LoadInt64(&p.connectTimeNs), atomic.LoadInt64(&p.lastSeenTimeNs)
+}
+
+// GetOffsets returns the received and acknowledged offsets safely
+func (p *LocalSubscriber) GetOffsets() (lastReceivedOffset, lastAckedOffset int64) {
+	return atomic.LoadInt64(&p.lastReceivedOffset), atomic.LoadInt64(&p.lastAckedOffset)
+}
+
+// GetCurrentOffset returns the acknowledged offset (for compatibility)
+func (p *LocalSubscriber) GetCurrentOffset() int64 {
+	return atomic.LoadInt64(&p.lastAckedOffset)
+}
+
+func NewLocalPartitionSubscribers() *LocalPartitionSubscribers {
+	return &LocalPartitionSubscribers{
+		Subscribers: make(map[string]*LocalSubscriber),
+	}
+}
+
+func (p *LocalPartitionSubscribers) AddSubscriber(clientName string, Subscriber *LocalSubscriber) {
+	p.SubscribersLock.Lock()
+	defer p.SubscribersLock.Unlock()
+
+	p.Subscribers[clientName] = Subscriber
+}
+
+func (p *LocalPartitionSubscribers) RemoveSubscriber(clientName string) {
+	p.SubscribersLock.Lock()
+	defer p.SubscribersLock.Unlock()
+
+	delete(p.Subscribers, clientName)
+}
+
+func (p *LocalPartitionSubscribers) SignalShutdown() {
+	p.SubscribersLock.RLock()
+	defer p.SubscribersLock.RUnlock()
+
+	for _, Subscriber := range p.Subscribers {
+		Subscriber.SignalShutdown()
+	}
+}
+
+func (p *LocalPartitionSubscribers) Size() int {
+	p.SubscribersLock.RLock()
+	defer p.SubscribersLock.RUnlock()
+
+	return len(p.Subscribers)
+}
+
+// GetSubscriberNames returns the names of all subscribers
+func (p *LocalPartitionSubscribers) GetSubscriberNames() []string {
+	p.SubscribersLock.RLock()
+	defer p.SubscribersLock.RUnlock()
+
+	names := make([]string, 0, len(p.Subscribers))
+	for name := range p.Subscribers {
+		names = append(names, name)
+	}
+	return names
+}
+
+// ForEachSubscriber iterates over all subscribers
+func (p *LocalPartitionSubscribers) ForEachSubscriber(fn func(name string, subscriber *LocalSubscriber)) {
+	p.SubscribersLock.RLock()
+	defer p.SubscribersLock.RUnlock()
+
+	for name, subscriber := range p.Subscribers {
+		fn(name, subscriber)
+	}
+}

weed/mq/topic/local_topic.go

@@ -0,0 +1,113 @@
+package topic
+
+import (
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+type LocalTopic struct {
+	Topic
+	Partitions    []*LocalPartition
+	partitionLock sync.RWMutex
+}
+
+func NewLocalTopic(topic Topic) *LocalTopic {
+	return &LocalTopic{
+		Topic:      topic,
+		Partitions: make([]*LocalPartition, 0),
+	}
+}
+
+func (localTopic *LocalTopic) findPartition(partition Partition) *LocalPartition {
+	localTopic.partitionLock.RLock()
+	defer localTopic.partitionLock.RUnlock()
+
+	glog.V(4).Infof("findPartition searching for %s in %d partitions", partition.String(), len(localTopic.Partitions))
+	for i, localPartition := range localTopic.Partitions {
+		glog.V(4).Infof("Comparing partition[%d]: %s with target %s", i, localPartition.Partition.String(), partition.String())
+		if localPartition.Partition.LogicalEquals(partition) {
+			glog.V(4).Infof("Found matching partition at index %d", i)
+			return localPartition
+		}
+	}
+	glog.V(4).Infof("No matching partition found for %s", partition.String())
+	return nil
+}
+func (localTopic *LocalTopic) removePartition(partition Partition) bool {
+	localTopic.partitionLock.Lock()
+	defer localTopic.partitionLock.Unlock()
+
+	foundPartitionIndex := -1
+	for i, localPartition := range localTopic.Partitions {
+		if localPartition.Partition.LogicalEquals(partition) {
+			foundPartitionIndex = i
+			localPartition.Shutdown()
+			break
+		}
+	}
+	if foundPartitionIndex == -1 {
+		return false
+	}
+	localTopic.Partitions = append(localTopic.Partitions[:foundPartitionIndex], localTopic.Partitions[foundPartitionIndex+1:]...)
+	return true
+}
+func (localTopic *LocalTopic) addPartition(localPartition *LocalPartition) {
+	localTopic.partitionLock.Lock()
+	defer localTopic.partitionLock.Unlock()
+	for _, partition := range localTopic.Partitions {
+		if localPartition.Partition.LogicalEquals(partition.Partition) {
+			return
+		}
+	}
+	localTopic.Partitions = append(localTopic.Partitions, localPartition)
+}
+
+func (localTopic *LocalTopic) closePartitionPublishers(unixTsNs int64) bool {
+	var wg sync.WaitGroup
+	for _, localPartition := range localTopic.Partitions {
+		if localPartition.UnixTimeNs != unixTsNs {
+			continue
+		}
+		wg.Add(1)
+		go func(localPartition *LocalPartition) {
+			defer wg.Done()
+			localPartition.closePublishers()
+		}(localPartition)
+	}
+	wg.Wait()
+	return true
+}
+
+func (localTopic *LocalTopic) closePartitionSubscribers(unixTsNs int64) bool {
+	var wg sync.WaitGroup
+	for _, localPartition := range localTopic.Partitions {
+		if localPartition.UnixTimeNs != unixTsNs {
+			continue
+		}
+		wg.Add(1)
+		go func(localPartition *LocalPartition) {
+			defer wg.Done()
+			localPartition.closeSubscribers()
+		}(localPartition)
+	}
+	wg.Wait()
+	return true
+}
+
+func (localTopic *LocalTopic) WaitUntilNoPublishers() {
+	for {
+		var wg sync.WaitGroup
+		for _, localPartition := range localTopic.Partitions {
+			wg.Add(1)
+			go func(localPartition *LocalPartition) {
+				defer wg.Done()
+				localPartition.WaitUntilNoPublishers()
+			}(localPartition)
+		}
+		wg.Wait()
+		if len(localTopic.Partitions) == 0 {
+			return
+		}
+	}
+}

weed/mq/topic/partition.go

@@ -0,0 +1,89 @@
+package topic
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+const PartitionCount = 4096
+
+type Partition struct {
+	RangeStart int32
+	RangeStop  int32 // exclusive
+	RingSize   int32
+	UnixTimeNs int64 // in nanoseconds
+}
+
+func (partition Partition) Equals(other Partition) bool {
+	if partition.RangeStart != other.RangeStart {
+		return false
+	}
+	if partition.RangeStop != other.RangeStop {
+		return false
+	}
+	if partition.RingSize != other.RingSize {
+		return false
+	}
+	if partition.UnixTimeNs != other.UnixTimeNs {
+		return false
+	}
+	return true
+}
+
+// LogicalEquals compares only the partition boundaries (RangeStart, RangeStop)
+// This is useful when comparing partitions that may have different timestamps or ring sizes
+// but represent the same logical partition range
+func (partition Partition) LogicalEquals(other Partition) bool {
+	return partition.RangeStart == other.RangeStart && partition.RangeStop == other.RangeStop
+}
+
+func FromPbPartition(partition *schema_pb.Partition) Partition {
+	return Partition{
+		RangeStart: partition.RangeStart,
+		RangeStop:  partition.RangeStop,
+		RingSize:   partition.RingSize,
+		UnixTimeNs: partition.UnixTimeNs,
+	}
+}
+
+func (partition Partition) ToPbPartition() *schema_pb.Partition {
+	return &schema_pb.Partition{
+		RangeStart: partition.RangeStart,
+		RangeStop:  partition.RangeStop,
+		RingSize:   partition.RingSize,
+		UnixTimeNs: partition.UnixTimeNs,
+	}
+}
+
+func (partition Partition) Overlaps(partition2 Partition) bool {
+	if partition.RangeStart >= partition2.RangeStop {
+		return false
+	}
+	if partition.RangeStop <= partition2.RangeStart {
+		return false
+	}
+	return true
+}
+
+func (partition Partition) String() string {
+	return fmt.Sprintf("%04d-%04d", partition.RangeStart, partition.RangeStop)
+}
+
+func ParseTopicVersion(name string) (t time.Time, err error) {
+	return time.Parse(PartitionGenerationFormat, name)
+}
+
+func ParsePartitionBoundary(name string) (start, stop int32) {
+	_, err := fmt.Sscanf(name, "%04d-%04d", &start, &stop)
+	if err != nil {
+		return 0, 0
+	}
+	return start, stop
+}
+
+func PartitionDir(t Topic, p Partition) string {
+	partitionGeneration := time.Unix(0, p.UnixTimeNs).UTC().Format(PartitionGenerationFormat)
+	return fmt.Sprintf("%s/%s/%04d-%04d", t.Dir(), partitionGeneration, p.RangeStart, p.RangeStop)
+}

weed/mq/topic/topic.go

@@ -0,0 +1,169 @@
+package topic
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	jsonpb "google.golang.org/protobuf/encoding/protojson"
+)
+
+type Topic struct {
+	Namespace string
+	Name      string
+}
+
+func NewTopic(namespace string, name string) Topic {
+	return Topic{
+		Namespace: namespace,
+		Name:      name,
+	}
+}
+func FromPbTopic(topic *schema_pb.Topic) Topic {
+	return Topic{
+		Namespace: topic.Namespace,
+		Name:      topic.Name,
+	}
+}
+
+func (t Topic) ToPbTopic() *schema_pb.Topic {
+	return &schema_pb.Topic{
+		Namespace: t.Namespace,
+		Name:      t.Name,
+	}
+}
+
+func (t Topic) String() string {
+	return fmt.Sprintf("%s.%s", t.Namespace, t.Name)
+}
+
+func (t Topic) Dir() string {
+	return fmt.Sprintf("%s/%s/%s", filer.TopicsDir, t.Namespace, t.Name)
+}
+
+func (t Topic) ReadConfFile(client filer_pb.SeaweedFilerClient) (*mq_pb.ConfigureTopicResponse, error) {
+	data, err := filer.ReadInsideFiler(context.Background(), client, t.Dir(), filer.TopicConfFile)
+	if errors.Is(err, filer_pb.ErrNotFound) {
+		return nil, err
+	}
+	if err != nil {
+		return nil, fmt.Errorf("read topic.conf of %v: %w", t, err)
+	}
+	// parse into filer conf object
+	conf := &mq_pb.ConfigureTopicResponse{}
+	if err = jsonpb.Unmarshal(data, conf); err != nil {
+		return nil, fmt.Errorf("unmarshal topic %v conf: %w", t, err)
+	}
+	return conf, nil
+}
+
+// ReadConfFileWithMetadata reads the topic configuration and returns it along with file metadata
+func (t Topic) ReadConfFileWithMetadata(client filer_pb.SeaweedFilerClient) (*mq_pb.ConfigureTopicResponse, int64, int64, error) {
+	// Use LookupDirectoryEntry to get both content and metadata
+	request := &filer_pb.LookupDirectoryEntryRequest{
+		Directory: t.Dir(),
+		Name:      filer.TopicConfFile,
+	}
+
+	resp, err := filer_pb.LookupEntry(context.Background(), client, request)
+	if err != nil {
+		if errors.Is(err, filer_pb.ErrNotFound) {
+			return nil, 0, 0, err
+		}
+		return nil, 0, 0, fmt.Errorf("lookup topic.conf of %v: %w", t, err)
+	}
+
+	// Get file metadata
+	var createdAtNs, modifiedAtNs int64
+	if resp.Entry.Attributes != nil {
+		createdAtNs = resp.Entry.Attributes.Crtime * 1e9 // convert seconds to nanoseconds
+		modifiedAtNs = resp.Entry.Attributes.Mtime * 1e9 // convert seconds to nanoseconds
+	}
+
+	// Parse the configuration
+	conf := &mq_pb.ConfigureTopicResponse{}
+	if err = jsonpb.Unmarshal(resp.Entry.Content, conf); err != nil {
+		return nil, 0, 0, fmt.Errorf("unmarshal topic %v conf: %w", t, err)
+	}
+
+	return conf, createdAtNs, modifiedAtNs, nil
+}
+
+func (t Topic) WriteConfFile(client filer_pb.SeaweedFilerClient, conf *mq_pb.ConfigureTopicResponse) error {
+	var buf bytes.Buffer
+	filer.ProtoToText(&buf, conf)
+	if err := filer.SaveInsideFiler(context.Background(), client, t.Dir(), filer.TopicConfFile, buf.Bytes()); err != nil {
+		return fmt.Errorf("save topic %v conf: %w", t, err)
+	}
+	return nil
+}
+
+// DiscoverPartitions discovers all partition directories for a topic by scanning the filesystem
+// This centralizes partition discovery logic used across query engine, shell commands, etc.
+func (t Topic) DiscoverPartitions(ctx context.Context, filerClient filer_pb.FilerClient) ([]string, error) {
+	var partitionPaths []string
+
+	// Scan the topic directory for version directories (e.g., v2025-09-01-07-16-34)
+	err := filer_pb.ReadDirAllEntries(ctx, filerClient, util.FullPath(t.Dir()), "", func(versionEntry *filer_pb.Entry, isLast bool) error {
+		if !versionEntry.IsDirectory {
+			return nil // Skip non-directories
+		}
+
+		// Parse version timestamp from directory name (e.g., "v2025-09-01-07-16-34")
+		if !IsValidVersionDirectory(versionEntry.Name) {
+			// Skip directories that don't match the version format
+			return nil
+		}
+
+		// Scan partition directories within this version (e.g., 0000-0630)
+		versionDir := fmt.Sprintf("%s/%s", t.Dir(), versionEntry.Name)
+		return filer_pb.ReadDirAllEntries(ctx, filerClient, util.FullPath(versionDir), "", func(partitionEntry *filer_pb.Entry, isLast bool) error {
+			if !partitionEntry.IsDirectory {
+				return nil // Skip non-directories
+			}
+
+			// Parse partition boundary from directory name (e.g., "0000-0630")
+			if !IsValidPartitionDirectory(partitionEntry.Name) {
+				return nil // Skip invalid partition names
+			}
+
+			// Add this partition path to the list
+			partitionPath := fmt.Sprintf("%s/%s", versionDir, partitionEntry.Name)
+			partitionPaths = append(partitionPaths, partitionPath)
+			return nil
+		})
+	})
+
+	return partitionPaths, err
+}
+
+// IsValidVersionDirectory checks if a directory name matches the topic version format
+// Format: v2025-09-01-07-16-34
+func IsValidVersionDirectory(name string) bool {
+	if !strings.HasPrefix(name, "v") || len(name) != 20 {
+		return false
+	}
+
+	// Try to parse the timestamp part
+	timestampStr := name[1:] // Remove 'v' prefix
+	_, err := time.Parse("2006-01-02-15-04-05", timestampStr)
+	return err == nil
+}
+
+// IsValidPartitionDirectory checks if a directory name matches the partition boundary format
+// Format: 0000-0630 (rangeStart-rangeStop)
+func IsValidPartitionDirectory(name string) bool {
+	// Use existing ParsePartitionBoundary function to validate
+	start, stop := ParsePartitionBoundary(name)
+
+	// Valid partition ranges should have start < stop (and not both be 0, which indicates parse error)
+	return start < stop && start >= 0
+}

weed/mq/topic/topic_partition.go

@@ -0,0 +1,12 @@
+package topic
+
+import "fmt"
+
+type TopicPartition struct {
+	Topic
+	Partition
+}
+
+func (tp *TopicPartition) TopicPartitionId() string {
+	return fmt.Sprintf("%v.%v-%04d-%04d", tp.Namespace, tp.Topic, tp.RangeStart, tp.RangeStop)
+}