snapshot of https://github.com/mattermost/mattermost.git (master, 2026-09-16; blobs > 131072B omitted)

server/platform/services/remotecluster/README.md

@@ -0,0 +1,36 @@
+## Remote Cluster Service
+
+Package `remotecluster` implements Mattermost's "Secured Connections" feature, which enables communication between different Mattermost clusters. Specifically, this package provides:
+
+ ### Service Management:
+
+- Manages inter-cluster communication via topic-based messages
+- Handles connection state (active/inactive) based on cluster leadership
+- Maintains concurrent send channels (MaxConcurrentSends = 10) for parallel message processing
+- Implements periodic health checks (pings) to monitor remote cluster connectivity
+
+ ### Message Handling:
+
+- Sends messages using a pool of goroutines to handle concurrent sends while preserving message order per remote
+- Uses hash-based routing to ensure messages for the same remote ID go to the same channel
+- Supports different types of sends: messages, files, and profile images
+- Implements topic-based message routing with listener callbacks
+
+ ### Connection Management:
+
+- Handles invitation confirmations between clusters
+- Maintains HTTP client connections with proper timeouts and transport settings
+- Supports connection state listeners for monitoring remote cluster availability
+- Implements ping mechanism to verify remote cluster health
+
+ ### Core Features:
+
+- Topic-based message routing
+- File transfer capabilities
+- Profile image synchronization
+- Invitation system for establishing connections
+- Health monitoring via pings
+- Concurrent message processing
+- Connection state management
+
+This package is designed to be thread-safe and handles leadership changes in clustered environments, only running active operations on the leader node.

server/platform/services/remotecluster/error.go

@@ -0,0 +1,31 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	RemoteClusterAlreadyConfirmedError = errors.New("the remote cluster has already been confirmed")
+)
+
+type BufferFullError struct {
+	capacity int
+}
+
+func NewBufferFullError(capacity int) BufferFullError {
+	return BufferFullError{
+		capacity: capacity,
+	}
+}
+
+func (e BufferFullError) Capacity() int {
+	return e.capacity
+}
+
+func (e BufferFullError) Error() string {
+	return fmt.Sprintf("buffer capacity (%d) exceeded", e.capacity)
+}

server/platform/services/remotecluster/invitation.go

@@ -0,0 +1,97 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/mattermost/mattermost/server/public/model"
+)
+
+// AcceptInvitation is called when accepting an invitation to connect with a remote cluster.
+func (rcs *Service) AcceptInvitation(invite *model.RemoteClusterInvite, name string, displayName string, creatorId string, siteURL string, defaultTeamId string) (*model.RemoteCluster, error) {
+	// Generate new token for RemoteToken only if invite version is 2 or greater
+	var remoteToken string
+	if invite.Version >= 2 {
+		remoteToken = model.NewId() // Generate new token for v2+ protocol
+	} else {
+		remoteToken = invite.Token // Use the token from the invite for backwards compatibility
+	}
+
+	rc := &model.RemoteCluster{
+		RemoteId:      invite.RemoteId,
+		Name:          name,
+		DisplayName:   displayName,
+		DefaultTeamId: defaultTeamId,
+		Token:         model.NewId(),
+		RemoteToken:   remoteToken,
+		SiteURL:       invite.SiteURL,
+		CreatorId:     creatorId,
+	}
+
+	rcSaved, err := rcs.server.GetStore().RemoteCluster().Save(rc)
+	if err != nil {
+		return nil, err
+	}
+
+	// confirm the invitation with the originating site
+	frame, err := makeConfirmFrame(rcSaved, siteURL)
+	if err != nil {
+		return nil, err
+	}
+
+	url := fmt.Sprintf("%s/%s", rcSaved.SiteURL, ConfirmInviteURL)
+
+	// for the invite confirm message, we need to use the token that
+	// the originating server sent in the invite instead of the one
+	// we're storing as a refresh
+	rc.RemoteToken = invite.Token
+
+	resp, err := rcs.sendFrameToRemote(PingTimeout, rc, frame, url)
+	if err != nil {
+		rcs.server.GetStore().RemoteCluster().Delete(rcSaved.RemoteId)
+		return nil, err
+	}
+
+	var response Response
+	err = json.Unmarshal(resp, &response)
+	if err != nil {
+		rcs.server.GetStore().RemoteCluster().Delete(rcSaved.RemoteId)
+		return nil, fmt.Errorf("invalid response from remote server: %w", err)
+	}
+
+	if !response.IsSuccess() {
+		rcs.server.GetStore().RemoteCluster().Delete(rcSaved.RemoteId)
+		return nil, errors.New(response.Err)
+	}
+
+	// issue the first ping right away. The goroutine will exit when ping completes or PingTimeout exceeded.
+	go rcs.PingNow(rcSaved)
+
+	return rcSaved, nil
+}
+
+func makeConfirmFrame(rc *model.RemoteCluster, siteURL string) (*model.RemoteClusterFrame, error) {
+	confirm := model.RemoteClusterInvite{
+		RemoteId:       rc.RemoteId,
+		SiteURL:        siteURL,
+		Token:          rc.Token,
+		RefreshedToken: rc.RemoteToken,
+		Version:        3,
+	}
+	confirmRaw, err := json.Marshal(confirm)
+	if err != nil {
+		return nil, err
+	}
+
+	msg := model.NewRemoteClusterMsg(InvitationTopic, confirmRaw)
+
+	frame := &model.RemoteClusterFrame{
+		RemoteId: rc.RemoteId,
+		Msg:      msg,
+	}
+	return frame, nil
+}

server/platform/services/remotecluster/mocks_test.go

@@ -0,0 +1,135 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"slices"
+	"sync"
+	"testing"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
+	"github.com/mattermost/mattermost/server/v8/einterfaces"
+)
+
+type mockServer struct {
+	remotes     []*model.RemoteCluster
+	logger      *mlog.Logger
+	user        *model.User
+	customStore store.Store // if set, GetStore() returns this instead of creating default
+}
+
+func newMockServer(t *testing.T, remotes []*model.RemoteCluster) *mockServer {
+	logger := mlog.CreateConsoleTestLogger(t)
+
+	return &mockServer{
+		remotes: remotes,
+		logger:  logger,
+	}
+}
+
+// newMockServerWithStore creates a mockServer with a custom store for fine-grained test control.
+// Use this when you need to mock specific store behaviors that aren't covered by the default.
+func newMockServerWithStore(t *testing.T, customStore store.Store) *mockServer {
+	logger := mlog.CreateConsoleTestLogger(t)
+
+	return &mockServer{
+		logger:      logger,
+		customStore: customStore,
+	}
+}
+
+func (ms *mockServer) SetUser(user *model.User) {
+	ms.user = user
+}
+
+func (ms *mockServer) Config() *model.Config                                  { return nil }
+func (ms *mockServer) GetMetrics() einterfaces.MetricsInterface               { return nil }
+func (ms *mockServer) IsLeader() bool                                         { return true }
+func (ms *mockServer) AddClusterLeaderChangedListener(listener func()) string { return model.NewId() }
+func (ms *mockServer) RemoveClusterLeaderChangedListener(id string)           {}
+func (ms *mockServer) Log() *mlog.Logger {
+	return ms.logger
+}
+func (ms *mockServer) GetStore() store.Store {
+	// If a custom store was provided, use it (for unit tests with fine-grained control)
+	if ms.customStore != nil {
+		return ms.customStore
+	}
+
+	// Otherwise, return the default pre-configured store (for integration-style tests)
+	anyQueryFilter := mock.MatchedBy(func(filter model.RemoteClusterQueryFilter) bool {
+		return true
+	})
+	anyUserId := mock.AnythingOfType("string")
+	anyId := mock.AnythingOfType("string")
+
+	remoteClusterStoreMock := &mocks.RemoteClusterStore{}
+	remoteClusterStoreMock.On("GetByTopic", "share").Return(ms.remotes, nil)
+	remoteClusterStoreMock.On("GetAll", 0, 999999, anyQueryFilter).Return(ms.remotes, nil)
+	remoteClusterStoreMock.On("SetLastPingAt", anyId).Return(nil)
+
+	userStoreMock := &mocks.UserStore{}
+	userStoreMock.On("Get", mock.Anything, anyUserId).Return(ms.user, nil)
+
+	storeMock := &mocks.Store{}
+	storeMock.On("RemoteCluster").Return(remoteClusterStoreMock)
+	storeMock.On("User").Return(userStoreMock)
+	return storeMock
+}
+
+type mockApp struct {
+	offlinePluginIDs []string
+
+	mux             sync.Mutex
+	totalPingCount  int
+	totalPingErrors int
+	pingCounts      map[string]int
+}
+
+func newMockApp(_ *testing.T, offlinePluginIDs []string) *mockApp {
+	return &mockApp{
+		offlinePluginIDs: offlinePluginIDs,
+		pingCounts:       make(map[string]int),
+	}
+}
+
+func (ma *mockApp) OnSharedChannelsPing(rc *model.RemoteCluster) bool {
+	ma.mux.Lock()
+	defer ma.mux.Unlock()
+
+	if slices.Contains(ma.offlinePluginIDs, rc.PluginID) {
+		ma.totalPingErrors++
+		return false
+	}
+
+	ma.totalPingCount++
+
+	count := ma.pingCounts[rc.PluginID]
+	ma.pingCounts[rc.PluginID] = count + 1
+
+	return true
+}
+
+func (ma *mockApp) GetTotalPingCount() int {
+	ma.mux.Lock()
+	defer ma.mux.Unlock()
+	return ma.totalPingCount
+}
+
+func (ma *mockApp) GetTotalPingErrorCount() int {
+	ma.mux.Lock()
+	defer ma.mux.Unlock()
+	return ma.totalPingErrors
+}
+
+func (ma *mockApp) GetPingCount(pluginID string) int {
+	ma.mux.Lock()
+	defer ma.mux.Unlock()
+
+	return ma.pingCounts[pluginID]
+}

server/platform/services/remotecluster/ping.go

@@ -0,0 +1,210 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+)
+
+// PingNow emits a ping immediately without waiting for next ping loop.
+func (rcs *Service) PingNow(rc *model.RemoteCluster) {
+	online := rc.IsOnline()
+
+	pingErr := rcs.pingRemote(rc)
+	if pingErr != nil {
+		rcs.server.Log().LogM(mlog.MlvlRemoteClusterServiceWarn, "Remote cluster ping failed",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("remote_id", rc.RemoteId),
+			mlog.String("plugin_id", rc.PluginID),
+			mlog.Err(pingErr),
+		)
+	}
+
+	pingSucceeded := pingErr == nil
+	hasPendingSync := false
+	if pingSucceeded {
+		// Consume the pending-sync-failure marker (if any). LoadAndDelete clears it so the
+		// map does not accumulate entries for remotes that have recovered.
+		_, hasPendingSync = rcs.syncFailedSinceLastPing.LoadAndDelete(rc.RemoteId)
+	}
+
+	// The connection-state-change counter tracks genuine online/offline transitions, so
+	// only bump it when IsOnline() actually flipped. The event itself must still fire on
+	// the pending-sync recovery path (online→online) to drive ForceSyncForRemote.
+	stateChanged := online != rc.IsOnline()
+	if stateChanged || (pingSucceeded && hasPendingSync) {
+		if stateChanged {
+			if metrics := rcs.server.GetMetrics(); metrics != nil {
+				metrics.IncrementRemoteClusterConnStateChangeCounter(rc.RemoteId, rc.IsOnline())
+			}
+		}
+		rcs.fireConnectionStateChgEvent(rc)
+	}
+}
+
+// pingAllNow emits a ping to all remotes immediately without waiting for next ping loop.
+func (rcs *Service) pingAllNow(filter model.RemoteClusterQueryFilter) {
+	// get all remotes, including any previously offline.
+	remotes, err := rcs.server.GetStore().RemoteCluster().GetAll(0, 999999, filter)
+	if err != nil {
+		rcs.server.Log().LogM(mlog.MlvlRemoteClusterServiceError, "Ping all remote clusters failed (could not get list of remotes)", mlog.Err(err))
+		return
+	}
+
+	for _, rc := range remotes {
+		// filter out unconfirmed invites so we don't ping them without permission
+		if rc.IsConfirmed() {
+			rcs.PingNow(rc)
+		}
+	}
+}
+
+// pingLoop periodically sends a ping to all remote clusters.
+func (rcs *Service) pingLoop(done <-chan struct{}) {
+	pingChan := make(chan *model.RemoteCluster, MaxConcurrentSends*2)
+
+	// create a thread pool to send pings concurrently to remotes.
+	for range MaxConcurrentSends {
+		go rcs.pingEmitter(pingChan, done)
+	}
+
+	go rcs.pingGenerator(pingChan, done)
+}
+
+func (rcs *Service) pingGenerator(pingChan chan *model.RemoteCluster, done <-chan struct{}) {
+	defer close(pingChan)
+
+	for {
+		pingFreq := rcs.GetPingFreq()
+		start := time.Now()
+
+		// ping all remotes, including any previously offline.
+		rcs.pingAllNow(model.RemoteClusterQueryFilter{})
+
+		// try to maintain frequency
+		elapsed := time.Since(start)
+		if elapsed < pingFreq {
+			sleep := time.Until(start.Add(pingFreq))
+			select {
+			case <-time.After(sleep):
+			case <-done:
+				return
+			}
+		}
+	}
+}
+
+// pingEmitter pulls Remotes from the ping queue (pingChan) and pings them.
+// Pinging a remote cannot take longer than PingTimeoutMillis.
+func (rcs *Service) pingEmitter(pingChan <-chan *model.RemoteCluster, done <-chan struct{}) {
+	for {
+		select {
+		case rc := <-pingChan:
+			if rc == nil {
+				return
+			}
+			rcs.PingNow(rc)
+		case <-done:
+			return
+		}
+	}
+}
+
+var ErrPluginPingFail = errors.New("plugin ping failed")
+
+// pingRemote make a synchronous ping to a remote cluster. Return is error if ping is
+// unsuccessful and nil on success.
+func (rcs *Service) pingRemote(rc *model.RemoteCluster) error {
+	ping := model.RemoteClusterPing{}
+
+	if rc.IsPlugin() {
+		ping.SentAt = model.GetMillis()
+		if ok := rcs.app.OnSharedChannelsPing(rc); !ok {
+			return ErrPluginPingFail
+		}
+		ping.RecvAt = model.GetMillis()
+	} else {
+		frame, err := makePingFrame(rc)
+		if err != nil {
+			return err
+		}
+		url := fmt.Sprintf("%s/%s", rc.SiteURL, PingURL)
+
+		resp, err := rcs.sendFrameToRemote(PingTimeout, rc, frame, url)
+		if err != nil {
+			return err
+		}
+		rc.LastPingAt = model.GetMillis()
+
+		err = json.Unmarshal(resp, &ping)
+		if err != nil {
+			return err
+		}
+	}
+
+	if err := rcs.server.GetStore().RemoteCluster().SetLastPingAt(rc.RemoteId); err != nil {
+		rcs.server.Log().LogM(mlog.MlvlRemoteClusterServiceError, "Failed to update LastPingAt for remote cluster",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("remote_id", rc.RemoteId),
+			mlog.Err(err),
+		)
+	}
+
+	if metrics := rcs.server.GetMetrics(); metrics != nil {
+		sentAt := time.Unix(0, ping.SentAt*int64(time.Millisecond))
+		elapsed := time.Since(sentAt).Seconds()
+		metrics.ObserveRemoteClusterPingDuration(rc.RemoteId, elapsed)
+
+		// we approximate clock skew between remotes.
+		skew := elapsed/2 - float64(ping.RecvAt-ping.SentAt)/1000
+		metrics.ObserveRemoteClusterClockSkew(rc.RemoteId, skew)
+	}
+
+	rcs.server.Log().Log(mlog.LvlRemoteClusterServiceDebug, "Remote cluster ping",
+		mlog.String("remote", rc.DisplayName),
+		mlog.String("remote_id", rc.RemoteId),
+		mlog.String("plugin_id", rc.PluginID),
+		mlog.Int("sent_at", ping.SentAt),
+		mlog.Int("recv_at", ping.RecvAt),
+		mlog.Int("diff", ping.RecvAt-ping.SentAt),
+	)
+	return nil
+}
+
+func makePingFrame(rc *model.RemoteCluster) (*model.RemoteClusterFrame, error) {
+	ping := model.RemoteClusterPing{
+		SentAt: model.GetMillis(),
+	}
+	pingRaw, err := json.Marshal(ping)
+	if err != nil {
+		return nil, err
+	}
+
+	msg := model.NewRemoteClusterMsg(PingTopic, pingRaw)
+
+	frame := &model.RemoteClusterFrame{
+		RemoteId: rc.RemoteId,
+		Msg:      msg,
+	}
+	return frame, nil
+}
+
+func (rcs *Service) fireConnectionStateChgEvent(rc *model.RemoteCluster) {
+	rcs.mux.RLock()
+	listeners := make([]ConnectionStateListener, 0, len(rcs.connectionStateListeners))
+	for _, l := range rcs.connectionStateListeners {
+		listeners = append(listeners, l)
+	}
+	rcs.mux.RUnlock()
+
+	for _, l := range listeners {
+		l(rc, rc.IsOnline())
+	}
+}

server/platform/services/remotecluster/ping_test.go

@@ -0,0 +1,291 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/wiggin77/merror"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
+)
+
+const (
+	Recent = 60000
+)
+
+func TestPing(t *testing.T) {
+	t.Run("No error", func(t *testing.T) {
+		merr := merror.New()
+
+		var remotes []*model.RemoteCluster
+		pingsReceived := make(map[string]struct{})
+		var mux sync.Mutex
+
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			defer w.WriteHeader(200)
+
+			var frame model.RemoteClusterFrame
+			err := json.NewDecoder(r.Body).Decode(&frame)
+			if err != nil {
+				merr.Append(err)
+				return
+			}
+			if len(frame.Msg.Payload) == 0 {
+				merr.Append(fmt.Errorf("Payload should not be empty; remote_id=%s", frame.RemoteId))
+				return
+			}
+
+			// Make sure ping is from a remote that was added for this test.
+			if !hasRemoteID(frame.RemoteId, remotes) {
+				merr.Append(fmt.Errorf("RemoteID not in list of remotes for this test; remote_id=%s", frame.RemoteId))
+				return
+			}
+
+			var ping model.RemoteClusterPing
+			err = json.Unmarshal(frame.Msg.Payload, &ping)
+			if err != nil {
+				merr.Append(err)
+				return
+			}
+
+			if !checkRecent(ping.SentAt, Recent) {
+				merr.Append(fmt.Errorf("timestamp out of range, got %d", ping.SentAt))
+				return
+			}
+			if ping.RecvAt != 0 {
+				merr.Append(fmt.Errorf("timestamp should be 0, got %d", ping.RecvAt))
+				return
+			}
+
+			mux.Lock()
+			defer mux.Unlock()
+			pingsReceived[frame.RemoteId] = struct{}{}
+		}))
+		defer ts.Close()
+
+		remotes = makeRemoteClusters(NumRemotes, ts.URL, false)
+		mockServer := newMockServer(t, remotes)
+		mockApp := newMockApp(t, nil)
+
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer service.Shutdown()
+
+		// wait up to 10 seconds for all remotes to get pinged. This will normally take less than 1 second
+		// unless the server is very busy.
+		assert.Eventually(t, func() bool {
+			mux.Lock()
+			defer mux.Unlock()
+			return len(pingsReceived) == NumRemotes
+		}, time.Second*10, time.Millisecond*50, "all remotes must get pinged")
+
+		assert.NoError(t, merr.ErrorOrNil())
+	})
+
+	t.Run("HTTP errors", func(t *testing.T) {
+		merr := merror.New()
+
+		var remotes []*model.RemoteCluster
+		pingsReceived := make(map[string]struct{})
+		var mux sync.Mutex
+
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			var frame model.RemoteClusterFrame
+			err := json.NewDecoder(r.Body).Decode(&frame)
+			if err != nil {
+				merr.Append(err)
+			}
+
+			// Make sure ping is from a remote that was added for this test.
+			if !hasRemoteID(frame.RemoteId, remotes) {
+				merr.Append(fmt.Errorf("RemoteID not in list of remotes for this test; remote_id=%s", frame.RemoteId))
+				return
+			}
+
+			var ping model.RemoteClusterPing
+			err = json.Unmarshal(frame.Msg.Payload, &ping)
+			if err != nil {
+				merr.Append(err)
+			}
+			if !checkRecent(ping.SentAt, Recent) {
+				merr.Append(fmt.Errorf("timestamp out of range, got %d", ping.SentAt))
+			}
+			if ping.RecvAt != 0 {
+				merr.Append(fmt.Errorf("timestamp should be 0, got %d", ping.RecvAt))
+			}
+
+			w.WriteHeader(500)
+
+			mux.Lock()
+			defer mux.Unlock()
+			pingsReceived[frame.RemoteId] = struct{}{}
+		}))
+		defer ts.Close()
+
+		remotes = makeRemoteClusters(NumRemotes, ts.URL, false)
+		mockServer := newMockServer(t, remotes)
+		mockApp := newMockApp(t, nil)
+
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer service.Shutdown()
+
+		// wait up to 10 seconds for all remotes to get pinged. This will normally take less than 1 second
+		// until the server is very busy.
+		assert.Eventually(t, func() bool {
+			mux.Lock()
+			defer mux.Unlock()
+			return len(pingsReceived) == NumRemotes
+		}, time.Second*10, time.Millisecond*50, "all remotes must get pinged")
+
+		assert.NoError(t, merr.ErrorOrNil())
+	})
+
+	t.Run("Plugin ping", func(t *testing.T) {
+		mockServer := newMockServer(t, makeRemoteClusters(NumRemotes, model.NewId(), true))
+		offline := []string{mockServer.remotes[0].PluginID, mockServer.remotes[1].PluginID}
+
+		mockApp := newMockApp(t, offline)
+
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		// high ping frequency so we don't delay unit tests.
+		service.SetPingFreq(time.Millisecond * 50)
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer service.Shutdown()
+
+		checkPingCount := func() bool {
+			return mockApp.GetTotalPingCount() >= NumRemotes
+		}
+
+		checkErrorCount := func() bool {
+			return mockApp.GetTotalPingErrorCount() >= 2
+		}
+
+		assert.Eventually(t, checkPingCount, time.Second*5, 10*time.Millisecond)
+		assert.Eventually(t, checkErrorCount, time.Second*5, 10*time.Millisecond)
+	})
+}
+
+// pingTestListener records connection-state-change callbacks for the PingNow tests.
+type pingTestListener struct {
+	mu     sync.Mutex
+	calls  int
+	online bool
+}
+
+func (l *pingTestListener) callback(rc *model.RemoteCluster, online bool) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.calls++
+	l.online = online
+}
+
+func (l *pingTestListener) snapshot() (calls int, online bool) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.calls, l.online
+}
+
+// newPingTestService builds a remote cluster service whose remotes ping a local test server
+// that always responds 200, and registers a connection-state listener to observe PingNow.
+// The returned remote is already online (recent LastPingAt), so IsOnline() will not flip.
+func newPingTestService(t *testing.T) (*Service, *pingTestListener, *model.RemoteCluster) {
+	t.Helper()
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		pong, _ := json.Marshal(model.RemoteClusterPing{SentAt: model.GetMillis(), RecvAt: model.GetMillis()})
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		w.Write(pong)
+	}))
+	t.Cleanup(ts.Close)
+
+	rcStoreMock := &mocks.RemoteClusterStore{}
+	rcStoreMock.On("SetLastPingAt", mock.AnythingOfType("string")).Return(nil)
+	storeMock := &mocks.Store{}
+	storeMock.On("RemoteCluster").Return(rcStoreMock)
+
+	service, err := NewRemoteClusterService(newMockServerWithStore(t, storeMock), newMockApp(t, nil))
+	require.NoError(t, err)
+
+	listener := &pingTestListener{}
+	listenerId := service.AddConnectionStateListener(listener.callback)
+	t.Cleanup(func() { service.RemoveConnectionStateListener(listenerId) })
+
+	rc := &model.RemoteCluster{
+		RemoteId:    model.NewId(),
+		DisplayName: "test-remote",
+		SiteURL:     ts.URL,
+		Token:       model.NewId(),
+		RemoteToken: model.NewId(),
+		LastPingAt:  model.GetMillis(),
+	}
+	return service, listener, rc
+}
+
+// TestPingNow_SyncFailureRecovery verifies that PingNow fires a connection-state-change
+// event when a sync failure was recorded since the last ping, even if the remote never
+// appeared offline (LastPingAt stayed within the 5-minute window).
+func TestPingNow_SyncFailureRecovery(t *testing.T) {
+	service, listener, rc := newPingTestService(t)
+
+	// Simulate a sync failure that occurred since the last ping.
+	service.NotifySyncFailed(rc.RemoteId)
+
+	service.PingNow(rc)
+
+	calls, wasOnline := listener.snapshot()
+	assert.Equal(t, 1, calls, "connection state listener should fire exactly once on sync failure recovery")
+	assert.True(t, wasOnline, "listener should report online=true")
+
+	// Marker must be cleared after a successful ping so subsequent pings don't re-fire.
+	_, ok := service.syncFailedSinceLastPing.Load(rc.RemoteId)
+	assert.False(t, ok, "syncFailedSinceLastPing marker should be cleared after a successful ping")
+}
+
+// TestPingNow_NoSpuriousFire verifies that PingNow does NOT fire a connection-state-change
+// event when the remote stays online and no sync failure marker is set.
+func TestPingNow_NoSpuriousFire(t *testing.T) {
+	service, listener, rc := newPingTestService(t)
+
+	service.PingNow(rc)
+
+	calls, _ := listener.snapshot()
+	assert.Equal(t, 0, calls, "listener must not fire when remote stays online and no sync failure marker is set")
+}
+
+func checkRecent(millis int64, within int64) bool {
+	now := model.GetMillis()
+	return millis > now-within && millis < now+within
+}
+
+func hasRemoteID(remoteID string, remotes []*model.RemoteCluster) bool {
+	for _, r := range remotes {
+		if r.RemoteId == remoteID {
+			return true
+		}
+	}
+	return false
+}

server/platform/services/remotecluster/recv.go

@@ -0,0 +1,93 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"fmt"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+)
+
+// ReceiveIncomingMsg is called by the Rest API layer, or websocket layer (future), when a Remote Cluster
+// message is received.  Here we route the message to any topic listeners.
+// `rc` and `msg` cannot be nil.
+func (rcs *Service) ReceiveIncomingMsg(rc *model.RemoteCluster, msg model.RemoteClusterMsg) Response {
+	rcs.mux.RLock()
+	defer rcs.mux.RUnlock()
+
+	if metrics := rcs.server.GetMetrics(); metrics != nil {
+		metrics.IncrementRemoteClusterMsgReceivedCounter(rc.RemoteId)
+	}
+
+	rcSanitized := *rc
+	rcSanitized.Token = ""
+	rcSanitized.RemoteToken = ""
+
+	var response Response
+	response.Status = ResponseStatusOK
+
+	listeners := rcs.getTopicListeners(msg.Topic)
+
+	for _, l := range listeners {
+		if err := callback(l, msg, &rcSanitized, &response); err != nil {
+			rcs.server.Log().LogM(mlog.MlvlRemoteClusterServiceError, "Error from remote cluster message listener",
+				mlog.String("msg_id", msg.Id), mlog.String("topic", msg.Topic), mlog.String("remote", rc.DisplayName), mlog.Err(err))
+
+			response.Status = ResponseStatusFail
+			response.Err = err.Error()
+		}
+	}
+	return response
+}
+
+func callback(listener TopicListener, msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("%v", r)
+		}
+	}()
+	err = listener(msg, rc, resp)
+	return
+}
+
+// ReceiveInviteConfirmation is called by the Rest API layer when a Remote Cluster accepts an invitation from this
+// local cluster.
+func (rcs *Service) ReceiveInviteConfirmation(confirm model.RemoteClusterInvite) (*model.RemoteCluster, error) {
+	store := rcs.server.GetStore().RemoteCluster()
+
+	rc, err := store.Get(confirm.RemoteId, false)
+	if err != nil {
+		return nil, fmt.Errorf("cannot accept invite confirmation for remote %s: %w", confirm.RemoteId, err)
+	}
+
+	if rc.IsConfirmed() {
+		return nil, fmt.Errorf("cannot accept invite confirmation for remote %s: %w", confirm.RemoteId, RemoteClusterAlreadyConfirmedError)
+	}
+
+	rc.SiteURL = confirm.SiteURL
+	rc.RemoteToken = confirm.Token
+
+	// If the accepting cluster sent a RefreshedToken (its RemoteToken), set it as our Token
+	if confirm.Version >= 2 && confirm.RefreshedToken != "" {
+		if confirm.RefreshedToken == rc.Token {
+			return nil, fmt.Errorf("cannot accept invite confirmation for remote %s: RefreshedToken must be different from the original invite token", confirm.RemoteId)
+		}
+		rc.Token = confirm.RefreshedToken
+	} else {
+		// For older versions or if no RefreshedToken was provided, generate a new token
+		// to invalidate the original invite token and prevent reuse
+		rc.Token = model.NewId()
+	}
+
+	rcUpdated, err := store.Update(rc)
+	if err != nil {
+		return nil, fmt.Errorf("cannot apply invite confirmation for remote %s: %w", confirm.RemoteId, err)
+	}
+
+	// issue the first ping right away. The goroutine will exit when ping completes or PingTimeout exceeded.
+	go rcs.PingNow(rcUpdated)
+
+	return rcUpdated, nil
+}

server/platform/services/remotecluster/recv_test.go

@@ -0,0 +1,387 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
+)
+
+// TestReceiveInviteConfirmation_TokenInvalidation tests that the invite token is properly
+// invalidated after the first successful confirmation, preventing token reuse attacks.
+// This addresses the security vulnerability described in MM-65152.
+func TestReceiveInviteConfirmation_TokenInvalidation(t *testing.T) {
+	t.Run("Protocol v2+ with RefreshedToken - token is rotated", func(t *testing.T) {
+		// Setup
+		originalToken := model.NewId()
+		refreshedToken := model.NewId()
+		remoteId := model.NewId()
+
+		originalRC := &model.RemoteCluster{
+			RemoteId: remoteId,
+			Token:    originalToken,
+			SiteURL:  model.SiteURLPending + model.NewId(),
+			CreateAt: model.GetMillis(),
+		}
+
+		// Mock store
+		remoteClusterStoreMock := &mocks.RemoteClusterStore{}
+		remoteClusterStoreMock.On("Get", remoteId, false).Return(originalRC, nil)
+
+		// Capture the Update call to verify the token was rotated
+		var capturedRC *model.RemoteCluster
+		remoteClusterStoreMock.On("Update", mock.AnythingOfType("*model.RemoteCluster")).Run(func(args mock.Arguments) {
+			capturedRC = args.Get(0).(*model.RemoteCluster)
+		}).Return(func(rc *model.RemoteCluster) *model.RemoteCluster {
+			return rc
+		}, nil)
+
+		storeMock := &mocks.Store{}
+		storeMock.On("RemoteCluster").Return(remoteClusterStoreMock)
+
+		mockServer := newMockServerWithStore(t, storeMock)
+		mockApp := newMockApp(t, nil)
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		// Create invite confirmation with protocol v2+ (includes RefreshedToken)
+		confirm := model.RemoteClusterInvite{
+			RemoteId:       remoteId,
+			SiteURL:        "http://example.com",
+			Token:          model.NewId(),
+			RefreshedToken: refreshedToken,
+			Version:        3, // v3 protocol
+		}
+
+		// Execute
+		rcUpdated, err := service.ReceiveInviteConfirmation(confirm)
+
+		// Assert
+		require.NoError(t, err)
+		require.NotNil(t, rcUpdated)
+		require.NotNil(t, capturedRC, "Update should have been called")
+
+		assert.Equal(t, refreshedToken, capturedRC.Token, "Token should be rotated to RefreshedToken")
+		assert.NotEqual(t, originalToken, capturedRC.Token, "Original invite token should be invalidated")
+		assert.Equal(t, remoteId, capturedRC.RemoteId, "RemoteId should be preserved")
+
+		remoteClusterStoreMock.AssertExpectations(t)
+	})
+
+	t.Run("Protocol v1 or no RefreshedToken - token is regenerated", func(t *testing.T) {
+		// Setup
+		originalToken := model.NewId()
+		remoteId := model.NewId()
+
+		originalRC := &model.RemoteCluster{
+			RemoteId: remoteId,
+			Token:    originalToken,
+			SiteURL:  model.SiteURLPending + model.NewId(),
+			CreateAt: model.GetMillis(),
+		}
+
+		// Mock store
+		remoteClusterStoreMock := &mocks.RemoteClusterStore{}
+		remoteClusterStoreMock.On("Get", remoteId, false).Return(originalRC, nil)
+
+		// Capture what Update was called with for explicit assertions
+		var capturedRC *model.RemoteCluster
+		remoteClusterStoreMock.On("Update", mock.AnythingOfType("*model.RemoteCluster")).Run(func(args mock.Arguments) {
+			capturedRC = args.Get(0).(*model.RemoteCluster)
+		}).Return(func(rc *model.RemoteCluster) *model.RemoteCluster {
+			return rc
+		}, nil)
+
+		storeMock := &mocks.Store{}
+		storeMock.On("RemoteCluster").Return(remoteClusterStoreMock)
+
+		mockServer := newMockServerWithStore(t, storeMock)
+		mockApp := newMockApp(t, nil)
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		// Create invite confirmation WITHOUT RefreshedToken (protocol v1)
+		confirm := model.RemoteClusterInvite{
+			RemoteId:       remoteId,
+			SiteURL:        "http://example.com",
+			Token:          model.NewId(),
+			RefreshedToken: "", // No refreshed token
+			Version:        1,  // v1 protocol
+		}
+
+		// Execute
+		rcUpdated, err := service.ReceiveInviteConfirmation(confirm)
+
+		// Assert
+		require.NoError(t, err)
+		require.NotNil(t, rcUpdated)
+		require.NotNil(t, capturedRC, "Update should have been called")
+
+		assert.NotEmpty(t, capturedRC.Token, "Token should be set")
+		assert.NotEqual(t, originalToken, capturedRC.Token, "Original invite token must be invalidated")
+		assert.Len(t, capturedRC.Token, 26, "New token should be a valid ID (26 chars)")
+		assert.Equal(t, remoteId, capturedRC.RemoteId, "RemoteId should be preserved")
+
+		remoteClusterStoreMock.AssertExpectations(t)
+	})
+
+	t.Run("Already confirmed cluster - returns error", func(t *testing.T) {
+		// Setup - cluster is already confirmed (has a real SiteURL)
+		remoteId := model.NewId()
+		confirmedRC := &model.RemoteCluster{
+			RemoteId: remoteId,
+			Token:    model.NewId(),
+			SiteURL:  "http://already-confirmed.com", // NOT a pending URL
+			CreateAt: model.GetMillis(),
+		}
+
+		// Mock store
+		remoteClusterStoreMock := &mocks.RemoteClusterStore{}
+		remoteClusterStoreMock.On("Get", remoteId, false).Return(confirmedRC, nil)
+		// Update should NOT be called
+
+		storeMock := &mocks.Store{}
+		storeMock.On("RemoteCluster").Return(remoteClusterStoreMock)
+
+		mockServer := newMockServerWithStore(t, storeMock)
+		mockApp := newMockApp(t, nil)
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		confirm := model.RemoteClusterInvite{
+			RemoteId:       remoteId,
+			SiteURL:        "http://attacker.com",
+			Token:          model.NewId(),
+			RefreshedToken: model.NewId(),
+			Version:        3,
+		}
+
+		// Execute
+		rcUpdated, err := service.ReceiveInviteConfirmation(confirm)
+
+		// Assert
+		require.Error(t, err, "Should return error for already confirmed cluster")
+		assert.Nil(t, rcUpdated)
+		assert.Contains(t, err.Error(), "already been confirmed",
+			"Error should indicate cluster is already confirmed")
+
+		// Verify Update was NOT called (token reuse prevented)
+		remoteClusterStoreMock.AssertNotCalled(t, "Update", mock.Anything)
+	})
+
+	t.Run("Token reuse scenario - second confirmation with old token fails", func(t *testing.T) {
+		// This test simulates the security vulnerability:
+		// 1. First confirmation succeeds and rotates token
+		// 2. Attacker tries to reuse the old token
+		// 3. Second confirmation should fail because token was rotated
+
+		originalToken := model.NewId()
+		newToken := model.NewId()
+		remoteId := model.NewId()
+
+		// Initial state - unconfirmed cluster
+		originalRC := &model.RemoteCluster{
+			RemoteId: remoteId,
+			Token:    originalToken,
+			SiteURL:  model.SiteURLPending + model.NewId(),
+			CreateAt: model.GetMillis(),
+		}
+
+		// After first confirmation - token has been rotated
+		confirmedRC := &model.RemoteCluster{
+			RemoteId:    remoteId,
+			Token:       newToken, // Token was rotated
+			RemoteToken: model.NewId(),
+			SiteURL:     "http://legitimate.com",
+			CreateAt:    model.GetMillis(),
+		}
+
+		// Mock store
+		remoteClusterStoreMock := &mocks.RemoteClusterStore{}
+
+		// First call: return unconfirmed cluster
+		remoteClusterStoreMock.On("Get", remoteId, false).Return(originalRC, nil).Once()
+
+		// First update: rotate token
+		remoteClusterStoreMock.On("Update", mock.MatchedBy(func(rc *model.RemoteCluster) bool {
+			return rc.RemoteId == remoteId
+		})).Return(confirmedRC, nil).Once()
+
+		// Second call: return confirmed cluster with new token
+		remoteClusterStoreMock.On("Get", remoteId, false).Return(confirmedRC, nil).Once()
+
+		storeMock := &mocks.Store{}
+		storeMock.On("RemoteCluster").Return(remoteClusterStoreMock)
+
+		mockServer := newMockServerWithStore(t, storeMock)
+		mockApp := newMockApp(t, nil)
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		// First confirmation (legitimate)
+		confirm1 := model.RemoteClusterInvite{
+			RemoteId:       remoteId,
+			SiteURL:        "http://legitimate.com",
+			Token:          model.NewId(),
+			RefreshedToken: newToken,
+			Version:        3,
+		}
+
+		rcUpdated, err := service.ReceiveInviteConfirmation(confirm1)
+		require.NoError(t, err)
+		require.NotNil(t, rcUpdated)
+		assert.Equal(t, newToken, rcUpdated.Token, "Token should be rotated after first confirmation")
+
+		// Second confirmation attempt (attacker trying to reuse invite)
+		confirm2 := model.RemoteClusterInvite{
+			RemoteId:       remoteId,
+			SiteURL:        "http://attacker.com", // Attacker's malicious URL
+			Token:          originalToken,         // Reusing OLD token from original invite
+			RefreshedToken: model.NewId(),
+			Version:        3,
+		}
+
+		rcUpdated2, err := service.ReceiveInviteConfirmation(confirm2)
+
+		// Assert: Second confirmation should fail
+		require.Error(t, err, "Second confirmation should fail - cluster already confirmed")
+		assert.Nil(t, rcUpdated2, "No cluster should be returned on failed reuse attempt")
+		assert.Contains(t, err.Error(), "already been confirmed",
+			"Should indicate cluster already confirmed, preventing token reuse")
+
+		remoteClusterStoreMock.AssertExpectations(t)
+	})
+
+	t.Run("Protocol v2+ with RefreshedToken equal to original token - rejected (MM-67098)", func(t *testing.T) {
+		// Security: RefreshedToken must be different from the original invite token.
+		// If the remote sends back the same token, they did not actually refresh; reject.
+		originalToken := model.NewId()
+		remoteId := model.NewId()
+
+		originalRC := &model.RemoteCluster{
+			RemoteId: remoteId,
+			Token:    originalToken,
+			SiteURL:  model.SiteURLPending + model.NewId(),
+			CreateAt: model.GetMillis(),
+		}
+
+		remoteClusterStoreMock := &mocks.RemoteClusterStore{}
+		remoteClusterStoreMock.On("Get", remoteId, false).Return(originalRC, nil)
+		// Update must NOT be called when RefreshedToken == rc.Token
+
+		storeMock := &mocks.Store{}
+		storeMock.On("RemoteCluster").Return(remoteClusterStoreMock)
+
+		mockServer := newMockServerWithStore(t, storeMock)
+		mockApp := newMockApp(t, nil)
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		confirm := model.RemoteClusterInvite{
+			RemoteId:       remoteId,
+			SiteURL:        "http://example.com",
+			Token:          model.NewId(),
+			RefreshedToken: originalToken, // Same as rc.Token - invalid, must be different
+			Version:        3,
+		}
+
+		rcUpdated, err := service.ReceiveInviteConfirmation(confirm)
+
+		require.Error(t, err)
+		assert.Nil(t, rcUpdated)
+		assert.Contains(t, err.Error(), "RefreshedToken must be different from the original invite token")
+		remoteClusterStoreMock.AssertNotCalled(t, "Update", mock.Anything)
+	})
+}
+
+// TestReceiveInviteConfirmation_EdgeCases tests various edge cases
+func TestReceiveInviteConfirmation_EdgeCases(t *testing.T) {
+	t.Run("Non-existent remote ID", func(t *testing.T) {
+		remoteId := model.NewId()
+
+		remoteClusterStoreMock := &mocks.RemoteClusterStore{}
+		remoteClusterStoreMock.On("Get", remoteId, false).Return(nil, &model.AppError{
+			Message: "not found",
+		})
+
+		storeMock := &mocks.Store{}
+		storeMock.On("RemoteCluster").Return(remoteClusterStoreMock)
+
+		mockServer := newMockServerWithStore(t, storeMock)
+		mockApp := newMockApp(t, nil)
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		confirm := model.RemoteClusterInvite{
+			RemoteId: remoteId,
+			SiteURL:  "http://example.com",
+			Token:    model.NewId(),
+			Version:  3,
+		}
+
+		rcUpdated, err := service.ReceiveInviteConfirmation(confirm)
+
+		require.Error(t, err)
+		assert.Nil(t, rcUpdated)
+		assert.Contains(t, err.Error(), "cannot accept invite confirmation")
+	})
+
+	t.Run("Protocol v2+ with empty RefreshedToken - falls back to NewId", func(t *testing.T) {
+		originalToken := model.NewId()
+		remoteId := model.NewId()
+
+		originalRC := &model.RemoteCluster{
+			RemoteId: remoteId,
+			Token:    originalToken,
+			SiteURL:  model.SiteURLPending + model.NewId(),
+			CreateAt: model.GetMillis(),
+		}
+
+		remoteClusterStoreMock := &mocks.RemoteClusterStore{}
+		remoteClusterStoreMock.On("Get", remoteId, false).Return(originalRC, nil)
+
+		// Capture what Update was called with for explicit assertions
+		var capturedRC *model.RemoteCluster
+		remoteClusterStoreMock.On("Update", mock.AnythingOfType("*model.RemoteCluster")).Run(func(args mock.Arguments) {
+			capturedRC = args.Get(0).(*model.RemoteCluster)
+		}).Return(func(rc *model.RemoteCluster) *model.RemoteCluster {
+			return rc
+		}, nil)
+
+		storeMock := &mocks.Store{}
+		storeMock.On("RemoteCluster").Return(remoteClusterStoreMock)
+
+		mockServer := newMockServerWithStore(t, storeMock)
+		mockApp := newMockApp(t, nil)
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		// v3 protocol but RefreshedToken is empty
+		confirm := model.RemoteClusterInvite{
+			RemoteId:       remoteId,
+			SiteURL:        "http://example.com",
+			Token:          model.NewId(),
+			RefreshedToken: "", // Empty - should trigger fallback
+			Version:        3,
+		}
+
+		rcUpdated, err := service.ReceiveInviteConfirmation(confirm)
+
+		require.NoError(t, err)
+		require.NotNil(t, rcUpdated)
+		require.NotNil(t, capturedRC, "Update should have been called")
+
+		assert.NotEqual(t, originalToken, capturedRC.Token, "Original token should be invalidated")
+		assert.Len(t, capturedRC.Token, 26, "Should generate new token via fallback (26 chars)")
+		assert.Equal(t, remoteId, capturedRC.RemoteId, "RemoteId should be preserved")
+
+		remoteClusterStoreMock.AssertExpectations(t)
+	})
+}

server/platform/services/remotecluster/response.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"encoding/json"
+)
+
+// Response represents the bytes replied from a remote server when a message is sent.
+type Response struct {
+	Status  string          `json:"status"`
+	Err     string          `json:"err"`
+	Payload json.RawMessage `json:"payload"`
+}
+
+// IsSuccess returns true if the response status indicates success.
+func (r *Response) IsSuccess() bool {
+	return r.Status == ResponseStatusOK
+}
+
+// SetPayload serializes an arbitrary struct as a RawMessage.
+func (r *Response) SetPayload(v any) error {
+	raw, err := json.Marshal(v)
+	if err != nil {
+		return err
+	}
+	r.Payload = raw
+	return nil
+}

server/platform/services/remotecluster/send.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"context"
+	"hash/fnv"
+)
+
+// enqueueTask adds a task to one of the send channels based on remoteId.
+//
+// There are a number  of send channels (`MaxConcurrentSends`) to allow for sending to multiple
+// remotes concurrently, while preserving message order for each remote.
+func (rcs *Service) enqueueTask(ctx context.Context, remoteId string, task any) error {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	h := hash(remoteId)
+	idx := h % uint32(len(rcs.send))
+
+	select {
+	case rcs.send[idx] <- task:
+		return nil
+	case <-ctx.Done():
+		return NewBufferFullError(cap(rcs.send))
+	}
+}
+
+func hash(s string) uint32 {
+	h := fnv.New32a()
+	h.Write([]byte(s))
+	return h.Sum32()
+}
+
+// sendLoop is called by each goroutine created for the send pool and waits for sendTask's until the
+// done channel is signalled.
+//
+// Each goroutine in the pool is assigned a specific channel, and tasks are placed in the
+// channel corresponding to the remoteId.
+func (rcs *Service) sendLoop(idx int, done chan struct{}) {
+	for {
+		select {
+		case t := <-rcs.send[idx]:
+			switch task := t.(type) {
+			case sendMsgTask:
+				rcs.sendMsg(task)
+			case sendFileTask:
+				rcs.sendFile(task)
+			case sendProfileImageTask:
+				rcs.sendProfileImage(task)
+			}
+		case <-done:
+			return
+		}
+	}
+}

server/platform/services/remotecluster/send_test.go

@@ -0,0 +1,209 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/wiggin77/merror"
+
+	"github.com/mattermost/mattermost/server/public/model"
+)
+
+const (
+	TestTopics  = " share incident "
+	TestTopic   = "share"
+	NumRemotes  = 50
+	NoteContent = "Woot!!"
+)
+
+type testPayload struct {
+	Note string `json:"note"`
+}
+
+func TestBroadcastMsg(t *testing.T) {
+	msgId := model.NewId()
+
+	t.Run("No error", func(t *testing.T) {
+		var countCallbacks atomic.Int32
+		var countWebReq atomic.Int32
+		merr := merror.New()
+
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			defer func() {
+				w.WriteHeader(200)
+				var resp Response
+				b, errMarshall := json.Marshal(&resp)
+				if errMarshall != nil {
+					merr.Append(errMarshall)
+					return
+				}
+				w.Write(b)
+			}()
+
+			countWebReq.Add(1)
+
+			var frame model.RemoteClusterFrame
+			jsonErr := json.NewDecoder(r.Body).Decode(&frame)
+			if jsonErr != nil {
+				merr.Append(jsonErr)
+				return
+			}
+			if len(frame.Msg.Payload) == 0 {
+				merr.Append(fmt.Errorf("webrequest missing Msg.Payload"))
+			}
+			if msgId != frame.Msg.Id {
+				merr.Append(fmt.Errorf("webrequest msgId expected %s, got %s", msgId, frame.Msg.Id))
+				return
+			}
+
+			note := testPayload{}
+			err := json.Unmarshal(frame.Msg.Payload, &note)
+			if err != nil {
+				merr.Append(err)
+				return
+			}
+			if note.Note != NoteContent {
+				merr.Append(fmt.Errorf("webrequest payload expected %s, got %s", NoteContent, note.Note))
+				return
+			}
+		}))
+		defer ts.Close()
+
+		mockServer := newMockServer(t, makeRemoteClusters(NumRemotes, ts.URL, false))
+		mockApp := newMockApp(t, nil)
+
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+		service.disablePing = true
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer service.Shutdown()
+
+		wg := &sync.WaitGroup{}
+		wg.Add(NumRemotes)
+
+		msg := makeRemoteClusterMsg(msgId, NoteContent)
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
+		defer cancel()
+
+		err = service.BroadcastMsg(ctx, msg, func(msg model.RemoteClusterMsg, remote *model.RemoteCluster, resp *Response, err error) {
+			defer wg.Done()
+			countCallbacks.Add(1)
+
+			if err != nil {
+				merr.Append(err)
+			}
+			if msgId != msg.Id {
+				merr.Append(fmt.Errorf("result callback msgId expected %s, got %s", msgId, msg.Id))
+			}
+
+			var note testPayload
+			err2 := json.Unmarshal(msg.Payload, &note)
+			if err2 != nil {
+				merr.Append(fmt.Errorf("unmarshal payload error: %w", err2))
+				return
+			}
+			if note.Note != NoteContent {
+				merr.Append(fmt.Errorf("compare payload failed: expected '%s', got '%s'", NoteContent, note))
+			}
+		})
+		assert.NoError(t, err)
+
+		wg.Wait()
+
+		assert.NoError(t, merr.ErrorOrNil())
+
+		assert.Equal(t, int32(NumRemotes), countCallbacks.Load())
+		assert.Equal(t, int32(NumRemotes), countWebReq.Load())
+		t.Logf("%d callbacks counted;  %d web requests counted;  %d expected",
+			countCallbacks.Load(), countWebReq.Load(), NumRemotes)
+	})
+
+	t.Run("HTTP error", func(t *testing.T) {
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(500)
+		}))
+		defer ts.Close()
+
+		mockServer := newMockServer(t, makeRemoteClusters(NumRemotes, ts.URL, false))
+		mockApp := newMockApp(t, nil)
+
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+		service.disablePing = true
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer service.Shutdown()
+
+		msg := makeRemoteClusterMsg(msgId, NoteContent)
+		var countCallbacks atomic.Int32
+		var countErrors atomic.Int32
+		wg := &sync.WaitGroup{}
+		wg.Add(NumRemotes)
+
+		err = service.BroadcastMsg(context.Background(), msg, func(msg model.RemoteClusterMsg, remote *model.RemoteCluster, resp *Response, err error) {
+			defer wg.Done()
+			countCallbacks.Add(1)
+			if err != nil {
+				countErrors.Add(1)
+			}
+		})
+		assert.NoError(t, err)
+
+		wg.Wait()
+
+		assert.Equal(t, int32(NumRemotes), countCallbacks.Load())
+		assert.Equal(t, int32(NumRemotes), countErrors.Load())
+	})
+}
+
+func makeRemoteClusters(num int, siteURL string, isPlugin bool) []*model.RemoteCluster {
+	var remotes []*model.RemoteCluster
+	for i := range num {
+		rc := makeRemoteCluster(fmt.Sprintf("test cluster %d", i+1), siteURL, TestTopics)
+		if isPlugin {
+			rc.PluginID = model.NewId()
+		}
+		remotes = append(remotes, rc)
+	}
+	return remotes
+}
+
+func makeRemoteCluster(name string, siteURL string, topics string) *model.RemoteCluster {
+	return &model.RemoteCluster{
+		RemoteId:   model.NewId(),
+		Name:       name,
+		SiteURL:    siteURL,
+		Token:      model.NewId(),
+		Topics:     topics,
+		CreateAt:   model.GetMillis(),
+		LastPingAt: model.GetMillis(),
+		CreatorId:  model.NewId(),
+	}
+}
+
+func makeRemoteClusterMsg(id string, note string) model.RemoteClusterMsg {
+	payload := testPayload{Note: note}
+	raw, _ := json.Marshal(payload)
+
+	return model.RemoteClusterMsg{
+		Id:       id,
+		Topic:    TestTopic,
+		CreateAt: model.GetMillis(),
+		Payload:  raw}
+}

server/platform/services/remotecluster/sendfile.go

@@ -0,0 +1,143 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+)
+
+type SendFileResultFunc func(us *model.UploadSession, rc *model.RemoteCluster, resp *Response, err error)
+
+type sendFileTask struct {
+	rc *model.RemoteCluster
+	us *model.UploadSession
+	fi *model.FileInfo
+	rp ReaderProvider
+	f  SendFileResultFunc
+}
+
+type ReaderProvider interface {
+	FileReader(path string) (filestore.ReadCloseSeeker, *model.AppError)
+}
+
+// SendFile asynchronously sends a file to a remote cluster.
+//
+// `ctx` determines behaviour when the outbound queue is full. A timeout or deadline context will return a
+// BufferFullError if the task cannot be enqueued before the timeout. A background context will block indefinitely.
+//
+// Nil or error return indicates success or failure of task enqueue only.
+//
+// An optional callback can be provided that receives the response from the remote cluster. The `err` provided to the
+// callback is regarding file delivery only. The `resp` contains the decoded bytes returned from the remote.
+// If a callback is provided it should return quickly.
+func (rcs *Service) SendFile(ctx context.Context, us *model.UploadSession, fi *model.FileInfo, rc *model.RemoteCluster, rp ReaderProvider, f SendFileResultFunc) error {
+	task := sendFileTask{
+		rc: rc,
+		us: us,
+		fi: fi,
+		rp: rp,
+		f:  f,
+	}
+	return rcs.enqueueTask(ctx, rc.RemoteId, task)
+}
+
+// sendFile is called when a sendFileTask is popped from the send channel.
+func (rcs *Service) sendFile(task sendFileTask) {
+	fi, err := rcs.sendFileToRemote(SendTimeout, task)
+	var response Response
+
+	if err != nil {
+		rcs.server.Log().LogM(mlog.MlvlRemoteClusterServiceError, "Remote Cluster send file failed",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("upload_id", task.us.Id),
+			mlog.Err(err),
+		)
+		response.Status = ResponseStatusFail
+		response.Err = err.Error()
+	} else {
+		rcs.server.Log().Log(mlog.LvlRemoteClusterServiceDebug, "Remote Cluster file sent successfully",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("upload_id", task.us.Id),
+		)
+		response.Status = ResponseStatusOK
+		response.SetPayload(fi)
+	}
+
+	// If callback provided then call it with the results.
+	if task.f != nil {
+		task.f(task.us, task.rc, &response, err)
+	}
+}
+
+func (rcs *Service) sendFileToRemote(timeout time.Duration, task sendFileTask) (*model.FileInfo, error) {
+	start := time.Now()
+	defer func() {
+		if metrics := rcs.server.GetMetrics(); metrics != nil {
+			metrics.ObserveSharedChannelsSyncSendStepDuration(task.rc.RemoteId, "Attachments", time.Since(start).Seconds())
+		}
+	}()
+
+	rcs.server.Log().Log(mlog.LvlRemoteClusterServiceDebug, "sending file to remote...",
+		mlog.String("remote", task.rc.DisplayName),
+		mlog.String("upload_id", task.us.Id),
+		mlog.String("file_path", task.us.Path),
+	)
+
+	r, appErr := task.rp.FileReader(task.fi.Path) // get Reader for the file
+	if appErr != nil {
+		return nil, fmt.Errorf("error opening file while sending file to remote %s: %w", task.rc.RemoteId, appErr)
+	}
+	defer r.Close()
+
+	u, err := url.Parse(task.rc.SiteURL)
+	if err != nil {
+		return nil, fmt.Errorf("invalid siteURL while sending file to remote %s: %w", task.rc.RemoteId, err)
+	}
+	u.Path = path.Join(u.Path, model.APIURLSuffix, "remotecluster", "upload", task.us.Id)
+
+	req, err := http.NewRequest("POST", u.String(), r)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set(model.HeaderRemoteclusterId, task.rc.RemoteId)
+	req.Header.Set(model.HeaderRemoteclusterToken, task.rc.RemoteToken)
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	resp, err := rcs.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected response: %d - %s", resp.StatusCode, resp.Status)
+	}
+
+	// body should be a FileInfo
+	var fi model.FileInfo
+	if err := json.Unmarshal(body, &fi); err != nil {
+		return nil, fmt.Errorf("unexpected response body: %w", err)
+	}
+
+	return &fi, nil
+}

server/platform/services/remotecluster/sendmsg.go

@@ -0,0 +1,175 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"time"
+
+	"github.com/wiggin77/merror"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+)
+
+type SendMsgResultFunc func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response, err error)
+
+type sendMsgTask struct {
+	rc  *model.RemoteCluster
+	msg model.RemoteClusterMsg
+	f   SendMsgResultFunc
+}
+
+// BroadcastMsg asynchronously sends a message to all remote clusters interested in the message's topic.
+//
+// `ctx` determines behaviour when the outbound queue is full. A timeout or deadline context will return a
+// BufferFullError if the message cannot be enqueued before the timeout. A background context will block indefinitely.
+//
+// An optional callback can be provided that receives the success or fail result of sending to each remote cluster.
+// Success or fail is regarding message delivery only.  If a callback is provided it should return quickly.
+func (rcs *Service) BroadcastMsg(ctx context.Context, msg model.RemoteClusterMsg, f SendMsgResultFunc) error {
+	// get list of interested remotes.
+	filter := model.RemoteClusterQueryFilter{
+		Topic: msg.Topic,
+	}
+	list, err := rcs.server.GetStore().RemoteCluster().GetAll(0, 999999, filter)
+	if err != nil {
+		return err
+	}
+
+	errs := merror.New()
+
+	for _, rc := range list {
+		if err := rcs.SendMsg(ctx, msg, rc, f); err != nil {
+			errs.Append(err)
+		}
+	}
+	return errs.ErrorOrNil()
+}
+
+// SendMsg asynchronously sends a message to a remote cluster.
+//
+// `ctx` determines behaviour when the outbound queue is full. A timeout or deadline context will return a
+// BufferFullError if the message cannot be enqueued before the timeout. A background context will block indefinitely.
+//
+// Nil or error return indicates success or failure of message enqueue only.
+//
+// An optional callback can be provided that receives the response from the remote cluster. The `err` provided to the
+// callback is regarding response decoding only. The `resp` contains the decoded bytes returned from the remote.
+// If a callback is provided it should return quickly.
+func (rcs *Service) SendMsg(ctx context.Context, msg model.RemoteClusterMsg, rc *model.RemoteCluster, f SendMsgResultFunc) error {
+	task := sendMsgTask{
+		rc:  rc,
+		msg: msg,
+		f:   f,
+	}
+	return rcs.enqueueTask(ctx, rc.RemoteId, task)
+}
+
+// sendMsg is called when a sendMsgTask is popped from the send channel.
+func (rcs *Service) sendMsg(task sendMsgTask) {
+	var errResp error
+	var response Response
+
+	// Ensure a panic from the callback does not exit the pool goroutine.
+	defer func() {
+		if errResp != nil {
+			response.Err = errResp.Error()
+		}
+
+		// If callback provided then call it with the results.
+		if task.f != nil {
+			task.f(task.msg, task.rc, &response, errResp)
+		}
+	}()
+
+	frame := &model.RemoteClusterFrame{
+		RemoteId: task.rc.RemoteId,
+		Msg:      task.msg,
+	}
+
+	u, err := url.Parse(task.rc.SiteURL)
+	if err != nil {
+		rcs.server.Log().LogM(mlog.MlvlRemoteClusterServiceError, "Invalid siteURL while sending message to remote",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("msg_id", task.msg.Id),
+			mlog.Err(err),
+		)
+		errResp = err
+		return
+	}
+	u.Path = path.Join(u.Path, SendMsgURL)
+
+	respJSON, err := rcs.sendFrameToRemote(SendTimeout, task.rc, frame, u.String())
+
+	if err != nil {
+		rcs.server.Log().LogM(mlog.MlvlRemoteClusterServiceError, "Remote Cluster send message failed",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("msg_id", task.msg.Id),
+			mlog.Err(err),
+		)
+		errResp = err
+	} else {
+		rcs.server.Log().Log(mlog.LvlRemoteClusterServiceDebug, "Remote Cluster message sent successfully",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("msg_id", task.msg.Id),
+		)
+
+		if err = json.Unmarshal(respJSON, &response); err != nil {
+			rcs.server.Log().Error("Invalid response sending message to remote cluster",
+				mlog.String("remote", task.rc.DisplayName),
+				mlog.Err(err),
+			)
+			errResp = err
+		}
+	}
+}
+
+func (rcs *Service) sendFrameToRemote(timeout time.Duration, rc *model.RemoteCluster, frame *model.RemoteClusterFrame, url string) ([]byte, error) {
+	body, err := json.Marshal(frame)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	req, err := http.NewRequest("POST", url, bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set(model.HeaderRemoteclusterId, rc.RemoteId)
+	req.Header.Set(model.HeaderRemoteclusterToken, rc.RemoteToken)
+
+	resp, err := rcs.httpClient.Do(req.WithContext(ctx))
+	if metrics := rcs.server.GetMetrics(); metrics != nil {
+		if err != nil || resp.StatusCode != http.StatusOK {
+			metrics.IncrementRemoteClusterMsgErrorsCounter(frame.RemoteId, os.IsTimeout(err))
+		} else {
+			metrics.IncrementRemoteClusterMsgSentCounter(frame.RemoteId)
+		}
+	}
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	body, err = io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return body, fmt.Errorf("unexpected response: %d - %s", resp.StatusCode, resp.Status)
+	}
+	return body, nil
+}

server/platform/services/remotecluster/sendprofileImage.go

@@ -0,0 +1,153 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"path"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+)
+
+type SendProfileImageResultFunc func(userId string, rc *model.RemoteCluster, resp *Response, err error)
+
+type sendProfileImageTask struct {
+	rc       *model.RemoteCluster
+	userID   string
+	provider ProfileImageProvider
+	f        SendProfileImageResultFunc
+}
+
+type ProfileImageProvider interface {
+	GetProfileImage(user *model.User) ([]byte, bool, *model.AppError)
+}
+
+// SendProfileImage asynchronously sends a user's profile image to a remote cluster.
+//
+// `ctx` determines behaviour when the outbound queue is full. A timeout or deadline context will return a
+// BufferFullError if the task cannot be enqueued before the timeout. A background context will block indefinitely.
+//
+// Nil or error return indicates success or failure of task enqueue only.
+//
+// An optional callback can be provided that receives the response from the remote cluster. The `err` provided to the
+// callback is regarding image delivery only. The `resp` contains the decoded bytes returned from the remote.
+// If a callback is provided it should return quickly.
+func (rcs *Service) SendProfileImage(ctx context.Context, userID string, rc *model.RemoteCluster, provider ProfileImageProvider, f SendProfileImageResultFunc) error {
+	task := sendProfileImageTask{
+		rc:       rc,
+		userID:   userID,
+		provider: provider,
+		f:        f,
+	}
+	return rcs.enqueueTask(ctx, rc.RemoteId, task)
+}
+
+// sendProfileImage is called when a sendProfileImageTask is popped from the send channel.
+func (rcs *Service) sendProfileImage(task sendProfileImageTask) {
+	err := rcs.sendProfileImageToRemote(SendTimeout, task)
+	var response Response
+
+	if err != nil {
+		rcs.server.Log().LogM(mlog.MlvlRemoteClusterServiceWarn, "Remote Cluster send profile image failed",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("user_id", task.userID),
+			mlog.Err(err),
+		)
+		response.Status = ResponseStatusFail
+		response.Err = err.Error()
+	} else {
+		rcs.server.Log().Log(mlog.LvlRemoteClusterServiceDebug, "Remote Cluster profile image sent successfully",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("user_id", task.userID),
+		)
+		response.Status = ResponseStatusOK
+	}
+
+	// If callback provided then call it with the results.
+	if task.f != nil {
+		task.f(task.userID, task.rc, &response, err)
+	}
+}
+
+func (rcs *Service) sendProfileImageToRemote(timeout time.Duration, task sendProfileImageTask) error {
+	start := time.Now()
+	defer func() {
+		if metrics := rcs.server.GetMetrics(); metrics != nil {
+			metrics.ObserveSharedChannelsSyncSendStepDuration(task.rc.RemoteId, "ProfileImages", time.Since(start).Seconds())
+		}
+	}()
+
+	rcs.server.Log().Log(mlog.LvlRemoteClusterServiceDebug, "sending profile image to remote...",
+		mlog.String("remote", task.rc.DisplayName),
+		mlog.String("user_id", task.userID),
+	)
+
+	user, err := rcs.server.GetStore().User().Get(request.EmptyContext(rcs.server.Log()), task.userID)
+	if err != nil {
+		return fmt.Errorf("error fetching user while sending profile image to remote %s: %w", task.rc.RemoteId, err)
+	}
+
+	img, _, appErr := task.provider.GetProfileImage(user) // get Reader for the file
+	if appErr != nil {
+		return fmt.Errorf("error fetching profile image for user (%s) while sending to remote %s: %w", task.userID, task.rc.RemoteId, appErr)
+	}
+
+	u, err := url.Parse(task.rc.SiteURL)
+	if err != nil {
+		return fmt.Errorf("invalid siteURL while sending file to remote %s: %w", task.rc.RemoteId, err)
+	}
+	u.Path = path.Join(u.Path, model.APIURLSuffix, "remotecluster", task.userID, "image")
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	part, err := writer.CreateFormFile("image", "profile.png")
+	if err != nil {
+		return err
+	}
+
+	if _, err = io.Copy(part, bytes.NewBuffer(img)); err != nil {
+		return err
+	}
+
+	if err = writer.Close(); err != nil {
+		return err
+	}
+
+	req, err := http.NewRequest("POST", u.String(), body)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+	req.Header.Set(model.HeaderRemoteclusterId, task.rc.RemoteId)
+	req.Header.Set(model.HeaderRemoteclusterToken, task.rc.RemoteToken)
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	resp, err := rcs.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	_, err = io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("unexpected response: %d - %s", resp.StatusCode, resp.Status)
+	}
+	return nil
+}

server/platform/services/remotecluster/sendprofileImage_test.go

@@ -0,0 +1,186 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"bytes"
+	"image"
+	"image/color"
+	"image/png"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+)
+
+const (
+	imageWidth  = 128
+	imageHeight = 128
+)
+
+func TestService_sendProfileImageToRemote(t *testing.T) {
+	shouldError := &flag{}
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer io.Copy(io.Discard, r.Body)
+
+		if shouldError.get() {
+			w.WriteHeader(http.StatusInternalServerError)
+			resp := make(map[string]string)
+			resp[model.STATUS] = model.StatusFail
+			w.Write([]byte(model.MapToJSON(resp)))
+			return
+		}
+
+		status := model.StatusOk
+		defer func(s *string) {
+			if *s != model.StatusOk {
+				w.WriteHeader(http.StatusInternalServerError)
+			}
+			resp := make(map[string]string)
+			resp[model.STATUS] = *s
+			w.Write([]byte(model.MapToJSON(resp)))
+		}(&status)
+
+		if err := r.ParseMultipartForm(1024 * 1024); err != nil {
+			status = model.StatusFail
+			assert.Fail(t, "connect parse multipart form", err)
+			return
+		}
+		m := r.MultipartForm
+		if m == nil {
+			status = model.StatusFail
+			assert.Fail(t, "multipart form missing")
+			return
+		}
+
+		imageArray, ok := m.File["image"]
+		if !ok || len(imageArray) != 1 {
+			status = model.StatusFail
+			assert.Fail(t, "image missing")
+			return
+		}
+
+		imageData := imageArray[0]
+		file, err := imageData.Open()
+		if err != nil {
+			status = model.StatusFail
+			assert.Fail(t, "cannot open multipart form file")
+			return
+		}
+		defer file.Close()
+
+		img, err := png.Decode(file)
+		if err != nil || imageWidth != img.Bounds().Max.X || imageHeight != img.Bounds().Max.Y {
+			status = model.StatusFail
+			assert.Fail(t, "cannot decode png", err)
+			return
+		}
+	}))
+	defer ts.Close()
+
+	rc := makeRemoteCluster("remote_test_profile_image", ts.URL, TestTopics)
+
+	user := &model.User{
+		Id:       model.NewId(),
+		RemoteId: new(rc.RemoteId),
+	}
+
+	provider := testImageProvider{}
+
+	mockServer := newMockServer(t, makeRemoteClusters(NumRemotes, ts.URL, false))
+	mockServer.SetUser(user)
+
+	mockApp := newMockApp(t, nil)
+
+	service, err := NewRemoteClusterService(mockServer, mockApp)
+	require.NoError(t, err)
+	service.disablePing = true
+
+	err = service.Start()
+	require.NoError(t, err)
+	defer service.Shutdown()
+
+	t.Run("Server response 200", func(t *testing.T) {
+		shouldError.set(false)
+
+		resultFunc := func(userId string, rc *model.RemoteCluster, resp *Response, err error) {
+			assert.Equal(t, user.Id, userId, "user ids should match")
+			assert.NoError(t, err)
+			assert.True(t, resp.IsSuccess())
+		}
+
+		task := sendProfileImageTask{
+			rc:       rc,
+			userID:   user.Id,
+			provider: provider,
+			f:        resultFunc,
+		}
+
+		err := service.sendProfileImageToRemote(time.Second*15, task)
+		assert.NoError(t, err, "request should not error")
+	})
+
+	t.Run("Server response 500", func(t *testing.T) {
+		shouldError.set(true)
+
+		resultFunc := func(userId string, rc *model.RemoteCluster, resp *Response, err error) {
+			assert.Equal(t, user.Id, userId, "user ids should match")
+			assert.False(t, resp.IsSuccess())
+		}
+
+		task := sendProfileImageTask{
+			rc:       rc,
+			userID:   user.Id,
+			provider: provider,
+			f:        resultFunc,
+		}
+
+		err := service.sendProfileImageToRemote(time.Second*15, task)
+		assert.Error(t, err, "request should error")
+	})
+}
+
+type testImageProvider struct {
+}
+
+func (tip testImageProvider) GetProfileImage(user *model.User) ([]byte, bool, *model.AppError) {
+	img := image.NewRGBA(image.Rectangle{image.Point{0, 0}, image.Point{imageWidth, imageHeight}})
+	red := color.RGBA{255, 50, 50, 0xff}
+
+	for x := range imageWidth {
+		for y := range imageHeight {
+			img.Set(x, y, red)
+		}
+	}
+
+	buf := &bytes.Buffer{}
+	png.Encode(buf, img)
+
+	return buf.Bytes(), true, nil
+}
+
+type flag struct {
+	mux sync.RWMutex
+	b   bool
+}
+
+func (f *flag) get() bool {
+	f.mux.RLock()
+	defer f.mux.RUnlock()
+	return f.b
+}
+
+func (f *flag) set(b bool) {
+	f.mux.Lock()
+	defer f.mux.Unlock()
+	f.b = b
+}

server/platform/services/remotecluster/service.go

@@ -0,0 +1,312 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/mattermost/mattermost/server/v8/einterfaces"
+)
+
+const (
+	SendChanBuffer                = 50
+	RecvChanBuffer                = 50
+	ResultsChanBuffer             = 50
+	ResultQueueDrainTimeoutMillis = 10000
+	MaxConcurrentSends            = 10
+	SendMsgURL                    = "api/v4/remotecluster/msg"
+	SendTimeout                   = time.Minute
+	SendFileTimeout               = time.Minute * 5
+	PingURL                       = "api/v4/remotecluster/ping"
+	// Raising PingFreq also raises the transport's IdleConnTimeout (PingFreq / 2),
+	// which must stay below peer idle timeouts (60s default). See MM-69982.
+	PingFreq           = time.Minute
+	PingTimeout        = time.Second * 15
+	ConfirmInviteURL   = "api/v4/remotecluster/confirm_invite"
+	InvitationTopic    = "invitation"
+	PingTopic          = "ping"
+	ResponseStatusOK   = model.StatusOk
+	ResponseStatusFail = model.StatusFail
+	InviteExpiresAfter = time.Hour * 48
+)
+
+type ServerIface interface {
+	Config() *model.Config
+	IsLeader() bool
+	AddClusterLeaderChangedListener(listener func()) string
+	RemoveClusterLeaderChangedListener(id string)
+	GetStore() store.Store
+	Log() *mlog.Logger
+	GetMetrics() einterfaces.MetricsInterface
+}
+
+type AppIface interface {
+	OnSharedChannelsPing(rc *model.RemoteCluster) bool
+}
+
+// RemoteClusterServiceIFace is used to allow mocking where a remote cluster service is used (for testing).
+// Unfortunately it lives here because the shared channel service, app layer, and server interface all need it.
+// Putting it in app layer means shared channel service must import app package.
+type RemoteClusterServiceIFace interface {
+	Shutdown() error
+	Start() error
+	Active() bool
+	AddTopicListener(topic string, listener TopicListener) string
+	RemoveTopicListener(listenerId string)
+	AddConnectionStateListener(listener ConnectionStateListener) string
+	RemoveConnectionStateListener(listenerId string)
+	SendMsg(ctx context.Context, msg model.RemoteClusterMsg, rc *model.RemoteCluster, f SendMsgResultFunc) error
+	SendFile(ctx context.Context, us *model.UploadSession, fi *model.FileInfo, rc *model.RemoteCluster, rp ReaderProvider, f SendFileResultFunc) error
+	SendProfileImage(ctx context.Context, userID string, rc *model.RemoteCluster, provider ProfileImageProvider, f SendProfileImageResultFunc) error
+	AcceptInvitation(invite *model.RemoteClusterInvite, name string, displayName string, creatorId string, siteURL string, defaultTeamId string) (*model.RemoteCluster, error)
+	ReceiveIncomingMsg(rc *model.RemoteCluster, msg model.RemoteClusterMsg) Response
+	ReceiveInviteConfirmation(invite model.RemoteClusterInvite) (*model.RemoteCluster, error)
+	PingNow(rc *model.RemoteCluster)
+	NotifySyncFailed(remoteId string)
+}
+
+// TopicListener is a callback signature used to listen for incoming messages for
+// a specific topic.
+type TopicListener func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response) error
+
+// ConnectionStateListener is used to listen to remote cluster connection state changes.
+type ConnectionStateListener func(rc *model.RemoteCluster, online bool)
+
+// Service provides inter-cluster communication via topic based messages. In product these are called "Secured Connections".
+type Service struct {
+	server      ServerIface
+	app         AppIface
+	httpClient  *http.Client
+	send        []chan any
+	done        chan struct{} // signals sendLoop workers to stop; lifecycle: Start -> Shutdown
+	active      atomic.Bool   // true after Start(), false after Shutdown()
+	disablePing bool          // when true, pingStart skips launching the ping loop; for testing
+
+	syncFailedSinceLastPing sync.Map // set of remoteIds (string) with a pending sync failure; presence is the marker
+
+	// everything below guarded by `mux`
+	mux                      sync.RWMutex
+	leaderListenerId         string
+	topicListeners           map[string]map[string]TopicListener // maps topic id to a map of listenerid->listener
+	connectionStateListeners map[string]ConnectionStateListener  // maps listener id to listener
+	pingDone                 chan struct{}                       // signals ping workers to stop; lifecycle: pingStart -> pingStop
+	pingFreq                 time.Duration
+}
+
+// NotifySyncFailed records that a sync has failed for the given remote. On the next
+// successful ping, PingNow will fire a connection-state-change event so the shared
+// channel service re-syncs any affected channels.
+func (rcs *Service) NotifySyncFailed(remoteId string) {
+	rcs.syncFailedSinceLastPing.Store(remoteId, struct{}{})
+}
+
+// NewRemoteClusterService creates a RemoteClusterService instance. In product this is called a "Secured Connection".
+func NewRemoteClusterService(server ServerIface, app AppIface) (*Service, error) {
+	transport := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+		ForceAttemptHTTP2:   true,
+		MaxIdleConns:        200,
+		MaxIdleConnsPerHost: 2,
+		// Must stay strictly below PingFreq so the pool always discards a connection
+		// before the next ping reuses it. Otherwise pings race the peer reaping its own
+		// idle keep-alive connections (ServiceSettings.IdleTimeout, default 60s) and fail
+		// intermittently with stale-connection errors. See MM-69982.
+		IdleConnTimeout:       PingFreq / 2,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		DisableCompression:    false,
+	}
+
+	client := &http.Client{
+		Transport: transport,
+		Timeout:   SendTimeout,
+	}
+
+	service := &Service{
+		server:                   server,
+		app:                      app,
+		httpClient:               client,
+		topicListeners:           make(map[string]map[string]TopicListener),
+		connectionStateListeners: make(map[string]ConnectionStateListener),
+	}
+
+	service.send = make([]chan any, MaxConcurrentSends)
+	for i := range service.send {
+		service.send[i] = make(chan any, SendChanBuffer)
+	}
+	service.pingFreq = PingFreq
+
+	return service, nil
+}
+
+// Start is called by the server on server start-up.
+func (rcs *Service) Start() error {
+	if !rcs.active.CompareAndSwap(false, true) {
+		return nil
+	}
+	rcs.done = make(chan struct{})
+	for i := range rcs.send {
+		go rcs.sendLoop(i, rcs.done)
+	}
+
+	rcs.mux.Lock()
+	rcs.leaderListenerId = rcs.server.AddClusterLeaderChangedListener(rcs.onClusterLeaderChange)
+	rcs.mux.Unlock()
+
+	rcs.onClusterLeaderChange()
+
+	return nil
+}
+
+// Shutdown is called by the server on server shutdown.
+func (rcs *Service) Shutdown() error {
+	if !rcs.active.CompareAndSwap(true, false) {
+		return nil
+	}
+	rcs.mux.Lock()
+	id := rcs.leaderListenerId
+	rcs.mux.Unlock()
+
+	rcs.server.RemoveClusterLeaderChangedListener(id)
+	rcs.pingStop()
+	close(rcs.done)
+	return nil
+}
+
+// Active returns true if this instance of the remote cluster service has been started.
+func (rcs *Service) Active() bool {
+	return rcs.active.Load()
+}
+
+// GetPingFreq gets the frequency of pings to each remote.
+func (rcs *Service) GetPingFreq() time.Duration {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+	return rcs.pingFreq
+}
+
+// SetPingFreq sets the frequency of pings to each remote. Defaults to `PingFreq`.
+// This is typically used to set a higher frequency for testing.
+func (rcs *Service) SetPingFreq(freq time.Duration) {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+	rcs.pingFreq = freq
+}
+
+// AddTopicListener registers a callback
+func (rcs *Service) AddTopicListener(topic string, listener TopicListener) string {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+
+	id := model.NewId()
+
+	listeners, ok := rcs.topicListeners[topic]
+	if !ok || listeners == nil {
+		rcs.topicListeners[topic] = make(map[string]TopicListener)
+	}
+	rcs.topicListeners[topic][id] = listener
+	return id
+}
+
+func (rcs *Service) RemoveTopicListener(listenerId string) {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+
+	for topic, listeners := range rcs.topicListeners {
+		if _, ok := listeners[listenerId]; ok {
+			delete(listeners, listenerId)
+			if len(listeners) == 0 {
+				delete(rcs.topicListeners, topic)
+			}
+			break
+		}
+	}
+}
+
+func (rcs *Service) getTopicListeners(topic string) []TopicListener {
+	rcs.mux.RLock()
+	defer rcs.mux.RUnlock()
+
+	listeners, ok := rcs.topicListeners[topic]
+	if !ok {
+		return nil
+	}
+
+	listenersCopy := make([]TopicListener, 0, len(listeners))
+	for _, l := range listeners {
+		listenersCopy = append(listenersCopy, l)
+	}
+	return listenersCopy
+}
+
+func (rcs *Service) AddConnectionStateListener(listener ConnectionStateListener) string {
+	id := model.NewId()
+
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+
+	rcs.connectionStateListeners[id] = listener
+	return id
+}
+
+func (rcs *Service) RemoveConnectionStateListener(listenerId string) {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+	delete(rcs.connectionStateListeners, listenerId)
+}
+
+// onClusterLeaderChange is called whenever the cluster leader may have changed.
+func (rcs *Service) onClusterLeaderChange() {
+	if rcs.server.IsLeader() {
+		rcs.pingStart()
+	} else {
+		rcs.pingStop()
+	}
+}
+
+func (rcs *Service) pingStart() {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+
+	if rcs.pingDone != nil {
+		return
+	}
+	rcs.pingDone = make(chan struct{})
+
+	if !rcs.disablePing {
+		// first ping all the plugin remotes immediately, synchronously.
+		rcs.pingAllNow(model.RemoteClusterQueryFilter{OnlyPlugins: true})
+
+		// start the async ping loop
+		rcs.pingLoop(rcs.pingDone)
+	}
+
+	rcs.server.Log().Debug("Remote Cluster Service ping active")
+}
+
+func (rcs *Service) pingStop() {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+
+	if rcs.pingDone == nil {
+		return
+	}
+	close(rcs.pingDone)
+	rcs.pingDone = nil
+
+	rcs.server.Log().Debug("Remote Cluster Service ping inactive")
+}

server/platform/services/remotecluster/service_test.go

@@ -0,0 +1,475 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
+	"github.com/mattermost/mattermost/server/v8/einterfaces"
+)
+
+func TestService_AddTopicListener(t *testing.T) {
+	var count atomic.Int32
+
+	l1 := func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response) error {
+		count.Add(1)
+		return nil
+	}
+	l2 := func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response) error {
+		count.Add(1)
+		return nil
+	}
+	l3 := func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response) error {
+		count.Add(1)
+		return nil
+	}
+
+	mockServer := newMockServer(t, makeRemoteClusters(NumRemotes, "", false))
+	mockApp := newMockApp(t, nil)
+
+	service, err := NewRemoteClusterService(mockServer, mockApp)
+	require.NoError(t, err)
+
+	l1id := service.AddTopicListener("test", l1)
+	l2id := service.AddTopicListener("test", l2)
+	l3id := service.AddTopicListener("different", l3)
+
+	listeners := service.getTopicListeners("test")
+	assert.Len(t, listeners, 2)
+
+	rc := &model.RemoteCluster{}
+	msg1 := model.RemoteClusterMsg{Topic: "test"}
+	msg2 := model.RemoteClusterMsg{Topic: "different"}
+
+	service.ReceiveIncomingMsg(rc, msg1)
+	assert.Equal(t, int32(2), count.Load())
+
+	service.ReceiveIncomingMsg(rc, msg2)
+	assert.Equal(t, int32(3), count.Load())
+
+	service.RemoveTopicListener(l1id)
+	service.ReceiveIncomingMsg(rc, msg1)
+	assert.Equal(t, int32(4), count.Load())
+
+	service.RemoveTopicListener(l2id)
+	service.ReceiveIncomingMsg(rc, msg1)
+	assert.Equal(t, int32(4), count.Load())
+
+	service.ReceiveIncomingMsg(rc, msg2)
+	assert.Equal(t, int32(5), count.Load())
+
+	service.RemoveTopicListener(l3id)
+	service.ReceiveIncomingMsg(rc, msg1)
+	service.ReceiveIncomingMsg(rc, msg2)
+	assert.Equal(t, int32(5), count.Load())
+
+	listeners = service.getTopicListeners("test")
+	assert.Empty(t, listeners)
+}
+
+func TestRemoteClusterTransportIdleConnTimeout(t *testing.T) {
+	mockServer := newMockServer(t, nil)
+	mockApp := newMockApp(t, nil)
+
+	service, err := NewRemoteClusterService(mockServer, mockApp)
+	require.NoError(t, err)
+
+	transport, ok := service.httpClient.Transport.(*http.Transport)
+	require.True(t, ok, "expected the remote cluster client to use an *http.Transport")
+
+	// Regression guard for MM-69982: the pooled-connection idle timeout must be
+	// strictly less than PingFreq. If it is >= PingFreq, a pooled connection
+	// survives from one ping to the next and every ping reuses a connection that
+	// the peer has likely already reaped (peers close idle keep-alives after their
+	// own IdleTimeout, default 60s), producing intermittent stale-connection ping
+	// failures. Keeping it below PingFreq means the pool discards the connection
+	// before the next ping, so each ping dials fresh and the race cannot occur.
+	require.Positive(t, transport.IdleConnTimeout, "IdleConnTimeout must be set")
+	assert.Less(t, transport.IdleConnTimeout, PingFreq,
+		"IdleConnTimeout (%s) must be strictly less than PingFreq (%s) to avoid stale keep-alive reuse on pings",
+		transport.IdleConnTimeout, PingFreq)
+}
+
+// leaderAwareMockServer is a mock server that supports toggling leader state
+// and firing leader-change listeners, allowing lifecycle tests to simulate
+// HA cluster leader transitions.
+type leaderAwareMockServer struct {
+	remotes  []*model.RemoteCluster
+	logger   *mlog.Logger
+	isLeader atomic.Bool
+
+	mux       sync.Mutex
+	listeners map[string]func()
+}
+
+func newLeaderAwareMockServer(t *testing.T, remotes []*model.RemoteCluster, leader bool) *leaderAwareMockServer {
+	ms := &leaderAwareMockServer{
+		remotes:   remotes,
+		logger:    mlog.CreateConsoleTestLogger(t),
+		listeners: make(map[string]func()),
+	}
+	ms.isLeader.Store(leader)
+	return ms
+}
+
+func (ms *leaderAwareMockServer) Config() *model.Config                    { return nil }
+func (ms *leaderAwareMockServer) GetMetrics() einterfaces.MetricsInterface { return nil }
+func (ms *leaderAwareMockServer) IsLeader() bool                           { return ms.isLeader.Load() }
+func (ms *leaderAwareMockServer) Log() *mlog.Logger                        { return ms.logger }
+
+func (ms *leaderAwareMockServer) AddClusterLeaderChangedListener(listener func()) string {
+	ms.mux.Lock()
+	defer ms.mux.Unlock()
+	id := model.NewId()
+	ms.listeners[id] = listener
+	return id
+}
+
+func (ms *leaderAwareMockServer) RemoveClusterLeaderChangedListener(id string) {
+	ms.mux.Lock()
+	defer ms.mux.Unlock()
+	delete(ms.listeners, id)
+}
+
+// setLeader changes leader status and fires all registered listeners.
+func (ms *leaderAwareMockServer) setLeader(leader bool) {
+	ms.isLeader.Store(leader)
+	ms.mux.Lock()
+	listeners := make([]func(), 0, len(ms.listeners))
+	for _, l := range ms.listeners {
+		listeners = append(listeners, l)
+	}
+	ms.mux.Unlock()
+	for _, l := range listeners {
+		l()
+	}
+}
+
+func (ms *leaderAwareMockServer) GetStore() store.Store {
+	anyQueryFilter := mock.MatchedBy(func(filter model.RemoteClusterQueryFilter) bool {
+		return true
+	})
+	anyId := mock.AnythingOfType("string")
+
+	remoteClusterStoreMock := &mocks.RemoteClusterStore{}
+	remoteClusterStoreMock.On("GetByTopic", "share").Return(ms.remotes, nil)
+	remoteClusterStoreMock.On("GetAll", 0, 999999, anyQueryFilter).Return(ms.remotes, nil)
+	remoteClusterStoreMock.On("SetLastPingAt", anyId).Return(nil)
+
+	storeMock := &mocks.Store{}
+	storeMock.On("RemoteCluster").Return(remoteClusterStoreMock)
+	return storeMock
+}
+
+func TestServiceLifecycle(t *testing.T) {
+	t.Run("Active after Start, inactive after Shutdown", func(t *testing.T) {
+		mockServer := newLeaderAwareMockServer(t, nil, false)
+		mockApp := newMockApp(t, nil)
+
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		assert.False(t, service.Active(), "service should not be active before Start")
+
+		err = service.Start()
+		require.NoError(t, err)
+
+		assert.True(t, service.Active(), "service should be active after Start")
+
+		err = service.Shutdown()
+		require.NoError(t, err)
+
+		assert.False(t, service.Active(), "service should not be active after Shutdown")
+	})
+
+	t.Run("Double Start is idempotent", func(t *testing.T) {
+		mockServer := newLeaderAwareMockServer(t, nil, false)
+		mockApp := newMockApp(t, nil)
+
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		err = service.Start()
+		require.NoError(t, err)
+
+		firstDone := service.done
+
+		// Second Start should be a no-op
+		err = service.Start()
+		require.NoError(t, err)
+
+		assert.Equal(t, firstDone, service.done, "second Start should not replace done channel")
+
+		require.NoError(t, service.Shutdown())
+	})
+
+	t.Run("Shutdown before Start does not panic", func(t *testing.T) {
+		mockServer := newLeaderAwareMockServer(t, nil, false)
+		mockApp := newMockApp(t, nil)
+
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		err = service.Shutdown()
+		require.NoError(t, err)
+	})
+
+	t.Run("Active on non-leader node", func(t *testing.T) {
+		mockServer := newLeaderAwareMockServer(t, nil, false)
+		mockApp := newMockApp(t, nil)
+
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer func() { require.NoError(t, service.Shutdown()) }()
+
+		assert.True(t, service.Active(), "service should be active on non-leader node")
+	})
+}
+
+func TestSendLoopLifecycle(t *testing.T) {
+	t.Run("sendLoop runs on non-leader node", func(t *testing.T) {
+		var webReqCount atomic.Int32
+
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			webReqCount.Add(1)
+			w.WriteHeader(200)
+			resp := Response{}
+			b, _ := json.Marshal(&resp)
+			_, _ = w.Write(b)
+		}))
+		defer ts.Close()
+
+		remotes := makeRemoteClusters(3, ts.URL, false)
+		mockServer := newLeaderAwareMockServer(t, remotes, false) // non-leader
+		mockApp := newMockApp(t, nil)
+
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+		service.disablePing = true
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer func() { require.NoError(t, service.Shutdown()) }()
+
+		// Verify pings are NOT running (non-leader)
+		service.mux.RLock()
+		pingRunning := service.pingDone != nil
+		service.mux.RUnlock()
+		assert.False(t, pingRunning, "pings should not be running on non-leader node")
+
+		// Send a message — sendLoop should process it even on non-leader
+		msg := makeRemoteClusterMsg(model.NewId(), NoteContent)
+		var callbackCount atomic.Int32
+		wg := &sync.WaitGroup{}
+		wg.Add(3)
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+		defer cancel()
+
+		err = service.BroadcastMsg(ctx, msg, func(msg model.RemoteClusterMsg, remote *model.RemoteCluster, resp *Response, err error) {
+			defer wg.Done()
+			callbackCount.Add(1)
+		})
+		require.NoError(t, err)
+
+		wg.Wait()
+
+		assert.Equal(t, int32(3), callbackCount.Load(), "all callbacks should fire on non-leader node")
+		assert.Equal(t, int32(3), webReqCount.Load(), "all HTTP requests should be made on non-leader node")
+	})
+
+	t.Run("sendLoop survives leader transitions", func(t *testing.T) {
+		var webReqCount atomic.Int32
+
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			webReqCount.Add(1)
+			w.WriteHeader(200)
+			resp := Response{}
+			b, _ := json.Marshal(&resp)
+			_, _ = w.Write(b)
+		}))
+		defer ts.Close()
+
+		remotes := makeRemoteClusters(3, ts.URL, false)
+		mockServer := newLeaderAwareMockServer(t, remotes, true) // start as leader
+		mockApp := newMockApp(t, nil)
+
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+		service.disablePing = true
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer func() { require.NoError(t, service.Shutdown()) }()
+
+		// Lose leadership
+		mockServer.setLeader(false)
+
+		// Send a message — should still work after losing leadership
+		msg := makeRemoteClusterMsg(model.NewId(), NoteContent)
+		wg := &sync.WaitGroup{}
+		wg.Add(3)
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+		defer cancel()
+
+		err = service.BroadcastMsg(ctx, msg, func(msg model.RemoteClusterMsg, remote *model.RemoteCluster, resp *Response, err error) {
+			defer wg.Done()
+		})
+		require.NoError(t, err)
+
+		wg.Wait()
+
+		assert.Equal(t, int32(3), webReqCount.Load(), "sends should work after losing leadership")
+	})
+}
+
+func TestPingLifecycle(t *testing.T) {
+	t.Run("pings start on leader, stop on non-leader", func(t *testing.T) {
+		mockServer := newLeaderAwareMockServer(t, nil, false)
+		mockApp := newMockApp(t, nil)
+
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer func() { require.NoError(t, service.Shutdown()) }()
+
+		// Non-leader: pings should not be running
+		service.mux.RLock()
+		assert.Nil(t, service.pingDone, "pingDone should be nil on non-leader")
+		service.mux.RUnlock()
+
+		// Become leader: pings should start
+		mockServer.setLeader(true)
+
+		service.mux.RLock()
+		assert.NotNil(t, service.pingDone, "pingDone should be set after becoming leader")
+		service.mux.RUnlock()
+
+		// Lose leadership: pings should stop
+		mockServer.setLeader(false)
+
+		service.mux.RLock()
+		assert.Nil(t, service.pingDone, "pingDone should be nil after losing leadership")
+		service.mux.RUnlock()
+	})
+
+	t.Run("pingStart is idempotent", func(t *testing.T) {
+		mockServer := newLeaderAwareMockServer(t, nil, true)
+		mockApp := newMockApp(t, nil)
+
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer func() { require.NoError(t, service.Shutdown()) }()
+
+		// pingDone should already be set (started as leader)
+		service.mux.RLock()
+		firstPingDone := service.pingDone
+		service.mux.RUnlock()
+		require.NotNil(t, firstPingDone)
+
+		// Call pingStart again — should be a no-op, same channel
+		service.pingStart()
+
+		service.mux.RLock()
+		secondPingDone := service.pingDone
+		service.mux.RUnlock()
+
+		assert.Equal(t, firstPingDone, secondPingDone, "pingStart should be idempotent")
+	})
+
+	t.Run("pingStop is idempotent", func(t *testing.T) {
+		mockServer := newLeaderAwareMockServer(t, nil, false)
+		mockApp := newMockApp(t, nil)
+
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer func() { require.NoError(t, service.Shutdown()) }()
+
+		// Already non-leader, pingDone is nil
+		service.mux.RLock()
+		assert.Nil(t, service.pingDone)
+		service.mux.RUnlock()
+
+		// Calling pingStop again should not panic
+		service.pingStop()
+
+		service.mux.RLock()
+		assert.Nil(t, service.pingDone)
+		service.mux.RUnlock()
+	})
+
+	t.Run("pings fire on leader with real ping loop", func(t *testing.T) {
+		var pingCount atomic.Int32
+
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			pingCount.Add(1)
+			w.WriteHeader(200)
+			resp := model.RemoteClusterPing{}
+			b, _ := json.Marshal(&resp)
+			_, _ = w.Write(b)
+		}))
+		defer ts.Close()
+
+		remotes := makeRemoteClusters(2, ts.URL, false)
+		mockServer := newLeaderAwareMockServer(t, remotes, false) // start as non-leader
+		mockApp := newMockApp(t, nil)
+
+		service, err := NewRemoteClusterService(mockServer, mockApp)
+		require.NoError(t, err)
+		service.SetPingFreq(time.Millisecond * 50)
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer func() { require.NoError(t, service.Shutdown()) }()
+
+		// Non-leader: no pings should be sent
+		assert.Never(t, func() bool {
+			return pingCount.Load() > 0
+		}, time.Millisecond*200, time.Millisecond*50, "no pings should fire on non-leader")
+
+		// Become leader: pings should start firing
+		mockServer.setLeader(true)
+
+		assert.Eventually(t, func() bool {
+			return pingCount.Load() >= 2
+		}, time.Second*5, time.Millisecond*50, "pings should fire after becoming leader")
+
+		// Lose leadership: pings should stop
+		mockServer.setLeader(false)
+
+		// Allow in-flight pings to drain, then verify no new pings arrive
+		assert.Eventually(t, func() bool {
+			snapshot := pingCount.Load()
+			time.Sleep(time.Millisecond * 150)
+			return pingCount.Load() == snapshot
+		}, time.Second*5, time.Millisecond*50, "no new pings should fire after losing leadership")
+	})
+}