// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. package sharedchannel import ( "context" "database/sql" "errors" "fmt" "maps" "sync" "testing" "time" "github.com/mattermost/mattermost/server/public/model" "github.com/mattermost/mattermost/server/public/shared/mlog" "github.com/mattermost/mattermost/server/v8/channels/store" "github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks" "github.com/mattermost/mattermost/server/v8/platform/services/remotecluster" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) func newTestService() *Service { return &Service{ changeSignal: make(chan struct{}, 1), tasks: make(map[string]syncTask), } } func TestProcessTask_RemoteClusterLookup(t *testing.T) { // Mirror how the store wraps the not-found case for a soft-deleted/missing remote. notFound := fmt.Errorf("failed to find RemoteCluster: %w", sql.ErrNoRows) newService := func(t *testing.T, remoteID string, rc *model.RemoteCluster, getErr error) (*Service, *mocks.SharedChannelStore) { t.Helper() mockRemoteClusterStore := &mocks.RemoteClusterStore{} mockRemoteClusterStore.On("Get", remoteID, false).Return(rc, getErr) mockSharedChannelStore := &mocks.SharedChannelStore{} mockStore := &mocks.Store{} mockStore.On("RemoteCluster").Return(mockRemoteClusterStore) mockStore.On("SharedChannel").Return(mockSharedChannelStore) mockServer := &MockServerIface{} mockServer.On("GetStore").Return(mockStore) mockServer.On("GetMetrics").Return(nil) mockServer.On("Log").Return(mlog.CreateConsoleTestLogger(t)) return &Service{ server: mockServer, changeSignal: make(chan struct{}, 1), tasks: make(map[string]syncTask), }, mockSharedChannelStore } t.Run("orphaned remote self-heals by soft-deleting the live SCR row", func(t *testing.T) { channelID := model.NewId() remoteID := model.NewId() scr := &model.SharedChannelRemote{Id: model.NewId(), ChannelId: channelID, RemoteId: remoteID, DeleteAt: 0} scs, scStore := newService(t, remoteID, nil, notFound) scStore.On("GetRemoteByIds", channelID, remoteID).Return(scr, nil) scStore.On("DeleteRemote", scr.Id).Return(true, nil) err := scs.processTask(newSyncTask(channelID, "", remoteID, nil, nil)) require.NoError(t, err, "an orphaned remote should be skipped, not surfaced as an error") scStore.AssertNumberOfCalls(t, "DeleteRemote", 1) scStore.AssertCalled(t, "DeleteRemote", scr.Id) }) t.Run("self-healed orphan is drained from the sync queue and not retried", func(t *testing.T) { channelID := model.NewId() remoteID := model.NewId() scr := &model.SharedChannelRemote{Id: model.NewId(), ChannelId: channelID, RemoteId: remoteID, DeleteAt: 0} scs, scStore := newService(t, remoteID, nil, notFound) scStore.On("GetRemoteByIds", channelID, remoteID).Return(scr, nil) scStore.On("DeleteRemote", scr.Id).Return(true, nil) task := newSyncTask(channelID, "", remoteID, nil, nil) scs.addTask(task) scs.doSync() assert.Empty(t, scs.tasks, "the orphaned task must be drained, not re-enqueued for retry") scStore.AssertNumberOfCalls(t, "DeleteRemote", 1) }) t.Run("already soft-deleted SCR row is not deleted again", func(t *testing.T) { channelID := model.NewId() remoteID := model.NewId() scr := &model.SharedChannelRemote{Id: model.NewId(), ChannelId: channelID, RemoteId: remoteID, DeleteAt: model.GetMillis()} scs, scStore := newService(t, remoteID, nil, notFound) scStore.On("GetRemoteByIds", channelID, remoteID).Return(scr, nil) err := scs.processTask(newSyncTask(channelID, "", remoteID, nil, nil)) require.NoError(t, err) scStore.AssertNotCalled(t, "DeleteRemote", mock.Anything) }) t.Run("missing SCR row is skipped without error or delete", func(t *testing.T) { channelID := model.NewId() remoteID := model.NewId() scs, scStore := newService(t, remoteID, nil, notFound) scStore.On("GetRemoteByIds", channelID, remoteID).Return(nil, store.NewErrNotFound("SharedChannelRemote", "missing")) err := scs.processTask(newSyncTask(channelID, "", remoteID, nil, nil)) require.NoError(t, err) scStore.AssertNotCalled(t, "DeleteRemote", mock.Anything) }) t.Run("failure to soft-delete the orphan is swallowed without retry", func(t *testing.T) { channelID := model.NewId() remoteID := model.NewId() scr := &model.SharedChannelRemote{Id: model.NewId(), ChannelId: channelID, RemoteId: remoteID, DeleteAt: 0} scs, scStore := newService(t, remoteID, nil, notFound) scStore.On("GetRemoteByIds", channelID, remoteID).Return(scr, nil) scStore.On("DeleteRemote", scr.Id).Return(false, errors.New("db is down")) err := scs.processTask(newSyncTask(channelID, "", remoteID, nil, nil)) require.NoError(t, err, "a self-heal delete failure must not surface as a sync error or trigger a retry") scStore.AssertCalled(t, "DeleteRemote", scr.Id) }) t.Run("transient error is propagated for retry", func(t *testing.T) { remoteID := model.NewId() dbErr := errors.New("write tcp: connection reset by peer") scs, scStore := newService(t, remoteID, nil, dbErr) err := scs.processTask(newSyncTask("channel-1", "", remoteID, nil, nil)) require.Error(t, err, "a transient lookup error must still be returned so the task retries") assert.ErrorContains(t, err, "connection reset by peer") scStore.AssertNotCalled(t, "GetRemoteByIds", mock.Anything, mock.Anything) scStore.AssertNotCalled(t, "DeleteRemote", mock.Anything) }) } func TestAddTask_OriginRemoteIDMerge(t *testing.T) { tests := []struct { name string firstOrigin string secondOrigin string expectedOrigin string }{ { name: "same remote origin is preserved", firstOrigin: "remote-A", secondOrigin: "remote-A", expectedOrigin: "remote-A", }, { name: "local then remote clears origin", firstOrigin: "", secondOrigin: "remote-A", expectedOrigin: "", }, { name: "remote then local clears origin", firstOrigin: "remote-A", secondOrigin: "", expectedOrigin: "", }, { name: "different remotes clears origin", firstOrigin: "remote-A", secondOrigin: "remote-B", expectedOrigin: "", }, { name: "both local stays empty", firstOrigin: "", secondOrigin: "", expectedOrigin: "", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { scs := newTestService() channelID := "channel-1" first := newSyncTask(channelID, "", "", nil, nil) first.originRemoteID = tc.firstOrigin scs.addTask(first) second := newSyncTask(channelID, "", "", nil, nil) second.originRemoteID = tc.secondOrigin scs.addTask(second) merged, ok := scs.tasks[first.id] require.True(t, ok, "task should exist") assert.Equal(t, tc.expectedOrigin, merged.originRemoteID) }) } } // mockRCSForSync is a minimal no-op implementation of RemoteClusterServiceIFace that // records NotifySyncFailed calls. By default its SendMsg simulates an unreachable remote // (delivery failure reported via the callback); set cbResp/cbErr to instead deliver a // specific response envelope (e.g. an unparseable or empty payload with no transport error). type mockRCSForSync struct { mu sync.Mutex notifyCalls []string cbConfigured bool cbResp *remotecluster.Response cbErr error enqueueErr error // when set, SendMsg returns this WITHOUT invoking the callback } func (m *mockRCSForSync) NotifySyncFailed(remoteId string) { m.mu.Lock() defer m.mu.Unlock() m.notifyCalls = append(m.notifyCalls, remoteId) } func (m *mockRCSForSync) Shutdown() error { return nil } func (m *mockRCSForSync) Start() error { return nil } func (m *mockRCSForSync) Active() bool { return true } func (m *mockRCSForSync) AddTopicListener(topic string, listener remotecluster.TopicListener) string { return "" } func (m *mockRCSForSync) RemoveTopicListener(listenerId string) {} func (m *mockRCSForSync) AddConnectionStateListener(listener remotecluster.ConnectionStateListener) string { return "" } func (m *mockRCSForSync) RemoveConnectionStateListener(listenerId string) {} func (m *mockRCSForSync) SendMsg(_ context.Context, msg model.RemoteClusterMsg, rc *model.RemoteCluster, f remotecluster.SendMsgResultFunc) error { // Simulate an enqueue failure (e.g. BufferFullError): the callback never runs. if m.enqueueErr != nil { return m.enqueueErr } // The enqueue succeeds (return nil); the outcome is reported asynchronously via the // callback, mirroring how remotecluster.Service.sendMsg surfaces a send result. if f != nil { if m.cbConfigured { f(msg, rc, m.cbResp, m.cbErr) } else { // Default: simulate an unreachable remote (transport error). f(msg, rc, &remotecluster.Response{Err: "remote unreachable"}, errors.New("remote unreachable")) } } return nil } func (m *mockRCSForSync) SendFile(_ context.Context, _ *model.UploadSession, _ *model.FileInfo, _ *model.RemoteCluster, _ remotecluster.ReaderProvider, _ remotecluster.SendFileResultFunc) error { return errors.New("remote unreachable") } func (m *mockRCSForSync) SendProfileImage(_ context.Context, _ string, _ *model.RemoteCluster, _ remotecluster.ProfileImageProvider, _ remotecluster.SendProfileImageResultFunc) error { return errors.New("remote unreachable") } func (m *mockRCSForSync) AcceptInvitation(_ *model.RemoteClusterInvite, _, _, _, _, _ string) (*model.RemoteCluster, error) { return nil, nil } func (m *mockRCSForSync) ReceiveIncomingMsg(_ *model.RemoteCluster, _ model.RemoteClusterMsg) remotecluster.Response { return remotecluster.Response{} } func (m *mockRCSForSync) ReceiveInviteConfirmation(_ model.RemoteClusterInvite) (*model.RemoteCluster, error) { return nil, nil } func (m *mockRCSForSync) PingNow(_ *model.RemoteCluster) {} var _ remotecluster.RemoteClusterServiceIFace = (*mockRCSForSync)(nil) // TestProcessTask_RetryDelay verifies that a failed per-remote sync re-enqueues the task // with a schedule at least SyncRetryDelay in the future. func TestProcessTask_RetryDelay(t *testing.T) { channelID := model.NewId() remoteID := model.NewId() rc := &model.RemoteCluster{ RemoteId: remoteID, DisplayName: "test-remote", LastPingAt: model.GetMillis(), // online } rcStoreMock := &mocks.RemoteClusterStore{} rcStoreMock.On("Get", remoteID, false).Return(rc, nil) mockStore := &mocks.Store{} mockStore.On("RemoteCluster").Return(rcStoreMock) mockServer := &MockServerIface{} mockServer.On("GetStore").Return(mockStore) mockServer.On("GetMetrics").Return(nil) mockServer.On("GetRemoteClusterService").Return(nil) // nil rcs causes syncForRemote to fail immediately scs := &Service{ server: mockServer, changeSignal: make(chan struct{}, 1), tasks: make(map[string]syncTask), } task := newSyncTask(channelID, "", remoteID, nil, nil) scs.addTask(task) before := time.Now() scs.doSync() scs.mux.RLock() tasksCopy := make(map[string]syncTask, len(scs.tasks)) maps.Copy(tasksCopy, scs.tasks) scs.mux.RUnlock() require.Len(t, tasksCopy, 1, "failed task should be re-enqueued for retry") for _, retried := range tasksCopy { assert.Equal(t, 1, retried.retryCount, "retry count should be 1 after first failure") assert.True(t, retried.schedule.After(before.Add(SyncRetryDelay)), "re-enqueued task schedule should be at least SyncRetryDelay in the future (got %s, want after %s)", retried.schedule, before.Add(SyncRetryDelay)) } } // TestProcessTask_FanOutRetryPerRemote verifies that when a task with no specific remote // fans out to several remotes and each fails, every remote keeps its own retry task (rather // than all but one being discarded by addTask's merge on the shared task id). func TestProcessTask_FanOutRetryPerRemote(t *testing.T) { channelID := model.NewId() rcA := &model.RemoteCluster{RemoteId: model.NewId(), DisplayName: "remote-A"} rcB := &model.RemoteCluster{RemoteId: model.NewId(), DisplayName: "remote-B"} rcStoreMock := &mocks.RemoteClusterStore{} // Both the in-channel and auto-invited lookups return the two remotes; remotesMap dedups. rcStoreMock.On("GetAll", 0, 999999, mock.Anything).Return([]*model.RemoteCluster{rcA, rcB}, nil) mockStore := &mocks.Store{} mockStore.On("RemoteCluster").Return(rcStoreMock) mockServer := &MockServerIface{} mockServer.On("GetStore").Return(mockStore) mockServer.On("GetMetrics").Return(nil) mockServer.On("GetRemoteClusterService").Return(nil) // nil rcs makes syncForRemote fail for each remote scs := &Service{ server: mockServer, changeSignal: make(chan struct{}, 1), tasks: make(map[string]syncTask), } // Fan-out task: empty remoteID. scs.addTask(newSyncTask(channelID, "", "", nil, nil)) scs.doSync() scs.mux.RLock() tasksCopy := make(map[string]syncTask, len(scs.tasks)) maps.Copy(tasksCopy, scs.tasks) scs.mux.RUnlock() require.Len(t, tasksCopy, 2, "each failed remote should retain its own retry task") gotRemotes := make(map[string]bool) for _, retried := range tasksCopy { assert.Equal(t, 1, retried.retryCount, "retry count should be 1 after first failure") gotRemotes[retried.remoteID] = true } assert.True(t, gotRemotes[rcA.RemoteId], "remote A should have its own retry task") assert.True(t, gotRemotes[rcB.RemoteId], "remote B should have its own retry task") } // TestSendSyncMsgToRemote_NotifiesOnDeliveryFailure guards the offline-recovery fix: when a // send fails asynchronously (remote unreachable), sendSyncMsgToRemote does NOT surface an // error to the caller — the send is fire-and-forget and the cursor is left un-advanced — but // it signals the remote cluster service via NotifySyncFailed so the next successful ping // drives a ForceSyncForRemote recovery. func TestSendSyncMsgToRemote_NotifiesOnDeliveryFailure(t *testing.T) { mockRCS := &mockRCSForSync{} mockServer := &MockServerIface{} mockServer.On("GetRemoteClusterService").Return(mockRCS) mockServer.On("Log").Return(mlog.CreateConsoleTestLogger(t)) scs := &Service{ server: mockServer, changeSignal: make(chan struct{}, 1), tasks: make(map[string]syncTask), } remoteID := model.NewId() rc := &model.RemoteCluster{RemoteId: remoteID, Name: "test-remote", SiteURL: "http://example.invalid"} msg := model.NewSyncMsg(model.NewId()) var callbackInvoked bool err := scs.sendSyncMsgToRemote(msg, rc, func(model.SyncResponse, error) { callbackInvoked = true }) require.NoError(t, err, "a delivery failure must not be surfaced as an error; the send is fire-and-forget") assert.False(t, callbackInvoked, "the result callback must not run on a delivery failure, so the cursor stays un-advanced") mockRCS.mu.Lock() notifyCalls := mockRCS.notifyCalls mockRCS.mu.Unlock() require.Len(t, notifyCalls, 1, "NotifySyncFailed should be called once on delivery failure") assert.Equal(t, remoteID, notifyCalls[0]) } // TestSendSyncMsgToRemote_NotifiesOnUnconfirmedResponse covers the case where the send // succeeds at the transport level (no errResp) but the response envelope can't be treated // as a confirmed sync: an unparseable payload, or an empty payload (the receive side always // sets a SyncResponse payload on success). In both cases sendSyncMsgToRemote must not error, // must not advance the cursor, and must signal NotifySyncFailed for a ping-driven resync. func TestSendSyncMsgToRemote_NotifiesOnUnconfirmedResponse(t *testing.T) { cases := []struct { name string resp *remotecluster.Response }{ {name: "unparseable payload", resp: &remotecluster.Response{Status: model.StatusOk, Payload: []byte("not json")}}, {name: "empty payload", resp: &remotecluster.Response{Status: model.StatusOk}}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { mockRCS := &mockRCSForSync{cbConfigured: true, cbResp: tc.resp, cbErr: nil} mockServer := &MockServerIface{} mockServer.On("GetRemoteClusterService").Return(mockRCS) mockServer.On("Log").Return(mlog.CreateConsoleTestLogger(t)) scs := &Service{ server: mockServer, changeSignal: make(chan struct{}, 1), tasks: make(map[string]syncTask), } remoteID := model.NewId() rc := &model.RemoteCluster{RemoteId: remoteID, Name: "test-remote", SiteURL: "http://example.invalid"} msg := model.NewSyncMsg(model.NewId()) var callbackInvoked bool err := scs.sendSyncMsgToRemote(msg, rc, func(model.SyncResponse, error) { callbackInvoked = true }) require.NoError(t, err, "an unconfirmed response must not be surfaced as an error") assert.False(t, callbackInvoked, "the result callback must not run for an unconfirmed response, so the cursor stays un-advanced") mockRCS.mu.Lock() notifyCalls := mockRCS.notifyCalls mockRCS.mu.Unlock() require.Len(t, notifyCalls, 1, "NotifySyncFailed should be called once for an unconfirmed response") assert.Equal(t, remoteID, notifyCalls[0]) }) } } // TestSendSyncMsgToRemote_EnqueueErrorDoesNotHang verifies that when SendMsg fails to enqueue // (e.g. BufferFullError) — so the result callback never runs — sendSyncMsgToRemote returns the // error instead of blocking forever on the callback's WaitGroup, which would freeze the single // doSync goroutine and stall all shared-channel sync. func TestSendSyncMsgToRemote_EnqueueErrorDoesNotHang(t *testing.T) { mockRCS := &mockRCSForSync{enqueueErr: errors.New("buffer full")} mockServer := &MockServerIface{} mockServer.On("GetRemoteClusterService").Return(mockRCS) mockServer.On("Log").Return(mlog.CreateConsoleTestLogger(t)) scs := &Service{ server: mockServer, changeSignal: make(chan struct{}, 1), tasks: make(map[string]syncTask), } rc := &model.RemoteCluster{RemoteId: model.NewId(), Name: "test-remote", SiteURL: "http://example.invalid"} msg := model.NewSyncMsg(model.NewId()) // Guard with a timeout so a regression (hang on wg.Wait) fails fast instead of blocking // the whole suite until the test binary times out. done := make(chan error, 1) go func() { done <- scs.sendSyncMsgToRemote(msg, rc, nil) }() select { case err := <-done: require.Error(t, err, "an enqueue failure must be returned to the caller") assert.ErrorContains(t, err, "buffer full") case <-time.After(5 * time.Second): require.Fail(t, "sendSyncMsgToRemote hung on wg.Wait() after an enqueue failure") } // The send never left the queue, so there is no delivery outcome to notify about; that // path is the caller's retry (the returned error), not a ping-driven resync. mockRCS.mu.Lock() notifyCount := len(mockRCS.notifyCalls) mockRCS.mu.Unlock() assert.Zero(t, notifyCount, "no sync-failed notification should fire when the enqueue itself failed") } func TestStripSharedChannelStatePostsForSync(t *testing.T) { sd := &syncData{ posts: []*model.Post{ {Id: "state-1", Type: model.PostTypeSharedChannelState, ChannelId: "ch1", Message: "ignored"}, {Id: "user-1", Type: model.PostTypeDefault, ChannelId: "ch1", Message: "hello"}, }, } stripSharedChannelStatePostsForSync(sd) require.Len(t, sd.posts, 1) assert.Equal(t, "user-1", sd.posts[0].Id) assert.Equal(t, model.PostTypeDefault, sd.posts[0].Type) }