// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. package remotecluster import ( "encoding/json" "fmt" "net/http" "net/http/httptest" "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/wiggin77/merror" "github.com/mattermost/mattermost/server/public/model" "github.com/mattermost/mattermost/server/public/plugin/plugintest/mock" "github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks" ) const ( Recent = 60000 ) func TestPing(t *testing.T) { t.Run("No error", func(t *testing.T) { merr := merror.New() var remotes []*model.RemoteCluster pingsReceived := make(map[string]struct{}) var mux sync.Mutex ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer w.WriteHeader(200) var frame model.RemoteClusterFrame err := json.NewDecoder(r.Body).Decode(&frame) if err != nil { merr.Append(err) return } if len(frame.Msg.Payload) == 0 { merr.Append(fmt.Errorf("Payload should not be empty; remote_id=%s", frame.RemoteId)) return } // Make sure ping is from a remote that was added for this test. if !hasRemoteID(frame.RemoteId, remotes) { merr.Append(fmt.Errorf("RemoteID not in list of remotes for this test; remote_id=%s", frame.RemoteId)) return } var ping model.RemoteClusterPing err = json.Unmarshal(frame.Msg.Payload, &ping) if err != nil { merr.Append(err) return } if !checkRecent(ping.SentAt, Recent) { merr.Append(fmt.Errorf("timestamp out of range, got %d", ping.SentAt)) return } if ping.RecvAt != 0 { merr.Append(fmt.Errorf("timestamp should be 0, got %d", ping.RecvAt)) return } mux.Lock() defer mux.Unlock() pingsReceived[frame.RemoteId] = struct{}{} })) defer ts.Close() remotes = makeRemoteClusters(NumRemotes, ts.URL, false) mockServer := newMockServer(t, remotes) mockApp := newMockApp(t, nil) service, err := NewRemoteClusterService(mockServer, mockApp) require.NoError(t, err) err = service.Start() require.NoError(t, err) defer service.Shutdown() // wait up to 10 seconds for all remotes to get pinged. This will normally take less than 1 second // unless the server is very busy. assert.Eventually(t, func() bool { mux.Lock() defer mux.Unlock() return len(pingsReceived) == NumRemotes }, time.Second*10, time.Millisecond*50, "all remotes must get pinged") assert.NoError(t, merr.ErrorOrNil()) }) t.Run("HTTP errors", func(t *testing.T) { merr := merror.New() var remotes []*model.RemoteCluster pingsReceived := make(map[string]struct{}) var mux sync.Mutex ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var frame model.RemoteClusterFrame err := json.NewDecoder(r.Body).Decode(&frame) if err != nil { merr.Append(err) } // Make sure ping is from a remote that was added for this test. if !hasRemoteID(frame.RemoteId, remotes) { merr.Append(fmt.Errorf("RemoteID not in list of remotes for this test; remote_id=%s", frame.RemoteId)) return } var ping model.RemoteClusterPing err = json.Unmarshal(frame.Msg.Payload, &ping) if err != nil { merr.Append(err) } if !checkRecent(ping.SentAt, Recent) { merr.Append(fmt.Errorf("timestamp out of range, got %d", ping.SentAt)) } if ping.RecvAt != 0 { merr.Append(fmt.Errorf("timestamp should be 0, got %d", ping.RecvAt)) } w.WriteHeader(500) mux.Lock() defer mux.Unlock() pingsReceived[frame.RemoteId] = struct{}{} })) defer ts.Close() remotes = makeRemoteClusters(NumRemotes, ts.URL, false) mockServer := newMockServer(t, remotes) mockApp := newMockApp(t, nil) service, err := NewRemoteClusterService(mockServer, mockApp) require.NoError(t, err) err = service.Start() require.NoError(t, err) defer service.Shutdown() // wait up to 10 seconds for all remotes to get pinged. This will normally take less than 1 second // until the server is very busy. assert.Eventually(t, func() bool { mux.Lock() defer mux.Unlock() return len(pingsReceived) == NumRemotes }, time.Second*10, time.Millisecond*50, "all remotes must get pinged") assert.NoError(t, merr.ErrorOrNil()) }) t.Run("Plugin ping", func(t *testing.T) { mockServer := newMockServer(t, makeRemoteClusters(NumRemotes, model.NewId(), true)) offline := []string{mockServer.remotes[0].PluginID, mockServer.remotes[1].PluginID} mockApp := newMockApp(t, offline) service, err := NewRemoteClusterService(mockServer, mockApp) require.NoError(t, err) // high ping frequency so we don't delay unit tests. service.SetPingFreq(time.Millisecond * 50) err = service.Start() require.NoError(t, err) defer service.Shutdown() checkPingCount := func() bool { return mockApp.GetTotalPingCount() >= NumRemotes } checkErrorCount := func() bool { return mockApp.GetTotalPingErrorCount() >= 2 } assert.Eventually(t, checkPingCount, time.Second*5, 10*time.Millisecond) assert.Eventually(t, checkErrorCount, time.Second*5, 10*time.Millisecond) }) } // pingTestListener records connection-state-change callbacks for the PingNow tests. type pingTestListener struct { mu sync.Mutex calls int online bool } func (l *pingTestListener) callback(rc *model.RemoteCluster, online bool) { l.mu.Lock() defer l.mu.Unlock() l.calls++ l.online = online } func (l *pingTestListener) snapshot() (calls int, online bool) { l.mu.Lock() defer l.mu.Unlock() return l.calls, l.online } // newPingTestService builds a remote cluster service whose remotes ping a local test server // that always responds 200, and registers a connection-state listener to observe PingNow. // The returned remote is already online (recent LastPingAt), so IsOnline() will not flip. func newPingTestService(t *testing.T) (*Service, *pingTestListener, *model.RemoteCluster) { t.Helper() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { pong, _ := json.Marshal(model.RemoteClusterPing{SentAt: model.GetMillis(), RecvAt: model.GetMillis()}) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) w.Write(pong) })) t.Cleanup(ts.Close) rcStoreMock := &mocks.RemoteClusterStore{} rcStoreMock.On("SetLastPingAt", mock.AnythingOfType("string")).Return(nil) storeMock := &mocks.Store{} storeMock.On("RemoteCluster").Return(rcStoreMock) service, err := NewRemoteClusterService(newMockServerWithStore(t, storeMock), newMockApp(t, nil)) require.NoError(t, err) listener := &pingTestListener{} listenerId := service.AddConnectionStateListener(listener.callback) t.Cleanup(func() { service.RemoveConnectionStateListener(listenerId) }) rc := &model.RemoteCluster{ RemoteId: model.NewId(), DisplayName: "test-remote", SiteURL: ts.URL, Token: model.NewId(), RemoteToken: model.NewId(), LastPingAt: model.GetMillis(), } return service, listener, rc } // TestPingNow_SyncFailureRecovery verifies that PingNow fires a connection-state-change // event when a sync failure was recorded since the last ping, even if the remote never // appeared offline (LastPingAt stayed within the 5-minute window). func TestPingNow_SyncFailureRecovery(t *testing.T) { service, listener, rc := newPingTestService(t) // Simulate a sync failure that occurred since the last ping. service.NotifySyncFailed(rc.RemoteId) service.PingNow(rc) calls, wasOnline := listener.snapshot() assert.Equal(t, 1, calls, "connection state listener should fire exactly once on sync failure recovery") assert.True(t, wasOnline, "listener should report online=true") // Marker must be cleared after a successful ping so subsequent pings don't re-fire. _, ok := service.syncFailedSinceLastPing.Load(rc.RemoteId) assert.False(t, ok, "syncFailedSinceLastPing marker should be cleared after a successful ping") } // TestPingNow_NoSpuriousFire verifies that PingNow does NOT fire a connection-state-change // event when the remote stays online and no sync failure marker is set. func TestPingNow_NoSpuriousFire(t *testing.T) { service, listener, rc := newPingTestService(t) service.PingNow(rc) calls, _ := listener.snapshot() assert.Equal(t, 0, calls, "listener must not fire when remote stays online and no sync failure marker is set") } func checkRecent(millis int64, within int64) bool { now := model.GetMillis() return millis > now-within && millis < now+within } func hasRemoteID(remoteID string, remotes []*model.RemoteCluster) bool { for _, r := range remotes { if r.RemoteId == remoteID { return true } } return false }