snapshot of https://github.com/mattermost/mattermost.git (master, 2026-09-16; blobs > 131072B omitted)

server/channels/app/platform/searchengine_watcher.go

@@ -0,0 +1,389 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
+)
+
+// Watcher tuning knobs -- declared as vars so tests can override them.
+var (
+	searchEngineRetryInitial        = 15 * time.Second
+	searchEngineRetryMax            = 5 * time.Minute
+	searchEngineHealthInterval      = 60 * time.Second
+	searchEngineHealthFailThreshold = 3
+	searchEngineStopTimeout         = 15 * time.Second
+)
+
+// searchEngineWatcher monitors an Elasticsearch engine's health and manages
+// its lifecycle (Start / Stop) in a single background goroutine. All engine
+// state mutations happen inside this goroutine, eliminating races between
+// concurrent callers.
+//
+// External code communicates with the watcher through two signals:
+//   - reevaluate():     wake the watcher for immediate re-evaluation
+//   - requestRestart(): tell the watcher to Stop() before re-evaluating
+type searchEngineWatcher struct {
+	ps     *PlatformService
+	engine searchengine.SearchEngineInterface
+
+	// Coordination -- accessed from multiple goroutines.
+	mu           sync.Mutex
+	cancel       context.CancelFunc // nil when not running
+	done         chan struct{}      // closed when goroutine exits
+	notifyCh     chan struct{}      // buffered(1), non-blocking wake signal
+	forceRestart int32              // atomic: 1 = stop engine before next evaluation
+}
+
+// watcherLoopState holds per-goroutine loop state. Declared locally in run()
+// and passed by pointer to helpers, so an abandoned goroutine and a fresh one
+// never share these fields.
+type watcherLoopState struct {
+	backoff             time.Duration
+	consecutiveFailures int
+}
+
+func newSearchEngineWatcher(ps *PlatformService) *searchEngineWatcher {
+	return &searchEngineWatcher{
+		ps:     ps,
+		engine: ps.SearchEngine.ElasticsearchEngine,
+	}
+}
+
+// start launches the watcher goroutine. Idempotent: no-op if already running.
+func (w *searchEngineWatcher) start() {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	if w.cancel != nil {
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	w.cancel = cancel
+	w.done = make(chan struct{})
+	w.notifyCh = make(chan struct{}, 1)
+
+	done := w.done
+	notifyCh := w.notifyCh
+	go func() {
+		defer close(done)
+		w.run(ctx, notifyCh)
+	}()
+}
+
+// stop cancels the watcher and waits for it to exit with a bounded timeout.
+// If the watcher is stuck (e.g. in a blocking Start() call), the goroutine is
+// abandoned and will self-terminate when the blocking call returns. The mutex
+// is NOT held during the wait.
+func (w *searchEngineWatcher) stop() {
+	w.mu.Lock()
+	cancel := w.cancel
+	done := w.done
+	w.cancel = nil
+	w.done = nil
+	w.notifyCh = nil
+	w.mu.Unlock()
+
+	if cancel == nil {
+		return
+	}
+
+	cancel()
+
+	select {
+	case <-done:
+	case <-time.After(searchEngineStopTimeout):
+		w.ps.Log().Warn("Search engine watcher did not stop in time; abandoning goroutine",
+			mlog.Duration("timeout", searchEngineStopTimeout))
+	}
+}
+
+// reevaluate sends a non-blocking signal to the watcher to re-evaluate engine
+// state immediately. Safe to call on a nil receiver.
+func (w *searchEngineWatcher) reevaluate() {
+	if w == nil {
+		return
+	}
+
+	w.mu.Lock()
+	ch := w.notifyCh
+	w.mu.Unlock()
+
+	if ch != nil {
+		select {
+		case ch <- struct{}{}:
+		default:
+		}
+	}
+}
+
+// requestRestart sets a flag that tells the watcher to Stop() the engine
+// before its next evaluation cycle, then wakes it for immediate re-evaluation.
+// Used when connection settings change or the license is removed.
+// Safe to call on a nil receiver.
+func (w *searchEngineWatcher) requestRestart() {
+	if w == nil {
+		return
+	}
+	atomic.StoreInt32(&w.forceRestart, 1)
+	w.reevaluate()
+}
+
+// run is the watcher's main loop. It drives a simple state machine:
+//
+//	(start)
+//	   |
+//	   v
+//	forceRestart? --yes--> Stop()
+//	   |
+//	   v
+//	IsEnabled()? --no---> PARK (wait for notify or cancel)
+//	   |
+//	  yes
+//	   |
+//	   v
+//	IsActive()? --yes---> HEALTH CHECK (periodic)
+//	   |                        |
+//	  no                 N consecutive failures
+//	   |                        |
+//	   v                   Stop() engine
+//	RETRY <---------------------+
+//	   |
+//	   | wait (exponential backoff, interruptible)
+//	   v
+//	Start()
+//	   |--error--> backoff*2 (capped), retry
+//	   |--ok-----> reset backoff, HEALTH CHECK
+//
+//	Any state -- ctx.Done() --> EXIT
+//	Any state -- reevaluate --> immediate re-evaluation
+func (w *searchEngineWatcher) run(ctx context.Context, notifyCh <-chan struct{}) {
+	if w.engine == nil {
+		return
+	}
+
+	// Safety net: if the engine is still active when the goroutine exits
+	// (e.g. Start() completed just as the context was canceled), stop it so
+	// it is not left running with no goroutine to manage it.
+	defer func() {
+		if w.engine.IsActive() {
+			w.ps.Log().Info("Search engine watcher: stopping engine on goroutine exit",
+				mlog.String("engine", w.engine.GetName()))
+			if err := w.engine.Stop(); err != nil {
+				w.ps.Log().Warn("Search engine watcher: Stop() failed on goroutine exit",
+					mlog.Err(err),
+					mlog.String("engine", w.engine.GetName()))
+			}
+		}
+	}()
+
+	s := &watcherLoopState{backoff: searchEngineRetryInitial}
+	rctx := request.EmptyContext(w.ps.logger)
+
+	timer := time.NewTimer(0) // immediate first evaluation
+	defer timer.Stop()
+
+	for {
+		if !w.waitForEvent(ctx, timer, notifyCh, s) {
+			return
+		}
+		w.applyForceRestart()
+		if w.parkIfDisabled(timer) {
+			continue
+		}
+		if w.startIfInactive(ctx, timer, s) {
+			continue
+		}
+		w.healthCheck(rctx, timer, s)
+	}
+}
+
+// waitForEvent blocks until the timer fires, a notify arrives, or the context
+// is cancelled. Returns false when the watcher should exit.
+func (w *searchEngineWatcher) waitForEvent(ctx context.Context, timer *time.Timer, notifyCh <-chan struct{}, s *watcherLoopState) bool {
+	for {
+		select {
+		case <-ctx.Done():
+			return false
+		case <-notifyCh:
+			// Something changed -- re-evaluate immediately.
+			timer.Reset(0)
+			s.backoff = searchEngineRetryInitial
+			s.consecutiveFailures = 0
+			continue
+		case <-timer.C:
+		}
+
+		// Prioritize shutdown: if select picked timer.C and ctx.Done()
+		// simultaneously (random when both are ready), exit now.
+		return ctx.Err() == nil
+	}
+}
+
+// applyForceRestart checks and clears the force-restart flag. If set and the
+// engine is active, it stops the engine so the next evaluation cycle will
+// create a fresh client.
+func (w *searchEngineWatcher) applyForceRestart() {
+	if !atomic.CompareAndSwapInt32(&w.forceRestart, 1, 0) {
+		return
+	}
+	if w.engine.IsActive() {
+		w.ps.Log().Info("Search engine watcher: force-restart requested, stopping engine",
+			mlog.String("engine", w.engine.GetName()))
+		if err := w.engine.Stop(); err != nil {
+			w.ps.Log().Warn("Search engine watcher: Stop() failed during force-restart",
+				mlog.Err(err),
+				mlog.String("engine", w.engine.GetName()))
+		}
+	}
+}
+
+// parkIfDisabled stops the engine (if active) and parks the watcher when the
+// engine is disabled. Returns true when parked (caller should continue the
+// loop without scheduling a timer -- the next wake comes from reevaluate).
+func (w *searchEngineWatcher) parkIfDisabled(timer *time.Timer) bool {
+	if w.engine.IsEnabled() {
+		return false
+	}
+	if w.engine.IsActive() {
+		if err := w.engine.Stop(); err != nil {
+			w.ps.Log().Warn("Search engine watcher: Stop() returned error while disabling",
+				mlog.Err(err),
+				mlog.String("engine", w.engine.GetName()))
+		}
+	}
+	w.ps.Log().Info("Search engine watcher: engine disabled, parking")
+	timer.Stop()
+	return true
+}
+
+// parkIfUnlicensed parks the watcher when the server has no Elasticsearch
+// license. Returns true when parked (caller should continue the loop).
+func (w *searchEngineWatcher) parkIfUnlicensed(timer *time.Timer) bool {
+	license := w.ps.License()
+	if license != nil && model.SafeDereference(license.Features.Elasticsearch) {
+		return false
+	}
+	w.ps.Log().Info("Search engine watcher: engine not active (no Elasticsearch license), parking",
+		mlog.String("engine", w.engine.GetName()))
+	timer.Stop()
+	return true
+}
+
+// startIfInactive attempts to start the engine when it is not active.
+// Returns true when it handled the state (caller should continue the loop).
+func (w *searchEngineWatcher) startIfInactive(ctx context.Context, timer *time.Timer, s *watcherLoopState) bool {
+	if w.engine.IsActive() {
+		return false
+	}
+
+	if err := w.engine.Start(ctx); err != nil {
+		s.consecutiveFailures++
+		w.ps.Log().Error("Search engine watcher: Start() failed, will retry",
+			mlog.Err(err),
+			mlog.Int("consecutive_failures", s.consecutiveFailures),
+			mlog.Duration("next_backoff", s.backoff),
+			mlog.String("engine", w.engine.GetName()))
+		timer.Reset(s.backoff)
+		s.backoff = min(s.backoff*2, searchEngineRetryMax)
+		return true
+	}
+
+	if ctx.Err() != nil {
+		return true // shutting down
+	}
+
+	// Start() returned nil but engine may not be active (e.g. no license).
+	if !w.engine.IsActive() {
+		if w.parkIfUnlicensed(timer) {
+			return true
+		}
+
+		s.consecutiveFailures++
+		w.ps.Log().Warn("Search engine watcher: Start() returned no error but engine is not active, will retry",
+			mlog.Int("consecutive_failures", s.consecutiveFailures),
+			mlog.Duration("next_backoff", s.backoff),
+			mlog.String("engine", w.engine.GetName()))
+		timer.Reset(s.backoff)
+		s.backoff = min(s.backoff*2, searchEngineRetryMax)
+		return true
+	}
+
+	w.ps.Log().Info("Search engine watcher: engine started successfully",
+		mlog.String("engine", w.engine.GetName()))
+	s.backoff = searchEngineRetryInitial
+	s.consecutiveFailures = 0
+
+	if model.SafeDereference(w.ps.Config().ElasticsearchSettings.EnableSearchPublicChannelsWithoutMembership) {
+		engine := w.engine
+		w.ps.Go(func() {
+			w.ps.backfillPostsChannelType(engine)
+		})
+	}
+
+	timer.Reset(searchEngineHealthInterval)
+	return true
+}
+
+// healthCheck runs a periodic health check on an active engine. After
+// reaching the consecutive failure threshold it stops the engine and
+// schedules a retry; otherwise it schedules the next health check.
+func (w *searchEngineWatcher) healthCheck(rctx request.CTX, timer *time.Timer, s *watcherLoopState) {
+	// Save the previous state so that we can log any state change
+	wasHealthy := w.engine.IsHealthy()
+
+	if err := w.engine.HealthCheck(rctx); err != nil {
+		// Mark unhealthy on the first failure so the broker skips this
+		// engine immediately (searches fall back to DB, indexing is
+		// deferred). The engine stays started until the consecutive
+		// failure threshold is reached.
+		w.engine.SetHealthy(false)
+		if wasHealthy {
+			w.ps.Log().Warn("Search engine health check failed: it is now marked as unhealthy",
+				mlog.String("engine", w.engine.GetName()))
+		}
+
+		s.consecutiveFailures++
+		if s.consecutiveFailures >= searchEngineHealthFailThreshold {
+			w.ps.Log().Error("Search engine health check failed repeatedly; stopping engine",
+				mlog.Err(err),
+				mlog.Int("consecutive_failures", s.consecutiveFailures),
+				mlog.String("engine", w.engine.GetName()))
+
+			if stopErr := w.engine.Stop(); stopErr != nil {
+				w.ps.Log().Warn("Search engine watcher: Stop() returned error (may already be stopped)",
+					mlog.Err(stopErr),
+					mlog.String("engine", w.engine.GetName()))
+			}
+			s.consecutiveFailures = 0
+			s.backoff = searchEngineRetryInitial
+			timer.Reset(s.backoff)
+			return
+		}
+
+		w.ps.Log().Warn("Search engine health check failed",
+			mlog.Err(err),
+			mlog.Int("consecutive_failures", s.consecutiveFailures),
+			mlog.Int("threshold", searchEngineHealthFailThreshold),
+			mlog.String("engine", w.engine.GetName()))
+	} else {
+		w.engine.SetHealthy(true)
+		if !wasHealthy {
+			w.ps.Log().Info("Search engine health check succeeded: it is now marked as healthy",
+				mlog.String("engine", w.engine.GetName()))
+		}
+		s.consecutiveFailures = 0
+	}
+
+	timer.Reset(searchEngineHealthInterval)
+}