upstream-mattermost/server/channels/app/platform/searchengine_watcher.go

// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.

package platform

import (
	"context"
	"sync"
	"sync/atomic"
	"time"

	"github.com/mattermost/mattermost/server/public/model"
	"github.com/mattermost/mattermost/server/public/shared/mlog"
	"github.com/mattermost/mattermost/server/public/shared/request"
	"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
)

// Watcher tuning knobs -- declared as vars so tests can override them.
var (
	searchEngineRetryInitial        = 15 * time.Second
	searchEngineRetryMax            = 5 * time.Minute
	searchEngineHealthInterval      = 60 * time.Second
	searchEngineHealthFailThreshold = 3
	searchEngineStopTimeout         = 15 * time.Second
)

// searchEngineWatcher monitors an Elasticsearch engine's health and manages
// its lifecycle (Start / Stop) in a single background goroutine. All engine
// state mutations happen inside this goroutine, eliminating races between
// concurrent callers.
//
// External code communicates with the watcher through two signals:
//   - reevaluate():     wake the watcher for immediate re-evaluation
//   - requestRestart(): tell the watcher to Stop() before re-evaluating
type searchEngineWatcher struct {
	ps     *PlatformService
	engine searchengine.SearchEngineInterface

	// Coordination -- accessed from multiple goroutines.
	mu           sync.Mutex
	cancel       context.CancelFunc // nil when not running
	done         chan struct{}      // closed when goroutine exits
	notifyCh     chan struct{}      // buffered(1), non-blocking wake signal
	forceRestart int32              // atomic: 1 = stop engine before next evaluation
}

// watcherLoopState holds per-goroutine loop state. Declared locally in run()
// and passed by pointer to helpers, so an abandoned goroutine and a fresh one
// never share these fields.
type watcherLoopState struct {
	backoff             time.Duration
	consecutiveFailures int
}

func newSearchEngineWatcher(ps *PlatformService) *searchEngineWatcher {
	return &searchEngineWatcher{
		ps:     ps,
		engine: ps.SearchEngine.ElasticsearchEngine,
	}
}

// start launches the watcher goroutine. Idempotent: no-op if already running.
func (w *searchEngineWatcher) start() {
	w.mu.Lock()
	defer w.mu.Unlock()

	if w.cancel != nil {
		return
	}

	ctx, cancel := context.WithCancel(context.Background())
	w.cancel = cancel
	w.done = make(chan struct{})
	w.notifyCh = make(chan struct{}, 1)

	done := w.done
	notifyCh := w.notifyCh
	go func() {
		defer close(done)
		w.run(ctx, notifyCh)
	}()
}

// stop cancels the watcher and waits for it to exit with a bounded timeout.
// If the watcher is stuck (e.g. in a blocking Start() call), the goroutine is
// abandoned and will self-terminate when the blocking call returns. The mutex
// is NOT held during the wait.
func (w *searchEngineWatcher) stop() {
	w.mu.Lock()
	cancel := w.cancel
	done := w.done
	w.cancel = nil
	w.done = nil
	w.notifyCh = nil
	w.mu.Unlock()

	if cancel == nil {
		return
	}

	cancel()

	select {
	case <-done:
	case <-time.After(searchEngineStopTimeout):
		w.ps.Log().Warn("Search engine watcher did not stop in time; abandoning goroutine",
			mlog.Duration("timeout", searchEngineStopTimeout))
	}
}

// reevaluate sends a non-blocking signal to the watcher to re-evaluate engine
// state immediately. Safe to call on a nil receiver.
func (w *searchEngineWatcher) reevaluate() {
	if w == nil {
		return
	}

	w.mu.Lock()
	ch := w.notifyCh
	w.mu.Unlock()

	if ch != nil {
		select {
		case ch <- struct{}{}:
		default:
		}
	}
}

// requestRestart sets a flag that tells the watcher to Stop() the engine
// before its next evaluation cycle, then wakes it for immediate re-evaluation.
// Used when connection settings change or the license is removed.
// Safe to call on a nil receiver.
func (w *searchEngineWatcher) requestRestart() {
	if w == nil {
		return
	}
	atomic.StoreInt32(&w.forceRestart, 1)
	w.reevaluate()
}

// run is the watcher's main loop. It drives a simple state machine:
//
//	(start)
//	   |
//	   v
//	forceRestart? --yes--> Stop()
//	   |
//	   v
//	IsEnabled()? --no---> PARK (wait for notify or cancel)
//	   |
//	  yes
//	   |
//	   v
//	IsActive()? --yes---> HEALTH CHECK (periodic)
//	   |                        |
//	  no                 N consecutive failures
//	   |                        |
//	   v                   Stop() engine
//	RETRY <---------------------+
//	   |
//	   | wait (exponential backoff, interruptible)
//	   v
//	Start()
//	   |--error--> backoff*2 (capped), retry
//	   |--ok-----> reset backoff, HEALTH CHECK
//
//	Any state -- ctx.Done() --> EXIT
//	Any state -- reevaluate --> immediate re-evaluation
func (w *searchEngineWatcher) run(ctx context.Context, notifyCh <-chan struct{}) {
	if w.engine == nil {
		return
	}

	// Safety net: if the engine is still active when the goroutine exits
	// (e.g. Start() completed just as the context was canceled), stop it so
	// it is not left running with no goroutine to manage it.
	defer func() {
		if w.engine.IsActive() {
			w.ps.Log().Info("Search engine watcher: stopping engine on goroutine exit",
				mlog.String("engine", w.engine.GetName()))
			if err := w.engine.Stop(); err != nil {
				w.ps.Log().Warn("Search engine watcher: Stop() failed on goroutine exit",
					mlog.Err(err),
					mlog.String("engine", w.engine.GetName()))
			}
		}
	}()

	s := &watcherLoopState{backoff: searchEngineRetryInitial}
	rctx := request.EmptyContext(w.ps.logger)

	timer := time.NewTimer(0) // immediate first evaluation
	defer timer.Stop()

	for {
		if !w.waitForEvent(ctx, timer, notifyCh, s) {
			return
		}
		w.applyForceRestart()
		if w.parkIfDisabled(timer) {
			continue
		}
		if w.startIfInactive(ctx, timer, s) {
			continue
		}
		w.healthCheck(rctx, timer, s)
	}
}

// waitForEvent blocks until the timer fires, a notify arrives, or the context
// is cancelled. Returns false when the watcher should exit.
func (w *searchEngineWatcher) waitForEvent(ctx context.Context, timer *time.Timer, notifyCh <-chan struct{}, s *watcherLoopState) bool {
	for {
		select {
		case <-ctx.Done():
			return false
		case <-notifyCh:
			// Something changed -- re-evaluate immediately.
			timer.Reset(0)
			s.backoff = searchEngineRetryInitial
			s.consecutiveFailures = 0
			continue
		case <-timer.C:
		}

		// Prioritize shutdown: if select picked timer.C and ctx.Done()
		// simultaneously (random when both are ready), exit now.
		return ctx.Err() == nil
	}
}

// applyForceRestart checks and clears the force-restart flag. If set and the
// engine is active, it stops the engine so the next evaluation cycle will
// create a fresh client.
func (w *searchEngineWatcher) applyForceRestart() {
	if !atomic.CompareAndSwapInt32(&w.forceRestart, 1, 0) {
		return
	}
	if w.engine.IsActive() {
		w.ps.Log().Info("Search engine watcher: force-restart requested, stopping engine",
			mlog.String("engine", w.engine.GetName()))
		if err := w.engine.Stop(); err != nil {
			w.ps.Log().Warn("Search engine watcher: Stop() failed during force-restart",
				mlog.Err(err),
				mlog.String("engine", w.engine.GetName()))
		}
	}
}

// parkIfDisabled stops the engine (if active) and parks the watcher when the
// engine is disabled. Returns true when parked (caller should continue the
// loop without scheduling a timer -- the next wake comes from reevaluate).
func (w *searchEngineWatcher) parkIfDisabled(timer *time.Timer) bool {
	if w.engine.IsEnabled() {
		return false
	}
	if w.engine.IsActive() {
		if err := w.engine.Stop(); err != nil {
			w.ps.Log().Warn("Search engine watcher: Stop() returned error while disabling",
				mlog.Err(err),
				mlog.String("engine", w.engine.GetName()))
		}
	}
	w.ps.Log().Info("Search engine watcher: engine disabled, parking")
	timer.Stop()
	return true
}

// parkIfUnlicensed parks the watcher when the server has no Elasticsearch
// license. Returns true when parked (caller should continue the loop).
func (w *searchEngineWatcher) parkIfUnlicensed(timer *time.Timer) bool {
	license := w.ps.License()
	if license != nil && model.SafeDereference(license.Features.Elasticsearch) {
		return false
	}
	w.ps.Log().Info("Search engine watcher: engine not active (no Elasticsearch license), parking",
		mlog.String("engine", w.engine.GetName()))
	timer.Stop()
	return true
}

// startIfInactive attempts to start the engine when it is not active.
// Returns true when it handled the state (caller should continue the loop).
func (w *searchEngineWatcher) startIfInactive(ctx context.Context, timer *time.Timer, s *watcherLoopState) bool {
	if w.engine.IsActive() {
		return false
	}

	if err := w.engine.Start(ctx); err != nil {
		s.consecutiveFailures++
		w.ps.Log().Error("Search engine watcher: Start() failed, will retry",
			mlog.Err(err),
			mlog.Int("consecutive_failures", s.consecutiveFailures),
			mlog.Duration("next_backoff", s.backoff),
			mlog.String("engine", w.engine.GetName()))
		timer.Reset(s.backoff)
		s.backoff = min(s.backoff*2, searchEngineRetryMax)
		return true
	}

	if ctx.Err() != nil {
		return true // shutting down
	}

	// Start() returned nil but engine may not be active (e.g. no license).
	if !w.engine.IsActive() {
		if w.parkIfUnlicensed(timer) {
			return true
		}

		s.consecutiveFailures++
		w.ps.Log().Warn("Search engine watcher: Start() returned no error but engine is not active, will retry",
			mlog.Int("consecutive_failures", s.consecutiveFailures),
			mlog.Duration("next_backoff", s.backoff),
			mlog.String("engine", w.engine.GetName()))
		timer.Reset(s.backoff)
		s.backoff = min(s.backoff*2, searchEngineRetryMax)
		return true
	}

	w.ps.Log().Info("Search engine watcher: engine started successfully",
		mlog.String("engine", w.engine.GetName()))
	s.backoff = searchEngineRetryInitial
	s.consecutiveFailures = 0

	if model.SafeDereference(w.ps.Config().ElasticsearchSettings.EnableSearchPublicChannelsWithoutMembership) {
		engine := w.engine
		w.ps.Go(func() {
			w.ps.backfillPostsChannelType(engine)
		})
	}

	timer.Reset(searchEngineHealthInterval)
	return true
}

// healthCheck runs a periodic health check on an active engine. After
// reaching the consecutive failure threshold it stops the engine and
// schedules a retry; otherwise it schedules the next health check.
func (w *searchEngineWatcher) healthCheck(rctx request.CTX, timer *time.Timer, s *watcherLoopState) {
	// Save the previous state so that we can log any state change
	wasHealthy := w.engine.IsHealthy()

	if err := w.engine.HealthCheck(rctx); err != nil {
		// Mark unhealthy on the first failure so the broker skips this
		// engine immediately (searches fall back to DB, indexing is
		// deferred). The engine stays started until the consecutive
		// failure threshold is reached.
		w.engine.SetHealthy(false)
		if wasHealthy {
			w.ps.Log().Warn("Search engine health check failed: it is now marked as unhealthy",
				mlog.String("engine", w.engine.GetName()))
		}

		s.consecutiveFailures++
		if s.consecutiveFailures >= searchEngineHealthFailThreshold {
			w.ps.Log().Error("Search engine health check failed repeatedly; stopping engine",
				mlog.Err(err),
				mlog.Int("consecutive_failures", s.consecutiveFailures),
				mlog.String("engine", w.engine.GetName()))

			if stopErr := w.engine.Stop(); stopErr != nil {
				w.ps.Log().Warn("Search engine watcher: Stop() returned error (may already be stopped)",
					mlog.Err(stopErr),
					mlog.String("engine", w.engine.GetName()))
			}
			s.consecutiveFailures = 0
			s.backoff = searchEngineRetryInitial
			timer.Reset(s.backoff)
			return
		}

		w.ps.Log().Warn("Search engine health check failed",
			mlog.Err(err),
			mlog.Int("consecutive_failures", s.consecutiveFailures),
			mlog.Int("threshold", searchEngineHealthFailThreshold),
			mlog.String("engine", w.engine.GetName()))
	} else {
		w.engine.SetHealthy(true)
		if !wasHealthy {
			w.ps.Log().Info("Search engine health check succeeded: it is now marked as healthy",
				mlog.String("engine", w.engine.GetName()))
		}
		s.consecutiveFailures = 0
	}

	timer.Reset(searchEngineHealthInterval)
}