snapshot of https://github.com/mattermost/mattermost.git (master, 2026-09-16; blobs > 131072B omitted)

server/channels/jobs/active_users/scheduler.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package active_users
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const schedFreq = 10 * time.Minute
+
+func MakeScheduler(jobServer *jobs.JobServer) *jobs.PeriodicScheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.MetricsSettings.Enable
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeActiveUsers, schedFreq, isEnabled)
+}

server/channels/jobs/active_users/worker.go

@@ -0,0 +1,35 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package active_users
+
+import (
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/mattermost/mattermost/server/v8/einterfaces"
+)
+
+func MakeWorker(jobServer *jobs.JobServer, store store.Store, getMetrics func() einterfaces.MetricsInterface) *jobs.SimpleWorker {
+	const workerName = "ActiveUsers"
+
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.MetricsSettings.Enable
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		count, err := store.User().Count(model.UserCountOptions{IncludeDeleted: false})
+		if err != nil {
+			return err
+		}
+
+		if getMetrics() != nil {
+			getMetrics().ObserveEnabledUsers(count)
+		}
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/base_schedulers.go

@@ -0,0 +1,89 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"crypto/rand"
+	"math/big"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+)
+
+type PeriodicScheduler struct {
+	jobs        *JobServer
+	period      time.Duration
+	jobType     string
+	enabledFunc func(cfg *model.Config) bool
+}
+
+var _ Scheduler = (*DailyScheduler)(nil)
+
+func NewPeriodicScheduler(jobs *JobServer, jobType string, period time.Duration, enabledFunc func(cfg *model.Config) bool) *PeriodicScheduler {
+	return &PeriodicScheduler{
+		period:      period,
+		jobType:     jobType,
+		enabledFunc: enabledFunc,
+		jobs:        jobs,
+	}
+}
+
+func (scheduler *PeriodicScheduler) Enabled(cfg *model.Config) bool {
+	return scheduler.enabledFunc(cfg)
+}
+
+func (scheduler *PeriodicScheduler) NextScheduleTime(_ *model.Config, _ time.Time /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) *time.Time {
+	nextTime := time.Now().Add(getRandomDelay(jitterRange)).Add(scheduler.period)
+	return &nextTime
+}
+
+func (scheduler *PeriodicScheduler) ScheduleJob(rctx request.CTX, _ *model.Config /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) (*model.Job, *model.AppError) {
+	return scheduler.jobs.CreateJob(rctx, scheduler.jobType, nil)
+}
+
+type DailyScheduler struct {
+	jobs          *JobServer
+	startTimeFunc func(cfg *model.Config) *time.Time
+	jobType       string
+	enabledFunc   func(cfg *model.Config) bool
+}
+
+var _ Scheduler = (*DailyScheduler)(nil)
+
+func NewDailyScheduler(jobs *JobServer, jobType string, startTimeFunc func(cfg *model.Config) *time.Time, enabledFunc func(cfg *model.Config) bool) *DailyScheduler {
+	return &DailyScheduler{
+		startTimeFunc: startTimeFunc,
+		jobType:       jobType,
+		enabledFunc:   enabledFunc,
+		jobs:          jobs,
+	}
+}
+
+func (scheduler *DailyScheduler) Enabled(cfg *model.Config) bool {
+	return scheduler.enabledFunc(cfg)
+}
+
+func (scheduler *DailyScheduler) NextScheduleTime(cfg *model.Config, now time.Time /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) *time.Time {
+	scheduledTime := scheduler.startTimeFunc(cfg)
+	if scheduledTime == nil {
+		return nil
+	}
+
+	return GenerateNextStartDateTime(now, *scheduledTime)
+}
+
+func (scheduler *DailyScheduler) ScheduleJob(rctx request.CTX, _ *model.Config /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) (*model.Job, *model.AppError) {
+	return scheduler.jobs.CreateJob(rctx, scheduler.jobType, nil)
+}
+
+const jitterRange = 2000 // milliseconds
+
+func getRandomDelay(limit int64) time.Duration {
+	num, err := rand.Int(rand.Reader, big.NewInt(limit))
+	if err != nil {
+		return time.Millisecond
+	}
+	return time.Millisecond * time.Duration(num.Int64())
+}

server/channels/jobs/base_workers.go

@@ -0,0 +1,111 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"net/http"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+)
+
+type SimpleWorker struct {
+	name      string
+	stop      chan bool
+	stopped   chan bool
+	jobs      chan model.Job
+	jobServer *JobServer
+	logger    mlog.LoggerIFace
+	execute   func(logger mlog.LoggerIFace, job *model.Job) error
+	isEnabled func(cfg *model.Config) bool
+}
+
+func NewSimpleWorker(name string, jobServer *JobServer, execute func(logger mlog.LoggerIFace, job *model.Job) error, isEnabled func(cfg *model.Config) bool) *SimpleWorker {
+	worker := SimpleWorker{
+		name:      name,
+		stop:      make(chan bool, 1),
+		stopped:   make(chan bool, 1),
+		jobs:      make(chan model.Job),
+		jobServer: jobServer,
+		logger:    jobServer.Logger().With(mlog.String("worker_name", name)),
+		execute:   execute,
+		isEnabled: isEnabled,
+	}
+	return &worker
+}
+
+func (worker *SimpleWorker) Run() {
+	worker.logger.Debug("Worker started")
+
+	defer func() {
+		worker.logger.Debug("Worker finished")
+		worker.stopped <- true
+	}()
+
+	for {
+		select {
+		case <-worker.stop:
+			worker.logger.Debug("Worker received stop signal")
+			return
+		case job := <-worker.jobs:
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *SimpleWorker) Stop() {
+	worker.logger.Debug("Worker stopping")
+	worker.stop <- true
+	<-worker.stopped
+}
+
+func (worker *SimpleWorker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+func (worker *SimpleWorker) IsEnabled(cfg *model.Config) bool {
+	return worker.isEnabled(cfg)
+}
+
+func (worker *SimpleWorker) DoJob(job *model.Job) {
+	logger := worker.logger.With(JobLoggerFields(job)...)
+	logger.Debug("Worker: Received a new candidate job.")
+
+	var appErr *model.AppError
+	job, appErr = worker.jobServer.ClaimJob(job)
+	if appErr != nil {
+		logger.Warn("SimpleWorker experienced an error while trying to claim job", mlog.Err(appErr))
+		return
+	} else if job == nil {
+		return
+	}
+
+	err := worker.execute(logger, job)
+	if err != nil {
+		logger.Error("SimpleWorker: job execution error", mlog.Err(err))
+		worker.setJobError(logger, job, model.NewAppError("DoJob", "app.job.error", nil, "", http.StatusInternalServerError).Wrap(err))
+		return
+	}
+
+	logger.Debug("SimpleWorker: Job is complete")
+	worker.setJobSuccess(logger, job)
+}
+
+func (worker *SimpleWorker) setJobSuccess(logger mlog.LoggerIFace, job *model.Job) {
+	if err := worker.jobServer.SetJobProgress(job, 100); err != nil {
+		logger.Error("Worker: Failed to update progress for job", mlog.Err(err))
+		worker.setJobError(logger, job, err)
+	}
+
+	if err := worker.jobServer.SetJobSuccess(job); err != nil {
+		logger.Error("SimpleWorker: Failed to set success for job", mlog.Err(err))
+		worker.setJobError(logger, job, err)
+	}
+}
+
+func (worker *SimpleWorker) setJobError(logger mlog.LoggerIFace, job *model.Job, appError *model.AppError) {
+	if err := worker.jobServer.SetJobError(job, appError); err != nil {
+		logger.Error("SimpleWorker: Failed to set job error", mlog.Err(err))
+	}
+}

server/channels/jobs/base_workers_test.go

@@ -0,0 +1,47 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"errors"
+	"os"
+	"testing"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSimpleWorkerPanic(t *testing.T) {
+	if os.Getenv("ENABLE_FULLY_PARALLEL_TESTS") == "true" {
+		t.Parallel()
+	}
+
+	jobServer, mockStore, mockMetrics := makeJobServer(t)
+
+	job := &model.Job{
+		Id:   "job_id",
+		Type: "job_type",
+	}
+
+	exec := func(_ mlog.LoggerIFace, _ *model.Job) error {
+		return nil
+	}
+
+	isEnabled := func(_ *model.Config) bool {
+		return true
+	}
+
+	mockStore.JobStore.On("UpdateStatusOptimistically", "job_id", model.JobStatusPending, model.JobStatusInProgress).Return(&model.Job{Id: "job_id", Type: "job_type"}, nil)
+	mockStore.JobStore.On("UpdateOptimistically", mock.AnythingOfType("*model.Job"), model.JobStatusInProgress).Return(&model.Job{Id: "job_id", Type: "job_type"}, nil)
+	mockStore.JobStore.On("UpdateStatus", "job_id", "success").Return(nil, errors.New("test"))
+	mockMetrics.On("IncrementJobActive", "job_type")
+	mockMetrics.On("DecrementJobActive", "job_type")
+	sWorker := NewSimpleWorker("test", jobServer, exec, isEnabled)
+
+	require.NotPanics(t, func() {
+		sWorker.DoJob(job)
+	})
+}

server/channels/jobs/batch_migration_worker.go

@@ -0,0 +1,123 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+type BatchMigrationWorkerAppIFace interface {
+	GetClusterStatus(rctx request.CTX) ([]*model.ClusterInfo, error)
+}
+
+// BatchMigrationWorker processes database migration jobs in batches to help avoid table locks.
+//
+// It uses the jobs infrastructure to ensure only one node in the cluster runs the migration at
+// any given time, avoids running the migration until the cluster is uniform, and automatically
+// resets the migration if the cluster version diverges after starting.
+//
+// In principle, the job infrastructure is overkill for this kind of work, as there's a worker
+// created per migration. There's also complication with edge cases, like having to restart the
+// server in order to retry a failed migration job. Refactoring the job infrastructure is left as
+// a future exercise.
+type BatchMigrationWorker struct {
+	*BatchWorker
+	app              BatchMigrationWorkerAppIFace
+	migrationKey     string
+	doMigrationBatch func(data model.StringMap, store store.Store) (model.StringMap, bool, error)
+}
+
+// MakeBatchMigrationWorker creates a worker to process the given migration batch function.
+func MakeBatchMigrationWorker(
+	jobServer *JobServer,
+	store store.Store,
+	app BatchMigrationWorkerAppIFace,
+	migrationKey string,
+	timeBetweenBatches time.Duration,
+	doMigrationBatch func(data model.StringMap, store store.Store) (model.StringMap, bool, error),
+) *BatchMigrationWorker {
+	worker := &BatchMigrationWorker{
+		app:              app,
+		migrationKey:     migrationKey,
+		doMigrationBatch: doMigrationBatch,
+	}
+	worker.BatchWorker = MakeBatchWorker(jobServer, store, timeBetweenBatches, worker.doBatch)
+	return worker
+}
+
+func (worker *BatchMigrationWorker) doBatch(rctx request.CTX, job *model.Job) bool {
+	// Ensure the cluster remains in sync, otherwise we restart the job to
+	// ensure a complete migration. Technically, the cluster could go out of
+	// sync briefly within a batch, but we accept that risk.
+	if !worker.checkIsClusterInSync(rctx) {
+		worker.logger.Warn("Worker: Resetting job")
+		worker.resetJob(worker.logger, job)
+		return true
+	}
+
+	nextData, done, err := worker.doMigrationBatch(job.Data, worker.store)
+	if err != nil {
+		worker.logger.Error("Worker: Failed to do migration batch. Exiting", mlog.Err(err))
+		worker.setJobError(worker.logger, job, model.NewAppError("doMigrationBatch", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err))
+		return true
+	} else if done {
+		worker.logger.Info("Worker: Job is complete")
+		worker.setJobSuccess(worker.logger, job)
+		worker.markAsComplete()
+		return true
+	}
+
+	job.Data = nextData
+
+	// Migrations currently don't support reporting meaningful progress.
+	if err := worker.jobServer.SetJobProgress(job, 0); err != nil {
+		worker.logger.Error("Worker: Failed to set job progress", mlog.Err(err))
+		return false
+	}
+	return false
+}
+
+// checkIsClusterInSync returns true if all nodes in the cluster are running the same version,
+// logging a warning on the first mismatch found.
+func (worker *BatchMigrationWorker) checkIsClusterInSync(rctx request.CTX) bool {
+	clusterStatus, err := worker.app.GetClusterStatus(rctx)
+	if err != nil {
+		worker.logger.Error("Worker: Failed to get cluster status", mlog.Err(err))
+		return false
+	}
+	for i := 1; i < len(clusterStatus); i++ {
+		if clusterStatus[i].SchemaVersion != clusterStatus[0].SchemaVersion {
+			rctx.Logger().Warn(
+				"Worker: cluster not in sync",
+				mlog.String("schema_version_a", clusterStatus[0].SchemaVersion),
+				mlog.String("schema_version_b", clusterStatus[1].SchemaVersion),
+				mlog.String("server_ip_a", clusterStatus[0].IPAddress),
+				mlog.String("server_ip_b", clusterStatus[1].IPAddress),
+			)
+			return false
+		}
+	}
+
+	return true
+}
+
+// markAsComplete records a discrete migration key to prevent this job from ever running again.
+func (worker *BatchMigrationWorker) markAsComplete() {
+	system := model.System{
+		Name:  worker.migrationKey,
+		Value: "true",
+	}
+
+	// Note that if this fails, then the job would have still succeeded. We will spuriously
+	// run the job again in the future, but as migrations are idempotent it won't be an issue.
+	if err := worker.jobServer.Store.System().Save(&system); err != nil {
+		worker.logger.Error("Worker: Failed to mark migration as completed in the systems table.", mlog.String("migration_key", worker.migrationKey), mlog.Err(err))
+	}
+}

server/channels/jobs/batch_migration_worker_test.go

@@ -0,0 +1,182 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs_test
+
+import (
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type MockApp struct {
+	clusterInfo []*model.ClusterInfo
+}
+
+func (ma MockApp) GetClusterStatus(rctx request.CTX) ([]*model.ClusterInfo, error) {
+	return ma.clusterInfo, nil
+}
+
+func (ma *MockApp) SetInSync() {
+	ma.clusterInfo = nil
+	ma.clusterInfo = append(ma.clusterInfo, &model.ClusterInfo{
+		SchemaVersion: "a",
+	})
+	ma.clusterInfo = append(ma.clusterInfo, &model.ClusterInfo{
+		SchemaVersion: "a",
+	})
+}
+
+func (ma *MockApp) SetOutOfSync() {
+	ma.clusterInfo = nil
+	ma.clusterInfo = append(ma.clusterInfo, &model.ClusterInfo{
+		SchemaVersion: "a",
+	})
+	ma.clusterInfo = append(ma.clusterInfo, &model.ClusterInfo{
+		SchemaVersion: "b",
+	})
+}
+
+func TestBatchMigrationWorker(t *testing.T) {
+	mainHelper.Parallel(t)
+
+	setupBatchWorker := func(t *testing.T, th *TestHelper, mockApp *MockApp, doMigrationBatch func(model.StringMap, store.Store) (model.StringMap, bool, error)) (model.Worker, *model.Job) {
+		t.Helper()
+
+		worker := jobs.MakeBatchMigrationWorker(
+			th.Server.Jobs,
+			th.Server.Store(),
+			mockApp,
+			model.NewId(),
+			1*time.Second,
+			doMigrationBatch,
+		)
+		job := th.SetupBatchWorker(t, worker.BatchWorker)
+
+		return worker, job
+	}
+
+	stopWorker := func(t *testing.T, worker model.Worker) {
+		t.Helper()
+
+		stopped := make(chan bool, 1)
+		go func() {
+			worker.Stop()
+			close(stopped)
+		}()
+
+		waitDone(t, stopped, "worker did not stop")
+	}
+
+	assertJobResetEventually := func(t *testing.T, th *TestHelper, job *model.Job) {
+		t.Helper()
+		assert.EventuallyWithT(t, func(t *assert.CollectT) {
+			actualJob, appErr := th.Server.Jobs.GetJob(th.Context, job.Id)
+			require.Nil(t, appErr)
+			assert.Empty(t, actualJob.Progress, "expected no job progress")
+			assert.Empty(t, actualJob.Data, "expected no job data")
+		}, 5*time.Second, 250*time.Millisecond, "job never reset")
+	}
+
+	getBatchNumberFromData := func(t *testing.T, data model.StringMap) int {
+		t.Helper()
+
+		if data["batch_number"] == "" {
+			data["batch_number"] = "1"
+		}
+		batchNumber, err := strconv.Atoi(data["batch_number"])
+		require.NoError(t, err)
+
+		return batchNumber
+	}
+
+	getDataFromBatchNumber := func(batchNumber int) model.StringMap {
+		data := make(model.StringMap)
+		data["batch_number"] = strconv.Itoa(batchNumber)
+
+		return data
+	}
+
+	t.Run("done after three batches", func(t *testing.T) {
+		th := Setup(t).InitBasic(t)
+
+		mockApp := &MockApp{}
+
+		var worker model.Worker
+		var job *model.Job
+		worker, job = setupBatchWorker(t, th, mockApp, func(data model.StringMap, s store.Store) (model.StringMap, bool, error) {
+			batchNumber := getBatchNumberFromData(t, data)
+			require.LessOrEqual(t, batchNumber, 3, "only 3 batches should have run")
+
+			if batchNumber >= 3 {
+				go worker.Stop() // Shut down the worker when the job is done
+				return getDataFromBatchNumber(batchNumber), true, nil
+			}
+
+			batchNumber++
+			return getDataFromBatchNumber(batchNumber), false, nil
+		})
+
+		// Queue the work to be done
+		worker.JobChannel() <- *job
+
+		th.WaitForJobStatus(t, job, model.JobStatusSuccess)
+		th.WaitForBatchNumber(t, job, 3)
+	})
+
+	t.Run("clusters not in sync before first batch", func(t *testing.T) {
+		th := Setup(t).InitBasic(t)
+
+		mockApp := &MockApp{}
+		mockApp.SetOutOfSync()
+
+		var worker model.Worker
+		var job *model.Job
+		worker, job = setupBatchWorker(t, th, mockApp, func(model.StringMap, store.Store) (model.StringMap, bool, error) {
+			require.Fail(t, "migration batch should never run while clusters not in sync")
+
+			return nil, false, nil
+		})
+
+		// Queue the work to be done
+		worker.JobChannel() <- *job
+
+		th.WaitForJobStatus(t, job, model.JobStatusPending)
+		assertJobResetEventually(t, th, job)
+
+		stopWorker(t, worker)
+	})
+
+	t.Run("clusters not in sync after first batch", func(t *testing.T) {
+		th := Setup(t).InitBasic(t)
+
+		mockApp := &MockApp{}
+
+		var worker model.Worker
+		var job *model.Job
+		worker, job = setupBatchWorker(t, th, mockApp, func(data model.StringMap, s store.Store) (model.StringMap, bool, error) {
+			batchNumber := getBatchNumberFromData(t, data)
+			require.Equal(t, 1, batchNumber, "only batch 1 should have run")
+
+			mockApp.SetOutOfSync()
+			batchNumber++
+
+			return getDataFromBatchNumber(batchNumber), false, nil
+		})
+
+		// Queue the work to be done
+		worker.JobChannel() <- *job
+
+		th.WaitForJobStatus(t, job, model.JobStatusPending)
+		assertJobResetEventually(t, th, job)
+
+		stopWorker(t, worker)
+	})
+}

server/channels/jobs/batch_report_worker.go

@@ -0,0 +1,140 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/pkg/errors"
+)
+
+type BatchReportWorkerAppIFace interface {
+	SaveReportChunk(format string, prefix string, count int, reportData []model.ReportableObject) *model.AppError
+	CompileReportChunks(format string, prefix string, numberOfChunks int, headers []string) *model.AppError
+	SendReportToUser(rctx request.CTX, job *model.Job, format string) *model.AppError
+	CleanupReportChunks(format string, prefix string, numberOfChunks int) *model.AppError
+}
+
+type BatchReportWorker struct {
+	*BatchWorker
+	app          BatchReportWorkerAppIFace
+	reportFormat string
+	headers      []string
+	getData      func(jobData model.StringMap) ([]model.ReportableObject, model.StringMap, bool, error)
+}
+
+func MakeBatchReportWorker(
+	jobServer *JobServer,
+	store store.Store,
+	app BatchReportWorkerAppIFace,
+	timeBetweenBatches time.Duration,
+	reportFormat string,
+	headers []string,
+	getData func(jobData model.StringMap) ([]model.ReportableObject, model.StringMap, bool, error),
+) *BatchReportWorker {
+	worker := &BatchReportWorker{
+		app:          app,
+		reportFormat: reportFormat,
+		headers:      headers,
+		getData:      getData,
+	}
+	worker.BatchWorker = MakeBatchWorker(jobServer, store, timeBetweenBatches, worker.doBatch)
+	return worker
+}
+
+func (worker *BatchReportWorker) doBatch(rctx request.CTX, job *model.Job) bool {
+	reportData, nextData, done, err := worker.getData(job.Data)
+	if err != nil {
+		worker.logger.Error("Worker: Failed to get data for report batch. Exiting", mlog.Err(err))
+		worker.setJobError(worker.logger, job, model.NewAppError("doBatch", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err))
+		return true
+	} else if done {
+		if err = worker.complete(rctx, job); err != nil {
+			worker.logger.Error("Worker: Failed to finish the batch report. Exiting", mlog.Err(err))
+			worker.setJobError(worker.logger, job, model.NewAppError("doBatch", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err))
+		} else {
+			worker.logger.Info("Worker: Report job complete")
+			worker.setJobSuccess(worker.logger, job)
+		}
+
+		return true
+	}
+
+	err = worker.processChunk(job, reportData)
+	if err != nil {
+		worker.logger.Error("Worker: Failed to save report batch. Exiting", mlog.Err(err))
+		worker.setJobError(worker.logger, job, model.NewAppError("doBatch", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err))
+		return true
+	}
+
+	job.Data = nextData
+
+	// We might be able to add progress for this type of job in the future
+	// But for now we can just set to 0
+	if err := worker.jobServer.SetJobProgress(job, 0); err != nil {
+		worker.logger.Error("Worker: Failed to set job progress", mlog.Err(err))
+		return false
+	}
+	return false
+}
+
+func getFileCount(jobData model.StringMap) (int, error) {
+	if jobData["file_count"] != "" {
+		parsedFileCount, parseErr := strconv.Atoi(jobData["file_count"])
+		if parseErr != nil {
+			return 0, errors.Wrap(parseErr, "failed to parse file_count")
+		}
+		return parsedFileCount, nil
+	}
+
+	// Assume it hasn't been set
+	return 0, nil
+}
+
+func (worker *BatchReportWorker) processChunk(job *model.Job, reportData []model.ReportableObject) error {
+	fileCount, err := getFileCount(job.Data)
+	if err != nil {
+		return err
+	}
+
+	appErr := worker.app.SaveReportChunk(worker.reportFormat, job.Id, fileCount, reportData)
+	if appErr != nil {
+		return appErr
+	}
+
+	fileCount++
+	job.Data["file_count"] = strconv.Itoa(fileCount)
+
+	return nil
+}
+
+func (worker *BatchReportWorker) complete(rctx request.CTX, job *model.Job) error {
+	fileCount, err := getFileCount(job.Data)
+	if err != nil {
+		return err
+	}
+
+	appErr := worker.app.CompileReportChunks(worker.reportFormat, job.Id, fileCount, worker.headers)
+	if appErr != nil {
+		return appErr
+	}
+
+	defer func() {
+		if err := worker.app.CleanupReportChunks(worker.reportFormat, job.Id, fileCount); err != nil {
+			worker.logger.Error("Worker: Failed to cleanup report chunks", mlog.Err(err))
+		}
+	}()
+
+	if appErr = worker.app.SendReportToUser(rctx, job, worker.reportFormat); appErr != nil {
+		return appErr
+	}
+
+	return nil
+}

server/channels/jobs/batch_report_worker_test.go

@@ -0,0 +1,138 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs_test
+
+import (
+	"errors"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/stretchr/testify/require"
+)
+
+type ReportMockApp struct{}
+
+func (rma *ReportMockApp) SaveReportChunk(format string, prefix string, count int, reportData []model.ReportableObject) *model.AppError {
+	return nil
+}
+
+func (rma *ReportMockApp) CompileReportChunks(format string, prefix string, numberOfChunks int, headers []string) *model.AppError {
+	return nil
+}
+
+func (rma *ReportMockApp) SendReportToUser(rctx request.CTX, job *model.Job, format string) *model.AppError {
+	return nil
+}
+
+func (rma *ReportMockApp) CleanupReportChunks(format string, prefix string, numberOfChunks int) *model.AppError {
+	return nil
+}
+
+func TestBatchReportWorker(t *testing.T) {
+	mainHelper.Parallel(t)
+
+	setupBatchWorker := func(
+		t *testing.T,
+		th *TestHelper,
+		getData func(jobData model.StringMap) ([]model.ReportableObject, model.StringMap, bool, error),
+	) (*jobs.BatchReportWorker, *model.Job) {
+		t.Helper()
+
+		worker := jobs.MakeBatchReportWorker(
+			th.Server.Jobs,
+			th.Server.Store(),
+			&ReportMockApp{},
+			1*time.Second,
+			"csv",
+			[]string{},
+			getData)
+		job := th.SetupBatchWorker(t, worker.BatchWorker)
+		return worker, job
+	}
+
+	waitForFileCount := func(t *testing.T, th *TestHelper, job *model.Job, fileCount int) {
+		t.Helper()
+
+		require.Eventuallyf(t, func() bool {
+			actualJob, appErr := th.Server.Jobs.GetJob(th.Context, job.Id)
+			require.Nil(t, appErr)
+			require.Equal(t, job.Id, actualJob.Id)
+
+			finalFileCount, err := strconv.Atoi(actualJob.Data["file_count"])
+			require.NoError(t, err)
+			return finalFileCount == fileCount
+		}, 5*time.Second, 250*time.Millisecond, "job did not stop at batch %d", fileCount)
+	}
+
+	getFileCountFromData := func(t *testing.T, data model.StringMap) int {
+		t.Helper()
+
+		if data["file_count"] == "" {
+			return 0
+		}
+
+		fileCount, err := strconv.Atoi(data["file_count"])
+		require.NoError(t, err)
+
+		return fileCount
+	}
+
+	createData := func(th *TestHelper, data model.StringMap) model.StringMap {
+		data["requesting_user_id"] = th.SystemAdminUser.Id
+		return data
+	}
+
+	t.Run("should finish when the report is done, incrementing file count along the way", func(t *testing.T) {
+		mainHelper.Parallel(t)
+
+		th := Setup(t).InitBasic(t)
+
+		var worker model.Worker
+		var job *model.Job
+
+		iterations := 0
+
+		worker, job = setupBatchWorker(t, th, func(data model.StringMap) ([]model.ReportableObject, model.StringMap, bool, error) {
+			fileCount := getFileCountFromData(t, data)
+			require.Equal(t, iterations, fileCount)
+			require.LessOrEqual(t, fileCount, 3, "only 3 batches should have run")
+
+			iterations++
+
+			if fileCount >= 3 {
+				go worker.Stop() // Shut down the worker when the job is done
+				return []model.ReportableObject{}, createData(th, data), true, nil
+			}
+
+			return []model.ReportableObject{}, createData(th, data), false, nil
+		})
+
+		// Queue the work to be done
+		worker.JobChannel() <- *job
+
+		th.WaitForJobStatus(t, job, model.JobStatusSuccess)
+		waitForFileCount(t, th, job, 3)
+	})
+
+	t.Run("should fail job when get data throws an error", func(t *testing.T) {
+		mainHelper.Parallel(t)
+		th := Setup(t).InitBasic(t)
+
+		var worker model.Worker
+		var job *model.Job
+		worker, job = setupBatchWorker(t, th, func(data model.StringMap) ([]model.ReportableObject, model.StringMap, bool, error) {
+			go worker.Stop() // Shut down the worker right after this
+			return []model.ReportableObject{}, createData(th, data), false, errors.New("failed to fetch data")
+		})
+
+		// Queue the work to be done
+		worker.JobChannel() <- *job
+
+		th.WaitForJobStatus(t, job, model.JobStatusError)
+	})
+}

server/channels/jobs/batch_worker.go

@@ -0,0 +1,183 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"sync"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+type BatchWorker struct {
+	jobServer *JobServer
+	logger    mlog.LoggerIFace
+	store     store.Store
+
+	// stateMut protects stopCh and helps enforce
+	// ordering in case subsequent Run or Stop calls are made.
+	stateMut  sync.Mutex
+	stopCh    chan struct{}
+	stoppedCh chan bool
+	stopped   bool
+	jobs      chan model.Job
+
+	timeBetweenBatches time.Duration
+	doBatch            func(rctx request.CTX, job *model.Job) bool
+}
+
+// MakeBatchWorker creates a worker to process the given batch function.
+func MakeBatchWorker(
+	jobServer *JobServer,
+	store store.Store,
+	timeBetweenBatches time.Duration,
+	doBatch func(rctx request.CTX, job *model.Job) bool,
+) *BatchWorker {
+	return &BatchWorker{
+		jobServer:          jobServer,
+		logger:             jobServer.Logger(),
+		store:              store,
+		stoppedCh:          make(chan bool, 1),
+		jobs:               make(chan model.Job),
+		timeBetweenBatches: timeBetweenBatches,
+		doBatch:            doBatch,
+		stopped:            true,
+	}
+}
+
+// Run starts the worker dedicated to the unique migration batch job it will be given to process.
+func (worker *BatchWorker) Run() {
+	worker.stateMut.Lock()
+	// We have to re-assign the stop channel again, because
+	// it might happen that the job was restarted due to a config change.
+	if worker.stopped {
+		worker.stopped = false
+		worker.stopCh = make(chan struct{})
+	} else {
+		worker.stateMut.Unlock()
+		return
+	}
+	// Run is called from a separate goroutine and doesn't return.
+	// So we cannot Unlock in a defer clause.
+	worker.stateMut.Unlock()
+
+	worker.logger.Debug("Worker started")
+
+	defer func() {
+		worker.logger.Debug("Worker finished")
+		worker.stoppedCh <- true
+	}()
+
+	for {
+		select {
+		case <-worker.stopCh:
+			worker.logger.Debug("Worker received stop signal")
+			return
+		case job := <-worker.jobs:
+			worker.DoJob(&job)
+		}
+	}
+}
+
+// Stop interrupts the worker even if the migration has not yet completed.
+func (worker *BatchWorker) Stop() {
+	worker.stateMut.Lock()
+	defer worker.stateMut.Unlock()
+
+	// Set to close, and if already closed before, then return.
+	if worker.stopped {
+		return
+	}
+	worker.stopped = true
+
+	worker.logger.Debug("Worker stopping")
+	close(worker.stopCh)
+	<-worker.stoppedCh
+}
+
+// JobChannel is the means by which the jobs infrastructure provides the worker the job to execute.
+func (worker *BatchWorker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+// IsEnabled is always true for batches.
+func (worker *BatchWorker) IsEnabled(_ *model.Config) bool {
+	return true
+}
+
+// DoJob executes the job picked up through the job channel.
+//
+// Note that this is a lot of distracting machinery here to claim the job, then double check the
+// status, and keep the status up to date in line with job infrastrcuture semantics. Unless an
+// error occurs, this worker should hold onto the job until its completed.
+func (worker *BatchWorker) DoJob(job *model.Job) {
+	logger := worker.logger.With(mlog.Any("job", job))
+	logger.Debug("Worker received a new candidate job.")
+	defer worker.jobServer.HandleJobPanic(logger, job)
+
+	var appErr *model.AppError
+	job, appErr = worker.jobServer.ClaimJob(job)
+	if appErr != nil {
+		logger.Warn("Worker experienced an error while trying to claim job", mlog.Err(appErr))
+		return
+	} else if job == nil {
+		return
+	}
+
+	if job.Data == nil {
+		job.Data = make(model.StringMap)
+	}
+
+	c := request.EmptyContext(logger)
+
+	for {
+		select {
+		case <-worker.stopCh:
+			logger.Info("Worker: Batch has been canceled via Worker Stop. Setting the job back to pending.")
+			if err := worker.jobServer.SetJobPending(job); err != nil {
+				worker.logger.Error("Worker: Failed to mark job as pending", mlog.Err(err))
+			}
+			return
+		case <-time.After(worker.timeBetweenBatches):
+			if stop := worker.doBatch(c, job); stop {
+				return
+			}
+		}
+	}
+}
+
+// resetJob erases the data tracking the next batch to execute and returns the job status to
+// pending to allow the job infrastructure to requeue it.
+func (worker *BatchWorker) resetJob(logger mlog.LoggerIFace, job *model.Job) {
+	job.Data = nil
+	job.Progress = 0
+	job.Status = model.JobStatusPending
+
+	if _, err := worker.store.Job().UpdateOptimistically(job, model.JobStatusInProgress); err != nil {
+		worker.logger.Error("Worker: Failed to reset job data. May resume instead of restarting.", mlog.Err(err))
+	}
+}
+
+// setJobSuccess records the job as successful.
+func (worker *BatchWorker) setJobSuccess(logger mlog.LoggerIFace, job *model.Job) {
+	if err := worker.jobServer.SetJobProgress(job, 100); err != nil {
+		logger.Error("Worker: Failed to update progress for job", mlog.Err(err))
+		worker.setJobError(logger, job, err)
+	}
+
+	if err := worker.jobServer.SetJobSuccess(job); err != nil {
+		logger.Error("Worker: Failed to set success for job", mlog.Err(err))
+		worker.setJobError(logger, job, err)
+	}
+}
+
+// setJobError puts the job into an error state, preventing the job from running again.
+func (worker *BatchWorker) setJobError(logger mlog.LoggerIFace, job *model.Job, appError *model.AppError) {
+	if err := worker.jobServer.SetJobError(job, appError); err != nil {
+		logger.Error("Worker: Failed to set job error", mlog.Err(err))
+	}
+}

server/channels/jobs/batch_worker_test.go

@@ -0,0 +1,169 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs_test
+
+import (
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/stretchr/testify/require"
+)
+
+// TestBatchWorkerRace tests race conditions during the start/stop
+// cases of the batch worker. Use the -race flag while testing this.
+func TestBatchWorkerRace(t *testing.T) {
+	mainHelper.Parallel(t)
+
+	th := Setup(t)
+
+	worker := jobs.MakeBatchWorker(th.Server.Jobs, th.Server.Store(), 1*time.Second, func(rctx request.CTX, job *model.Job) bool {
+		return false
+	})
+
+	go worker.Run()
+	worker.Stop()
+}
+
+func TestBatchWorker(t *testing.T) {
+	mainHelper.Parallel(t)
+
+	createBatchWorker := func(t *testing.T, th *TestHelper, doBatch func(rctx request.CTX, job *model.Job) bool) (*jobs.BatchWorker, *model.Job) {
+		t.Helper()
+
+		worker := jobs.MakeBatchWorker(th.Server.Jobs, th.Server.Store(), 1*time.Second, doBatch)
+		job := th.SetupBatchWorker(t, worker)
+		return worker, job
+	}
+
+	getBatchNumberFromData := func(t *testing.T, data model.StringMap) int {
+		t.Helper()
+
+		batchNumber, err := strconv.Atoi(data["batch_number"])
+		require.NoError(t, err)
+
+		return batchNumber
+	}
+
+	incrementBatchNumber := func(t *testing.T, th *TestHelper, job *model.Job) {
+		t.Helper()
+
+		batchNumber, err := strconv.Atoi(job.Data["batch_number"])
+		require.NoError(t, err)
+
+		batchNumber++
+		job.Data["batch_number"] = strconv.Itoa(batchNumber)
+		appErr := th.Server.Jobs.SetJobProgress(job, 0)
+		require.Nil(t, appErr)
+	}
+
+	t.Run("stop after first batch", func(t *testing.T) {
+		mainHelper.Parallel(t)
+
+		th := Setup(t).InitBasic(t)
+
+		var worker *jobs.BatchWorker
+		worker, job := createBatchWorker(t, th, func(rctx request.CTX, job *model.Job) bool {
+			batchNumber := getBatchNumberFromData(t, job.Data)
+
+			require.Equal(t, 1, batchNumber, "only batch 1 should have run")
+
+			// Shut down the worker after the first batch to prevent subsequent ones.
+			if batchNumber >= 1 {
+				go worker.Stop()
+			} else {
+				incrementBatchNumber(t, th, job)
+			}
+
+			return false
+		})
+
+		// Queue the work to be done
+		worker.JobChannel() <- *job
+
+		th.WaitForJobStatus(t, job, model.JobStatusPending)
+		th.WaitForBatchNumber(t, job, 1)
+	})
+
+	t.Run("stop after second batch", func(t *testing.T) {
+		mainHelper.Parallel(t)
+
+		th := Setup(t).InitBasic(t)
+
+		var worker *jobs.BatchWorker
+		worker, job := createBatchWorker(t, th, func(rctx request.CTX, job *model.Job) bool {
+			batchNumber := getBatchNumberFromData(t, job.Data)
+
+			require.LessOrEqual(t, batchNumber, 2, "only batches 1 and 2 should have run")
+
+			// Shut down the worker after the second batch to prevent subsequent ones.
+			if batchNumber >= 2 {
+				go worker.Stop()
+			} else {
+				incrementBatchNumber(t, th, job)
+			}
+
+			return false
+		})
+
+		// Queue the work to be done
+		worker.JobChannel() <- *job
+
+		th.WaitForJobStatus(t, job, model.JobStatusPending)
+		th.WaitForBatchNumber(t, job, 2)
+	})
+
+	t.Run("done after first batch", func(t *testing.T) {
+		mainHelper.Parallel(t)
+
+		th := Setup(t).InitBasic(t)
+
+		var worker *jobs.BatchWorker
+		worker, job := createBatchWorker(t, th, func(rctx request.CTX, job *model.Job) bool {
+			batchNumber := getBatchNumberFromData(t, job.Data)
+			require.Equal(t, 1, batchNumber, "only batch 1 should have run")
+
+			if batchNumber >= 1 {
+				go worker.Stop() // Shut down the worker when the job is done
+				return true
+			}
+
+			incrementBatchNumber(t, th, job)
+			return false
+		})
+
+		// Queue the work to be done
+		worker.JobChannel() <- *job
+
+		th.WaitForBatchNumber(t, job, 1)
+	})
+
+	t.Run("done after three batches", func(t *testing.T) {
+		mainHelper.Parallel(t)
+
+		th := Setup(t).InitBasic(t)
+
+		var worker *jobs.BatchWorker
+		worker, job := createBatchWorker(t, th, func(rctx request.CTX, job *model.Job) bool {
+			batchNumber := getBatchNumberFromData(t, job.Data)
+			require.LessOrEqual(t, batchNumber, 3, "only 3 batches should have run")
+
+			if batchNumber >= 3 {
+				go worker.Stop() // Shut down the worker when the job is done
+				return true
+			}
+
+			incrementBatchNumber(t, th, job)
+			return false
+		})
+
+		// Queue the work to be done
+		worker.JobChannel() <- *job
+
+		th.WaitForBatchNumber(t, job, 3)
+	})
+}

server/channels/jobs/cleanup_desktop_tokens/scheduler.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cleanup_desktop_tokens
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const schedFreq = 1 * time.Hour
+
+func MakeScheduler(jobServer *jobs.JobServer) *jobs.PeriodicScheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return true
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeCleanupDesktopTokens, schedFreq, isEnabled)
+}

server/channels/jobs/cleanup_desktop_tokens/worker.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cleanup_desktop_tokens
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const jobName = "CleanupDesktopTokens"
+const maxAge = 5 * time.Minute
+
+func MakeWorker(jobServer *jobs.JobServer) *jobs.SimpleWorker {
+	isEnabled := func(cfg *model.Config) bool {
+		return true
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		return jobServer.Store.DesktopTokens().DeleteOlderThan(time.Now().Add(-maxAge).Unix())
+	}
+	worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/cleanup_expired_access_tokens/scheduler.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cleanup_expired_access_tokens
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const schedFreq = 1 * time.Hour
+
+func MakeScheduler(jobServer *jobs.JobServer) *jobs.PeriodicScheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.ServiceSettings.EnableUserAccessTokens
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeCleanupExpiredAccessTokens, schedFreq, isEnabled)
+}

server/channels/jobs/cleanup_expired_access_tokens/worker.go

@@ -0,0 +1,134 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cleanup_expired_access_tokens
+
+import (
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const (
+	workerName = "CleanupExpiredAccessTokens"
+	// batchLimit bounds both the number of rows fetched by GetExpiredBefore
+	// and the corresponding DeleteByIds call, keeping the transaction
+	// footprint bounded even when a large number of tokens expire at once.
+	batchLimit = 1000
+	// maxBatches caps the number of iterations per job execution so that very
+	// large expiry backlogs are drained across multiple scheduled runs rather
+	// than a single unbounded loop.
+	maxBatches = 1000
+)
+
+// expiredTokenStore is the subset of UserAccessTokenStore used by the worker.
+// Defined here rather than depending on the full store interface so the
+// orchestration logic can be unit-tested with a small fake.
+type expiredTokenStore interface {
+	GetExpiredBefore(cutoff int64, limit int) ([]*model.UserAccessToken, error)
+	DeleteByIds(tokenIDs []string) (int64, error)
+}
+
+// MakeWorker creates a worker that periodically deletes personal access tokens
+// whose ExpiresAt has passed, along with any sessions created from them.
+// The work is done in batches to keep the transaction footprint bounded.
+//
+// clearSessionCache is called for each affected user after their tokens are
+// deleted so that in-memory session caches don't serve stale sessions.
+//
+// notifyExpired is called with each batch of expired tokens immediately before
+// they are deleted, so the owner can be DMed that their token has been removed.
+// It is best-effort: any failure is handled internally and must not block the
+// cleanup. Note that after this change PAT expiry notifications come from two
+// jobs — the pre-expiry warning cascade (pat_expiry_notify) and this
+// at-deletion notice — so a future reader shouldn't be surprised to find an
+// expiry DM originating from the cleanup job.
+func MakeWorker(jobServer *jobs.JobServer, clearSessionCache func(userID string), notifyExpired func(rctx request.CTX, tokens []*model.UserAccessToken)) *jobs.SimpleWorker {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.ServiceSettings.EnableUserAccessTokens
+	}
+
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+		return cleanupExpired(
+			request.EmptyContext(logger),
+			jobServer.Store.UserAccessToken(),
+			clearSessionCache,
+			notifyExpired,
+			model.GetMillis(),
+			batchLimit,
+			maxBatches,
+		)
+	}
+
+	return jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+}
+
+// cleanupExpired drains expired personal access tokens in batches up to
+// maxBatches iterations. It is extracted from MakeWorker so that the batching
+// and error-propagation logic can be exercised by unit tests with a fake store.
+//
+// clearSessionCache is called for each unique user whose tokens were deleted so
+// that in-memory session caches don't continue serving the removed sessions.
+//
+// notifyExpired is called with each batch immediately before deletion so token
+// owners can be notified. It runs best-effort: it is invoked before the delete
+// so the token→owner mapping is still available, and the delete proceeds
+// regardless of what it does. The worst case (a crash between notify and
+// delete) is a single duplicate DM on the next run, which is acceptable. The
+// request context is passed through so the notifier logs under the job's logger.
+func cleanupExpired(
+	rctx request.CTX,
+	store expiredTokenStore,
+	clearSessionCache func(userID string),
+	notifyExpired func(rctx request.CTX, tokens []*model.UserAccessToken),
+	cutoff int64,
+	limit int,
+	maxIter int,
+) error {
+	var totalDeleted int64
+
+	for range maxIter {
+		expired, err := store.GetExpiredBefore(cutoff, limit)
+		if err != nil {
+			return err
+		}
+		if len(expired) == 0 {
+			break
+		}
+
+		ids := make([]string, len(expired))
+		userIDs := make(map[string]struct{}, len(expired))
+		for i, token := range expired {
+			ids[i] = token.Id
+			userIDs[token.UserId] = struct{}{}
+		}
+
+		if notifyExpired != nil {
+			notifyExpired(rctx, expired)
+		}
+
+		deleted, err := store.DeleteByIds(ids)
+		if err != nil {
+			return err
+		}
+		totalDeleted += deleted
+
+		for userID := range userIDs {
+			clearSessionCache(userID)
+		}
+
+		if len(expired) < limit {
+			break
+		}
+	}
+
+	rctx.Logger().Info(
+		"Cleaned up expired personal access tokens",
+		mlog.Int("deleted", int(totalDeleted)),
+		mlog.Int("cutoff", int(cutoff)),
+	)
+
+	return nil
+}

server/channels/jobs/cleanup_expired_access_tokens/worker_test.go

@@ -0,0 +1,202 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cleanup_expired_access_tokens
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+)
+
+// fakeStore implements expiredTokenStore. Each call to GetExpiredBefore pops
+// the next pre-programmed batch off batches, then returns the configured error
+// (which can be nil). DeleteByIds returns deleteCount/deleteErr and records
+// the ids it was called with.
+type fakeStore struct {
+	batches    [][]*model.UserAccessToken
+	getCalls   int
+	getErrAt   int // 1-based call index that returns getErr; 0 == no error
+	getErr     error
+	deleteCnt  int64
+	deleteErr  error
+	deletedIDs [][]string
+}
+
+func (f *fakeStore) GetExpiredBefore(_ int64, _ int) ([]*model.UserAccessToken, error) {
+	f.getCalls++
+	if f.getErrAt != 0 && f.getCalls == f.getErrAt {
+		return nil, f.getErr
+	}
+	if len(f.batches) == 0 {
+		return nil, nil
+	}
+	next := f.batches[0]
+	f.batches = f.batches[1:]
+	return next, nil
+}
+
+func (f *fakeStore) DeleteByIds(ids []string) (int64, error) {
+	f.deletedIDs = append(f.deletedIDs, ids)
+	if f.deleteErr != nil {
+		return 0, f.deleteErr
+	}
+	if f.deleteCnt != 0 {
+		return f.deleteCnt, nil
+	}
+	return int64(len(ids)), nil
+}
+
+func makeTokens(n int, base int64) []*model.UserAccessToken {
+	out := make([]*model.UserAccessToken, n)
+	for i := range n {
+		out[i] = &model.UserAccessToken{
+			Id:        model.NewId(),
+			UserId:    model.NewId(),
+			ExpiresAt: base + int64(i),
+			IsActive:  true,
+		}
+	}
+	return out
+}
+
+func nopClearSession(_ string) {}
+
+func nopNotify(_ request.CTX, _ []*model.UserAccessToken) {}
+
+func TestCleanupExpired(t *testing.T) {
+	rctx := request.EmptyContext(mlog.CreateConsoleTestLogger(t))
+
+	t.Run("happy path single batch", func(t *testing.T) {
+		tokens := makeTokens(3, 1000)
+		store := &fakeStore{batches: [][]*model.UserAccessToken{tokens}}
+
+		err := cleanupExpired(rctx, store, nopClearSession, nopNotify, 9999, 1000, 10)
+		require.NoError(t, err)
+
+		// Exactly one DeleteByIds call with the three token ids. A partial first
+		// batch (len < limit) must short-circuit the loop, so GetExpiredBefore is
+		// called exactly once.
+		require.Len(t, store.deletedIDs, 1)
+		require.Len(t, store.deletedIDs[0], 3)
+		require.Equal(t, 1, store.getCalls)
+	})
+
+	t.Run("empty result is no-op", func(t *testing.T) {
+		store := &fakeStore{} // no batches, no errors
+
+		err := cleanupExpired(rctx, store, nopClearSession, nopNotify, 9999, 1000, 10)
+		require.NoError(t, err)
+
+		require.Equal(t, 1, store.getCalls)
+		require.Empty(t, store.deletedIDs)
+	})
+
+	t.Run("full batch triggers next iteration", func(t *testing.T) {
+		const limit = 5
+		first := makeTokens(limit, 1000) // full batch -> loop continues
+		second := makeTokens(2, 2000)    // partial batch -> loop stops
+		store := &fakeStore{batches: [][]*model.UserAccessToken{first, second}}
+
+		err := cleanupExpired(rctx, store, nopClearSession, nopNotify, 9999, limit, 10)
+		require.NoError(t, err)
+
+		require.Equal(t, 2, store.getCalls)
+		require.Len(t, store.deletedIDs, 2)
+		require.Len(t, store.deletedIDs[0], limit)
+		require.Len(t, store.deletedIDs[1], 2)
+	})
+
+	t.Run("max iter cap", func(t *testing.T) {
+		const limit = 3
+		const maxIter = 2
+		store := &fakeStore{batches: [][]*model.UserAccessToken{
+			makeTokens(limit, 1000),
+			makeTokens(limit, 2000),
+			makeTokens(limit, 3000), // never reached
+		}}
+
+		err := cleanupExpired(rctx, store, nopClearSession, nopNotify, 9999, limit, maxIter)
+		require.NoError(t, err)
+
+		require.Equal(t, maxIter, store.getCalls, "loop must cap at maxIter")
+		require.Len(t, store.deletedIDs, maxIter)
+	})
+
+	t.Run("get error propagates", func(t *testing.T) {
+		wantErr := errors.New("get failed")
+		store := &fakeStore{
+			batches:  [][]*model.UserAccessToken{makeTokens(2, 1000)},
+			getErrAt: 1,
+			getErr:   wantErr,
+		}
+
+		err := cleanupExpired(rctx, store, nopClearSession, nopNotify, 9999, 1000, 10)
+		require.ErrorIs(t, err, wantErr)
+		require.Empty(t, store.deletedIDs, "delete must not run when get fails")
+	})
+
+	t.Run("delete error propagates", func(t *testing.T) {
+		wantErr := errors.New("delete failed")
+		store := &fakeStore{
+			batches:   [][]*model.UserAccessToken{makeTokens(2, 1000)},
+			deleteErr: wantErr,
+		}
+
+		err := cleanupExpired(rctx, store, nopClearSession, nopNotify, 9999, 1000, 10)
+		require.ErrorIs(t, err, wantErr)
+		require.Len(t, store.deletedIDs, 1, "DeleteByIds was called once before failing")
+	})
+
+	t.Run("session cache cleared for each unique user after delete", func(t *testing.T) {
+		sharedUserID := model.NewId()
+		tokens := []*model.UserAccessToken{
+			{Id: model.NewId(), UserId: sharedUserID, ExpiresAt: 1000, IsActive: true},
+			{Id: model.NewId(), UserId: sharedUserID, ExpiresAt: 1001, IsActive: true},
+			{Id: model.NewId(), UserId: model.NewId(), ExpiresAt: 1002, IsActive: true},
+		}
+		store := &fakeStore{batches: [][]*model.UserAccessToken{tokens}}
+
+		cleared := map[string]int{}
+		err := cleanupExpired(rctx, store, func(userID string) { cleared[userID]++ }, nopNotify, 9999, 1000, 10)
+		require.NoError(t, err)
+
+		require.Len(t, cleared, 2, "cache must be cleared for each unique user")
+		require.Equal(t, 1, cleared[sharedUserID], "each user cleared exactly once per batch")
+	})
+
+	t.Run("notify is called with each batch before delete", func(t *testing.T) {
+		const limit = 3
+		first := makeTokens(limit, 1000) // full batch -> loop continues
+		second := makeTokens(1, 2000)    // partial batch -> loop stops
+		store := &fakeStore{batches: [][]*model.UserAccessToken{first, second}}
+
+		var notified [][]*model.UserAccessToken
+		notify := func(_ request.CTX, tokens []*model.UserAccessToken) {
+			// Each batch must be notified before it is deleted: at the Nth
+			// notify call exactly N-1 prior batches should have been deleted.
+			require.Len(t, store.deletedIDs, len(notified), "notify must run before this batch's DeleteByIds")
+			notified = append(notified, tokens)
+		}
+
+		err := cleanupExpired(rctx, store, nopClearSession, notify, 9999, limit, 10)
+		require.NoError(t, err)
+
+		require.Len(t, notified, 2, "notify called once per batch")
+		require.Equal(t, first, notified[0])
+		require.Equal(t, second, notified[1])
+	})
+
+	t.Run("nil notify is tolerated", func(t *testing.T) {
+		store := &fakeStore{batches: [][]*model.UserAccessToken{makeTokens(2, 1000)}}
+
+		err := cleanupExpired(rctx, store, nopClearSession, nil, 9999, 1000, 10)
+		require.NoError(t, err)
+		require.Len(t, store.deletedIDs, 1)
+	})
+}

server/channels/jobs/delete_dms_preferences_migration/delete_dms_preferences_migration.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package delete_dms_preferences_migration
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/pkg/errors"
+)
+
+const (
+	timeBetweenBatches = 1 * time.Second
+)
+
+// MakeWorker creates a batch migration worker to delete empty drafts.
+func MakeWorker(jobServer *jobs.JobServer, store store.Store, app jobs.BatchMigrationWorkerAppIFace) model.Worker {
+	return jobs.MakeBatchMigrationWorker(
+		jobServer,
+		store,
+		app,
+		model.MigrationKeyDeleteDmsPreferences,
+		timeBetweenBatches,
+		doDeleteDmsPreferencesMigrationBatch,
+	)
+}
+
+// doDeleteDmsPreferencesMigrationBatch deletes any limit_visible_dms_gms preferences with a value > 40 and less than 1.
+func doDeleteDmsPreferencesMigrationBatch(data model.StringMap, store store.Store) (model.StringMap, bool, error) {
+	rowAffected, err := store.Preference().DeleteInvalidVisibleDmsGms()
+	if err != nil {
+		return nil, false, errors.Wrapf(err, "failed to delete invalid limit_visible_dms_gms")
+	}
+
+	if rowAffected == 0 {
+		return nil, true, nil
+	}
+
+	return nil, false, nil
+}

server/channels/jobs/delete_dms_preferences_migration/delete_dms_preferences_migration_test.go

@@ -0,0 +1,57 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package delete_dms_preferences_migration
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDoDeleteDMsPreferencesMigrationBatch(t *testing.T) {
+	t.Run("failure deleting batch", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		mockStore.PreferenceStore.On("DeleteInvalidVisibleDmsGms").Return(int64(0), errors.New("failure"))
+
+		data, done, err := doDeleteDmsPreferencesMigrationBatch(nil, mockStore)
+		require.EqualError(t, err, "failed to delete invalid limit_visible_dms_gms: failure")
+		assert.False(t, done)
+		assert.Nil(t, data)
+	})
+
+	t.Run("do batches", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		mockStore.PreferenceStore.On("DeleteInvalidVisibleDmsGms").Return(int64(10), nil)
+
+		data, done, err := doDeleteDmsPreferencesMigrationBatch(nil, mockStore)
+		require.NoError(t, err)
+		assert.False(t, done)
+		assert.Nil(t, data)
+	})
+
+	t.Run("done batches", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		mockStore.PreferenceStore.On("DeleteInvalidVisibleDmsGms").Return(int64(0), nil)
+
+		data, done, err := doDeleteDmsPreferencesMigrationBatch(nil, mockStore)
+		require.NoError(t, err)
+		assert.True(t, done)
+		assert.Nil(t, data)
+	})
+}

server/channels/jobs/delete_empty_drafts_migration/delete_empty_drafts_migration.go

@@ -0,0 +1,85 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package delete_empty_drafts_migration
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/pkg/errors"
+)
+
+const (
+	timeBetweenBatches = 1 * time.Second
+)
+
+// MakeWorker creates a batch migration worker to delete empty drafts.
+func MakeWorker(jobServer *jobs.JobServer, store store.Store, app jobs.BatchMigrationWorkerAppIFace) model.Worker {
+	return jobs.MakeBatchMigrationWorker(
+		jobServer,
+		store,
+		app,
+		model.MigrationKeyDeleteEmptyDrafts,
+		timeBetweenBatches,
+		doDeleteEmptyDraftsMigrationBatch,
+	)
+}
+
+// parseJobMetadata parses the opaque job metadata to return the information needed to decide which
+// batch to process next.
+func parseJobMetadata(data model.StringMap) (int64, string, error) {
+	createAt := int64(0)
+	if data["create_at"] != "" {
+		parsedCreateAt, parseErr := strconv.ParseInt(data["create_at"], 10, 64)
+		if parseErr != nil {
+			return 0, "", errors.Wrap(parseErr, "failed to parse create_at")
+		}
+		createAt = parsedCreateAt
+	}
+
+	userID := data["user_id"]
+
+	return createAt, userID, nil
+}
+
+// makeJobMetadata encodes the information needed to decide which batch to process next back into
+// the opaque job metadata.
+func makeJobMetadata(createAt int64, userID string) model.StringMap {
+	data := make(model.StringMap)
+	data["create_at"] = strconv.FormatInt(createAt, 10)
+	data["user_id"] = userID
+
+	return data
+}
+
+// doDeleteEmptyDraftsMigrationBatch iterates through all drafts, deleting empty drafts within each
+// batch keyed by the compound primary key (createAt, userID)
+func doDeleteEmptyDraftsMigrationBatch(data model.StringMap, store store.Store) (model.StringMap, bool, error) {
+	createAt, userID, err := parseJobMetadata(data)
+	if err != nil {
+		return nil, false, errors.Wrap(err, "failed to parse job metadata")
+	}
+
+	// Determine the /next/ (createAt, userId) by finding the last record in the batch we're
+	// about to delete.
+	nextCreateAt, nextUserID, err := store.Draft().GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration(createAt, userID)
+	if err != nil {
+		return nil, false, errors.Wrapf(err, "failed to get the next batch (create_at=%v, user_id=%v)", createAt, userID)
+	}
+
+	// If we get the nil values, it means the batch was empty and we're done.
+	if nextCreateAt == 0 && nextUserID == "" {
+		return nil, true, nil
+	}
+
+	err = store.Draft().DeleteEmptyDraftsByCreateAtAndUserId(createAt, userID)
+	if err != nil {
+		return nil, false, errors.Wrapf(err, "failed to delete empty drafts (create_at=%v, user_id=%v)", createAt, userID)
+	}
+
+	return makeJobMetadata(nextCreateAt, nextUserID), false, nil
+}

server/channels/jobs/delete_empty_drafts_migration/delete_empty_drafts_migration_test.go

@@ -0,0 +1,180 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package delete_empty_drafts_migration
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestJobMetadata(t *testing.T) {
+	t.Run("parse nil data", func(t *testing.T) {
+		var data model.StringMap
+		createAt, userID, err := parseJobMetadata(data)
+		require.NoError(t, err)
+		assert.Empty(t, createAt)
+		assert.Empty(t, userID)
+	})
+
+	t.Run("parse invalid create_at", func(t *testing.T) {
+		data := make(model.StringMap)
+		data["user_id"] = "user_id"
+		data["create_at"] = "invalid"
+		_, _, err := parseJobMetadata(data)
+		require.Error(t, err)
+	})
+
+	t.Run("parse valid", func(t *testing.T) {
+		data := make(model.StringMap)
+		data["user_id"] = "user_id"
+		data["create_at"] = "1695918431"
+
+		createAt, userID, err := parseJobMetadata(data)
+		require.NoError(t, err)
+		assert.EqualValues(t, 1695918431, createAt)
+		assert.Equal(t, "user_id", userID)
+	})
+
+	t.Run("parse/make", func(t *testing.T) {
+		data := makeJobMetadata(1695918431, "user_id")
+		assert.Equal(t, "1695918431", data["create_at"])
+		assert.Equal(t, "user_id", data["user_id"])
+
+		createAt, userID, err := parseJobMetadata(data)
+		require.NoError(t, err)
+		assert.EqualValues(t, 1695918431, createAt)
+		assert.Equal(t, "user_id", userID)
+	})
+}
+
+func TestDoDeleteEmptyDraftsMigrationBatch(t *testing.T) {
+	t.Run("invalid job metadata", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		data := make(model.StringMap)
+		data["user_id"] = "user_id"
+		data["create_at"] = "invalid"
+		data, done, err := doDeleteEmptyDraftsMigrationBatch(data, mockStore)
+		require.Error(t, err)
+		assert.False(t, done)
+		assert.Nil(t, data)
+	})
+
+	t.Run("failure getting next offset", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		createAt, userID := int64(1695920000), "user_id_1"
+		nextCreateAt, nextUserID := int64(0), ""
+
+		mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, errors.New("failure"))
+
+		data, done, err := doDeleteEmptyDraftsMigrationBatch(makeJobMetadata(createAt, userID), mockStore)
+		require.EqualError(t, err, "failed to get the next batch (create_at=1695920000, user_id=user_id_1): failure")
+		assert.False(t, done)
+		assert.Nil(t, data)
+	})
+
+	t.Run("failure deleting batch", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		createAt, userID := int64(1695920000), "user_id_1"
+		nextCreateAt, nextUserID := int64(1695922034), "user_id_2"
+
+		mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, nil)
+		mockStore.DraftStore.On("DeleteEmptyDraftsByCreateAtAndUserId", createAt, userID).Return(errors.New("failure"))
+
+		data, done, err := doDeleteEmptyDraftsMigrationBatch(makeJobMetadata(createAt, userID), mockStore)
+		require.EqualError(t, err, "failed to delete empty drafts (create_at=1695920000, user_id=user_id_1): failure")
+		assert.False(t, done)
+		assert.Nil(t, data)
+	})
+
+	t.Run("do first batch (nil job metadata)", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		createAt, userID := int64(0), ""
+		nextCreateAt, nextUserID := int64(1695922034), "user_id_2"
+
+		mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, nil)
+		mockStore.DraftStore.On("DeleteEmptyDraftsByCreateAtAndUserId", createAt, userID).Return(nil)
+
+		data, done, err := doDeleteEmptyDraftsMigrationBatch(nil, mockStore)
+		require.NoError(t, err)
+		assert.False(t, done)
+		assert.Equal(t, model.StringMap{
+			"create_at": "1695922034",
+			"user_id":   "user_id_2",
+		}, data)
+	})
+
+	t.Run("do first batch (empty job metadata)", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		createAt, userID := int64(0), ""
+		nextCreateAt, nextUserID := int64(1695922034), "user_id_2"
+
+		mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, nil)
+		mockStore.DraftStore.On("DeleteEmptyDraftsByCreateAtAndUserId", createAt, userID).Return(nil)
+
+		data, done, err := doDeleteEmptyDraftsMigrationBatch(model.StringMap{}, mockStore)
+		require.NoError(t, err)
+		assert.False(t, done)
+		assert.Equal(t, makeJobMetadata(nextCreateAt, nextUserID), data)
+	})
+
+	t.Run("do batch", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		createAt, userID := int64(1695922000), "user_id_1"
+		nextCreateAt, nextUserID := int64(1695922034), "user_id_2"
+
+		mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, nil)
+		mockStore.DraftStore.On("DeleteEmptyDraftsByCreateAtAndUserId", createAt, userID).Return(nil)
+
+		data, done, err := doDeleteEmptyDraftsMigrationBatch(makeJobMetadata(createAt, userID), mockStore)
+		require.NoError(t, err)
+		assert.False(t, done)
+		assert.Equal(t, makeJobMetadata(nextCreateAt, nextUserID), data)
+	})
+
+	t.Run("done batches", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		createAt, userID := int64(1695922000), "user_id_1"
+		nextCreateAt, nextUserID := int64(0), ""
+
+		mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, nil)
+
+		data, done, err := doDeleteEmptyDraftsMigrationBatch(makeJobMetadata(createAt, userID), mockStore)
+		require.NoError(t, err)
+		assert.True(t, done)
+		assert.Nil(t, data)
+	})
+}

server/channels/jobs/delete_expired_posts/scheduler.go

@@ -0,0 +1,23 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package delete_expired_posts
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+func MakeScheduler(jobServer *jobs.JobServer) *jobs.PeriodicScheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		featureFlagEnabled := cfg.FeatureFlags.BurnOnRead
+		serviceSettingEnabled := model.SafeDereference(cfg.ServiceSettings.EnableBurnOnRead)
+
+		return featureFlagEnabled && serviceSettingEnabled
+	}
+
+	schedFreq := time.Duration(model.SafeDereference(jobServer.Config().ServiceSettings.BurnOnReadSchedulerFrequencySeconds)) * time.Second
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeDeleteExpiredPosts, schedFreq, isEnabled)
+}

server/channels/jobs/delete_expired_posts/worker.go

@@ -0,0 +1,80 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package delete_expired_posts
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+const (
+	expiredPostsBatchSize        = 100
+	expiredPostsJobBatchWaitTime = 100 * time.Millisecond
+)
+
+type AppIface interface {
+	DeletePost(rctx request.CTX, postID, deleteByID string) (*model.Post, *model.AppError)
+	PermanentDeletePostDataRetainStub(rctx request.CTX, post *model.Post, deleteByID string) (*model.PostDeletionReport, *model.AppError)
+	GetSinglePost(rctx request.CTX, postID string, includeDeleted bool) (*model.Post, *model.AppError)
+	GetPostsByIds(postIDs []string) ([]*model.Post, int64, *model.AppError)
+}
+
+func MakeWorker(jobServer *jobs.JobServer, store store.Store, app AppIface) *jobs.SimpleWorker {
+	const workerName = "DeleteExpiredPosts"
+
+	isEnabled := func(cfg *model.Config) bool {
+		return model.SafeDereference(cfg.ServiceSettings.EnableBurnOnRead)
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		if job.Data == nil {
+			job.Data = make(model.StringMap)
+		}
+
+		deletedPostIDs := make([]string, 0)
+		lastPostId := ""
+		for {
+			time.Sleep(expiredPostsJobBatchWaitTime)
+			postIDs, err := store.TemporaryPost().GetExpiredPosts(request.EmptyContext(logger), lastPostId, expiredPostsBatchSize)
+			if err != nil {
+				return err
+			}
+
+			if len(postIDs) == 0 {
+				break
+			}
+
+			lastPostId = postIDs[len(postIDs)-1]
+
+			expiredPosts, _, appErr := app.GetPostsByIds(postIDs)
+			if appErr != nil {
+				logger.Error("Failed to get expired posts by IDs", mlog.Err(appErr))
+				return appErr
+			}
+
+			for _, post := range expiredPosts {
+				_, appErr = app.PermanentDeletePostDataRetainStub(request.EmptyContext(logger), post, "")
+				if appErr != nil {
+					logger.Error("Failed to delete expired post", mlog.Err(appErr), mlog.String("post_id", post.Id))
+					continue
+				}
+				deletedPostIDs = append(deletedPostIDs, post.Id)
+			}
+		}
+
+		deletedPostIDsJSON, err := json.Marshal(deletedPostIDs)
+		if err != nil {
+			logger.Error("Failed to marshal deleted post IDs", mlog.Err(err))
+			return err
+		}
+		job.Data["deleted_post_ids"] = string(deletedPostIDsJSON)
+		return nil
+	}
+	return jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+}

server/channels/jobs/delete_orphan_drafts_migration/delete_orphan_drafts_migration.go

@@ -0,0 +1,85 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package delete_orphan_drafts_migration
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/pkg/errors"
+)
+
+const (
+	timeBetweenBatches = 1 * time.Second
+)
+
+// MakeWorker creates a batch migration worker to delete empty drafts.
+func MakeWorker(jobServer *jobs.JobServer, store store.Store, app jobs.BatchMigrationWorkerAppIFace) model.Worker {
+	return jobs.MakeBatchMigrationWorker(
+		jobServer,
+		store,
+		app,
+		model.MigrationKeyDeleteOrphanDrafts,
+		timeBetweenBatches,
+		doDeleteOrphanDraftsMigrationBatch,
+	)
+}
+
+// parseJobMetadata parses the opaque job metadata to return the information needed to decide which
+// batch to process next.
+func parseJobMetadata(data model.StringMap) (int64, string, error) {
+	createAt := int64(0)
+	if data["create_at"] != "" {
+		parsedCreateAt, parseErr := strconv.ParseInt(data["create_at"], 10, 64)
+		if parseErr != nil {
+			return 0, "", errors.Wrap(parseErr, "failed to parse create_at")
+		}
+		createAt = parsedCreateAt
+	}
+
+	userID := data["user_id"]
+
+	return createAt, userID, nil
+}
+
+// makeJobMetadata encodes the information needed to decide which batch to process next back into
+// the opaque job metadata.
+func makeJobMetadata(createAt int64, userID string) model.StringMap {
+	data := make(model.StringMap)
+	data["create_at"] = strconv.FormatInt(createAt, 10)
+	data["user_id"] = userID
+
+	return data
+}
+
+// doDeleteOrphanDraftsMigrationBatch iterates through all drafts, deleting orphan drafts within each
+// batch keyed by the compound primary key (createAt, userID)
+func doDeleteOrphanDraftsMigrationBatch(data model.StringMap, store store.Store) (model.StringMap, bool, error) {
+	createAt, userID, err := parseJobMetadata(data)
+	if err != nil {
+		return nil, false, errors.Wrap(err, "failed to parse job metadata")
+	}
+
+	// Determine the /next/ (createAt, userId) by finding the last record in the batch we're
+	// about to delete.
+	nextCreateAt, nextUserID, err := store.Draft().GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration(createAt, userID)
+	if err != nil {
+		return nil, false, errors.Wrapf(err, "failed to get the next batch (create_at=%v, user_id=%v)", createAt, userID)
+	}
+
+	// If we get the nil values, it means the batch was empty and we're done.
+	if nextCreateAt == 0 && nextUserID == "" {
+		return nil, true, nil
+	}
+
+	err = store.Draft().DeleteOrphanDraftsByCreateAtAndUserId(createAt, userID)
+	if err != nil {
+		return nil, false, errors.Wrapf(err, "failed to delete orphan drafts (create_at=%v, user_id=%v)", createAt, userID)
+	}
+
+	return makeJobMetadata(nextCreateAt, nextUserID), false, nil
+}

server/channels/jobs/delete_orphan_drafts_migration/delete_orphan_drafts_migration_test.go

@@ -0,0 +1,180 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package delete_orphan_drafts_migration
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestJobMetadata(t *testing.T) {
+	t.Run("parse nil data", func(t *testing.T) {
+		var data model.StringMap
+		createAt, userID, err := parseJobMetadata(data)
+		require.NoError(t, err)
+		assert.Empty(t, createAt)
+		assert.Empty(t, userID)
+	})
+
+	t.Run("parse invalid create_at", func(t *testing.T) {
+		data := make(model.StringMap)
+		data["user_id"] = "user_id"
+		data["create_at"] = "invalid"
+		_, _, err := parseJobMetadata(data)
+		require.Error(t, err)
+	})
+
+	t.Run("parse valid", func(t *testing.T) {
+		data := make(model.StringMap)
+		data["user_id"] = "user_id"
+		data["create_at"] = "1695918431"
+
+		createAt, userID, err := parseJobMetadata(data)
+		require.NoError(t, err)
+		assert.EqualValues(t, 1695918431, createAt)
+		assert.Equal(t, "user_id", userID)
+	})
+
+	t.Run("parse/make", func(t *testing.T) {
+		data := makeJobMetadata(1695918431, "user_id")
+		assert.Equal(t, "1695918431", data["create_at"])
+		assert.Equal(t, "user_id", data["user_id"])
+
+		createAt, userID, err := parseJobMetadata(data)
+		require.NoError(t, err)
+		assert.EqualValues(t, 1695918431, createAt)
+		assert.Equal(t, "user_id", userID)
+	})
+}
+
+func TestDoDeleteOrphanDraftsMigrationBatch(t *testing.T) {
+	t.Run("invalid job metadata", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		data := make(model.StringMap)
+		data["user_id"] = "user_id"
+		data["create_at"] = "invalid"
+		data, done, err := doDeleteOrphanDraftsMigrationBatch(data, mockStore)
+		require.Error(t, err)
+		assert.False(t, done)
+		assert.Nil(t, data)
+	})
+
+	t.Run("failure getting next offset", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		createAt, userID := int64(1695920000), "user_id_1"
+		nextCreateAt, nextUserID := int64(0), ""
+
+		mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, errors.New("failure"))
+
+		data, done, err := doDeleteOrphanDraftsMigrationBatch(makeJobMetadata(createAt, userID), mockStore)
+		require.EqualError(t, err, "failed to get the next batch (create_at=1695920000, user_id=user_id_1): failure")
+		assert.False(t, done)
+		assert.Nil(t, data)
+	})
+
+	t.Run("failure deleting batch", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		createAt, userID := int64(1695920000), "user_id_1"
+		nextCreateAt, nextUserID := int64(1695922034), "user_id_2"
+
+		mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, nil)
+		mockStore.DraftStore.On("DeleteOrphanDraftsByCreateAtAndUserId", createAt, userID).Return(errors.New("failure"))
+
+		data, done, err := doDeleteOrphanDraftsMigrationBatch(makeJobMetadata(createAt, userID), mockStore)
+		require.EqualError(t, err, "failed to delete orphan drafts (create_at=1695920000, user_id=user_id_1): failure")
+		assert.False(t, done)
+		assert.Nil(t, data)
+	})
+
+	t.Run("do first batch (nil job metadata)", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		createAt, userID := int64(0), ""
+		nextCreateAt, nextUserID := int64(1695922034), "user_id_2"
+
+		mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, nil)
+		mockStore.DraftStore.On("DeleteOrphanDraftsByCreateAtAndUserId", createAt, userID).Return(nil)
+
+		data, done, err := doDeleteOrphanDraftsMigrationBatch(nil, mockStore)
+		require.NoError(t, err)
+		assert.False(t, done)
+		assert.Equal(t, model.StringMap{
+			"create_at": "1695922034",
+			"user_id":   "user_id_2",
+		}, data)
+	})
+
+	t.Run("do first batch (empty job metadata)", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		createAt, userID := int64(0), ""
+		nextCreateAt, nextUserID := int64(1695922034), "user_id_2"
+
+		mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, nil)
+		mockStore.DraftStore.On("DeleteOrphanDraftsByCreateAtAndUserId", createAt, userID).Return(nil)
+
+		data, done, err := doDeleteOrphanDraftsMigrationBatch(model.StringMap{}, mockStore)
+		require.NoError(t, err)
+		assert.False(t, done)
+		assert.Equal(t, makeJobMetadata(nextCreateAt, nextUserID), data)
+	})
+
+	t.Run("do batch", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		createAt, userID := int64(1695922000), "user_id_1"
+		nextCreateAt, nextUserID := int64(1695922034), "user_id_2"
+
+		mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, nil)
+		mockStore.DraftStore.On("DeleteOrphanDraftsByCreateAtAndUserId", createAt, userID).Return(nil)
+
+		data, done, err := doDeleteOrphanDraftsMigrationBatch(makeJobMetadata(createAt, userID), mockStore)
+		require.NoError(t, err)
+		assert.False(t, done)
+		assert.Equal(t, makeJobMetadata(nextCreateAt, nextUserID), data)
+	})
+
+	t.Run("done batches", func(t *testing.T) {
+		mockStore := &storetest.Store{}
+		t.Cleanup(func() {
+			mockStore.AssertExpectations(t)
+		})
+
+		createAt, userID := int64(1695922000), "user_id_1"
+		nextCreateAt, nextUserID := int64(0), ""
+
+		mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, nil)
+
+		data, done, err := doDeleteOrphanDraftsMigrationBatch(makeJobMetadata(createAt, userID), mockStore)
+		require.NoError(t, err)
+		assert.True(t, done)
+		assert.Nil(t, data)
+	})
+}

server/channels/jobs/expirynotify/scheduler.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package expirynotify
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const schedFreq = 10 * time.Minute
+
+func MakeScheduler(jobServer *jobs.JobServer) *jobs.PeriodicScheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.ServiceSettings.ExtendSessionLengthWithActivity
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeExpiryNotify, schedFreq, isEnabled)
+}

server/channels/jobs/expirynotify/worker.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package expirynotify
+
+import (
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+func MakeWorker(jobServer *jobs.JobServer, notifySessionsExpired func() error) *jobs.SimpleWorker {
+	const workerName = "ExpiryNotify"
+
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.ServiceSettings.ExtendSessionLengthWithActivity
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		return notifySessionsExpired()
+	}
+	return jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+}

server/channels/jobs/export_delete/scheduler.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package export_delete
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const schedFreq = 24 * time.Hour
+
+func MakeScheduler(jobServer *jobs.JobServer) *jobs.PeriodicScheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.ExportSettings.Directory != "" && *cfg.ExportSettings.RetentionDays > 0
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeExportDelete, schedFreq, isEnabled)
+}

server/channels/jobs/export_delete/worker.go

@@ -0,0 +1,77 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package export_delete
+
+import (
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/wiggin77/merror"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/configservice"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+type AppIface interface {
+	configservice.ConfigService
+	ListExportDirectory(path string) ([]string, *model.AppError)
+	ExportFileModTime(path string) (time.Time, *model.AppError)
+	RemoveExportFile(path string) *model.AppError
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface) *jobs.SimpleWorker {
+	const workerName = "ExportDelete"
+
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.ExportSettings.Directory != "" && *cfg.ExportSettings.RetentionDays > 0
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		exportPath := *app.Config().ExportSettings.Directory
+		retentionTime := time.Duration(*app.Config().ExportSettings.RetentionDays) * 24 * time.Hour
+		exports, appErr := app.ListExportDirectory(exportPath)
+		if appErr != nil {
+			return appErr
+		}
+
+		errors := merror.New()
+		for i := range exports {
+			filename := filepath.Base(exports[i])
+
+			// Ignore files that were not created by the bulk export command
+			if !strings.HasSuffix(filename, "_export.zip") {
+				continue
+			}
+
+			modTime, appErr := app.ExportFileModTime(filepath.Join(exportPath, filename))
+			if appErr != nil {
+				logger.Debug("Worker: Failed to get file modification time",
+					mlog.Err(appErr), mlog.String("export", exports[i]))
+				errors.Append(appErr)
+				continue
+			}
+
+			if time.Now().After(modTime.Add(retentionTime)) {
+				// remove file data from storage.
+				if appErr := app.RemoveExportFile(exports[i]); appErr != nil {
+					logger.Debug("Worker: Failed to remove file",
+						mlog.Err(appErr), mlog.String("export", exports[i]))
+					errors.Append(appErr)
+					continue
+				}
+			}
+		}
+
+		if err := errors.ErrorOrNil(); err != nil {
+			logger.Warn("Worker: errors occurred", mlog.Err(err))
+		}
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/export_delete_test.go

@@ -0,0 +1,102 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestExportDelete(t *testing.T) {
+	mainHelper.Parallel(t)
+	// Create a temporary export directory
+	fileSettingsDir, err := os.MkdirTemp("", "")
+	require.NoError(t, err)
+	relExportDir := "./export"
+	exportDir := filepath.Join(fileSettingsDir, relExportDir)
+
+	t.Cleanup(func() {
+		err = os.RemoveAll(fileSettingsDir)
+		assert.NoError(t, err)
+	})
+
+	retentionDays := 1
+
+	updateConfig := func(cfg *model.Config) {
+		*cfg.FileSettings.DriverName = model.ImageDriverLocal
+		*cfg.FileSettings.Directory = fileSettingsDir
+		*cfg.ExportSettings.Directory = relExportDir
+		*cfg.ExportSettings.RetentionDays = retentionDays
+	}
+
+	th := SetupWithUpdateCfg(t, updateConfig)
+
+	// Create test files with different timestamps
+	files := []string{
+		"old_export.zip",
+		"recent_export.zip",
+		"normal.txt",
+		"test_export.zip",
+		"data.json",
+	}
+
+	// Create directories that should not be deleted
+	dirs := []string{
+		"data",
+		"data/subfolder",
+	}
+
+	for _, dir := range dirs {
+		err = os.MkdirAll(filepath.Join(exportDir, dir), 0755)
+		require.NoError(t, err)
+		// Add a file in each directory that should not be deleted
+		err = os.WriteFile(filepath.Join(exportDir, dir, "file.txt"), []byte("test"), 0644)
+		require.NoError(t, err)
+	}
+
+	for _, file := range files {
+		err = os.WriteFile(filepath.Join(exportDir, file), []byte("test"), 0644)
+		require.NoError(t, err)
+	}
+
+	// Set old timestamps for files that should be deleted
+	oldTime := time.Now().Add(-(time.Duration(retentionDays) * 24 * time.Hour) - 1*time.Hour)
+	err = os.Chtimes(filepath.Join(exportDir, "old_export.zip"), oldTime, oldTime)
+	require.NoError(t, err)
+	err = os.Chtimes(filepath.Join(exportDir, "test_export.zip"), oldTime, oldTime)
+	require.NoError(t, err)
+
+	// Start the workers
+	th.SetupWorkers(t)
+
+	// Run the export delete job
+	th.RunJob(t, model.JobTypeExportDelete, nil)
+
+	// Verify files that should still exist
+	for _, name := range []string{
+		"recent_export.zip",
+		"normal.txt",
+		"data.json",
+		"data/file.txt",
+		"data/subfolder/file.txt",
+	} {
+		_, err := os.Stat(filepath.Join(exportDir, name))
+		require.NoError(t, err, "Expected file/directory to exist: %s", name)
+	}
+
+	// Verify files that should be deleted
+	for _, name := range []string{
+		"old_export.zip",
+		"test_export.zip",
+	} {
+		_, err := os.Stat(filepath.Join(exportDir, name))
+		require.True(t, os.IsNotExist(err), "Expected file to be deleted: %s", name)
+	}
+}

server/channels/jobs/export_process/worker.go

@@ -0,0 +1,83 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package export_process
+
+import (
+	"context"
+	"io"
+	"path/filepath"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/configservice"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+type AppIface interface {
+	configservice.ConfigService
+	WriteExportFileContext(ctx context.Context, fr io.Reader, path string) (int64, *model.AppError)
+	BulkExport(rctx request.CTX, writer io.Writer, outPath string, job *model.Job, opts model.BulkExportOpts) *model.AppError
+	Log() *mlog.Logger
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface) *jobs.SimpleWorker {
+	const workerName = "ExportProcess"
+
+	isEnabled := func(cfg *model.Config) bool { return true }
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		opts := model.BulkExportOpts{
+			CreateArchive: true,
+		}
+
+		includeAttachments, ok := job.Data["include_attachments"]
+		if ok && includeAttachments == "true" {
+			opts.IncludeAttachments = true
+		}
+
+		includeArchivedChannels, ok := job.Data["include_archived_channels"]
+		if ok && includeArchivedChannels == "true" {
+			opts.IncludeArchivedChannels = true
+		}
+
+		includeProfilePictures, ok := job.Data["include_profile_pictures"]
+		if ok && includeProfilePictures == "true" {
+			opts.IncludeProfilePictures = true
+		}
+
+		includeRolesAndSchemes, ok := job.Data["include_roles_and_schemes"]
+		if ok && includeRolesAndSchemes == "true" {
+			opts.IncludeRolesAndSchemes = true
+		}
+
+		outPath := *app.Config().ExportSettings.Directory
+		exportFilename := job.Id + "_export.zip"
+
+		rd, wr := io.Pipe()
+
+		go func() {
+			_, appErr := app.WriteExportFileContext(context.Background(), rd, filepath.Join(outPath, exportFilename))
+			if appErr != nil {
+				// we close the reader here to prevent a deadlock when the bulk exporter tries to
+				// write into the pipe while app.WriteFile has already returned. The error will be
+				// returned by the writer part of the pipe when app.BulkExport tries to call
+				// wr.Write() on it.
+				rd.CloseWithError(appErr) // CloseWithError never returns an error
+			}
+		}()
+
+		appErr := app.BulkExport(request.EmptyContext(logger), wr, outPath, job, opts)
+		wr.Close() // Close never returns an error
+
+		if appErr != nil {
+			return appErr
+		}
+
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/export_users_to_csv/export_users_to_csv.go

@@ -0,0 +1,137 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package export_users_to_csv
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/pkg/errors"
+)
+
+const (
+	timeBetweenBatches = 1 * time.Second
+)
+
+// csvExportColumns defines the header row of the user CSV export. Its order must
+// match the values returned by model.UserReport.ToReport.
+var csvExportColumns = []string{
+	"Id",
+	"Username",
+	"Email",
+	"CreateAt",
+	"Name",
+	"Roles",
+	"LastLogin",
+	"LastStatusAt",
+	"LastPostDate",
+	"DaysActive",
+	"TotalPosts",
+	"ChannelCount",
+	"Teams",
+	"DeletedAt",
+}
+
+type ExportUsersToCSVAppIFace interface {
+	jobs.BatchReportWorkerAppIFace
+	GetUsersForReporting(filter *model.UserReportOptions) ([]*model.UserReport, *model.AppError)
+}
+
+// MakeWorker creates a batch report worker to generate CSV user reports.
+func MakeWorker(jobServer *jobs.JobServer, store store.Store, app ExportUsersToCSVAppIFace) model.Worker {
+	return jobs.MakeBatchReportWorker(
+		jobServer,
+		store,
+		app,
+		timeBetweenBatches,
+		"csv",
+		csvExportColumns,
+		getData(app),
+	)
+}
+
+// parseJobMetadata parses the opaque job metadata to return the information needed to decide which
+// batch to process next.
+func parseJobMetadata(data model.StringMap) (*model.UserReportOptions, error) {
+	startAt, err := strconv.ParseInt(data["start_at"], 10, 64)
+	if err != nil {
+		return nil, err
+	}
+	endAt, err := strconv.ParseInt(data["end_at"], 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	hideInactive := false
+	if val, ok := data["hide_inactive"]; ok && val != "" {
+		hideInactive, err = strconv.ParseBool(val)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse hide_inactive: %w", err)
+		}
+	}
+
+	hideActive := false
+	if val, ok := data["hide_active"]; ok && val != "" {
+		hideActive, err = strconv.ParseBool(val)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse hide_active: %w", err)
+		}
+	}
+
+	options := model.UserReportOptions{
+		ReportingBaseOptions: model.ReportingBaseOptions{
+			SortColumn:      "Username",
+			PageSize:        100,
+			FromColumnValue: data["last_column_value"],
+			FromId:          data["last_user_id"],
+			StartAt:         startAt,
+			EndAt:           endAt,
+		},
+		HideInactive: hideInactive,
+		HideActive:   hideActive,
+		Role:         data["role"],
+		Team:         data["team"],
+		GuestFilter:  data["guest_filter"],
+		SearchTerm:   data["search_term"],
+	}
+
+	return &options, nil
+}
+
+// makeJobMetadata encodes the information needed to decide which batch to process next back into
+// the opaque job metadata.
+func makeJobMetadata(jobData model.StringMap, lastColumnValue string, userID string) model.StringMap {
+	jobData["last_column_value"] = lastColumnValue
+	jobData["last_user_id"] = userID
+	return jobData
+}
+
+func getData(app ExportUsersToCSVAppIFace) func(jobData model.StringMap) ([]model.ReportableObject, model.StringMap, bool, error) {
+	return func(jobData model.StringMap) ([]model.ReportableObject, model.StringMap, bool, error) {
+		filter, err := parseJobMetadata(jobData)
+		if err != nil {
+			return nil, nil, false, errors.Wrap(err, "failed to parse job metadata")
+		}
+
+		users, appErr := app.GetUsersForReporting(filter)
+		if appErr != nil {
+			return nil, nil, false, errors.Wrapf(appErr, "failed to get the next batch (column_value=%v, user_id=%v)", filter.FromColumnValue, filter.FromId)
+		}
+
+		if len(users) == 0 {
+			return nil, nil, true, nil
+		}
+
+		reportableObjects := []model.ReportableObject{}
+		for i := range users {
+			reportableObjects = append(reportableObjects, users[i])
+		}
+
+		return reportableObjects, makeJobMetadata(jobData, users[len(users)-1].Username, users[len(users)-1].Id), false, nil
+	}
+}

server/channels/jobs/export_users_to_csv/export_users_to_csv_test.go

@@ -0,0 +1,145 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package export_users_to_csv
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+)
+
+func TestCSVExportColumns(t *testing.T) {
+	t.Run("header has the expected columns in order", func(t *testing.T) {
+		// Pin the exact header so a rename or reorder is caught. Teams sits
+		// between ChannelCount and DeletedAt.
+		require.Equal(t, []string{
+			"Id",
+			"Username",
+			"Email",
+			"CreateAt",
+			"Name",
+			"Roles",
+			"LastLogin",
+			"LastStatusAt",
+			"LastPostDate",
+			"DaysActive",
+			"TotalPosts",
+			"ChannelCount",
+			"Teams",
+			"DeletedAt",
+		}, csvExportColumns)
+	})
+
+	t.Run("header column count matches the report row values", func(t *testing.T) {
+		// The CSV header is written separately from the data rows, so a count
+		// mismatch between the header and model.UserReport.ToReport would
+		// silently shift every column. Keep them in lock-step.
+		report := (&model.UserReport{}).ToReport()
+		require.Equal(t, len(csvExportColumns), len(report))
+	})
+}
+
+// recordingExportApp records the filters the export queries users with.
+type recordingExportApp struct {
+	filters []*model.UserReportOptions
+	users   []*model.UserReport
+}
+
+func (a *recordingExportApp) SaveReportChunk(format string, prefix string, count int, reportData []model.ReportableObject) *model.AppError {
+	return nil
+}
+
+func (a *recordingExportApp) CompileReportChunks(format string, prefix string, numberOfChunks int, headers []string) *model.AppError {
+	return nil
+}
+
+func (a *recordingExportApp) SendReportToUser(rctx request.CTX, job *model.Job, format string) *model.AppError {
+	return nil
+}
+
+func (a *recordingExportApp) CleanupReportChunks(format string, prefix string, numberOfChunks int) *model.AppError {
+	return nil
+}
+
+func (a *recordingExportApp) GetUsersForReporting(filter *model.UserReportOptions) ([]*model.UserReport, *model.AppError) {
+	a.filters = append(a.filters, filter)
+	return a.users, nil
+}
+
+func TestGetData(t *testing.T) {
+	jobData := func() model.StringMap {
+		return model.StringMap{
+			"requesting_user_id": "cxzhnzbpqbnufc5qhnr63zcpdw",
+			"date_range":         model.ReportDurationAllTime,
+			"start_at":           "100",
+			"end_at":             "200",
+			"role":               "system_user",
+			"team":               "hn6jhbbutbyd8mmwbtkr3sozbo",
+			"hide_active":        "false",
+			"hide_inactive":      "true",
+			"guest_filter":       "all",
+			"search_term":        "alice",
+		}
+	}
+
+	batchOfOne := func() []*model.UserReport {
+		return []*model.UserReport{{
+			User: model.User{Id: "wbtkr3sozbohn6jhbbutbyd8mm", Username: "alice-smith"},
+		}}
+	}
+
+	t.Run("should query users with every filter the export was started with", func(t *testing.T) {
+		app := &recordingExportApp{users: batchOfOne()}
+
+		reportData, nextData, done, err := getData(app)(jobData())
+		require.NoError(t, err)
+		require.False(t, done)
+		require.Len(t, reportData, 1)
+
+		require.Len(t, app.filters, 1)
+		filter := app.filters[0]
+		require.Equal(t, "alice", filter.SearchTerm)
+		require.Equal(t, "system_user", filter.Role)
+		require.Equal(t, "hn6jhbbutbyd8mmwbtkr3sozbo", filter.Team)
+		require.Equal(t, "all", filter.GuestFilter)
+		require.False(t, filter.HideActive)
+		require.True(t, filter.HideInactive)
+		require.Equal(t, int64(100), filter.StartAt)
+		require.Equal(t, int64(200), filter.EndAt)
+
+		require.Equal(t, "alice-smith", nextData["last_column_value"])
+		require.Equal(t, "wbtkr3sozbohn6jhbbutbyd8mm", nextData["last_user_id"])
+	})
+
+	t.Run("should keep filtering on the search term once the cursor advances", func(t *testing.T) {
+		app := &recordingExportApp{users: batchOfOne()}
+
+		_, nextData, _, err := getData(app)(jobData())
+		require.NoError(t, err)
+
+		_, _, _, err = getData(app)(nextData)
+		require.NoError(t, err)
+
+		require.Len(t, app.filters, 2)
+		require.Equal(t, "alice", app.filters[1].SearchTerm)
+		require.Equal(t, "alice-smith", app.filters[1].FromColumnValue)
+		require.Equal(t, "wbtkr3sozbohn6jhbbutbyd8mm", app.filters[1].FromId)
+	})
+
+	t.Run("should export everything for jobs queued before search terms were recorded", func(t *testing.T) {
+		app := &recordingExportApp{users: batchOfOne()}
+
+		data := jobData()
+		delete(data, "search_term")
+
+		_, _, _, err := getData(app)(data)
+		require.NoError(t, err)
+
+		require.Len(t, app.filters, 1)
+		require.Empty(t, app.filters[0].SearchTerm)
+	})
+}

server/channels/jobs/extract_content/scheduler.go

@@ -0,0 +1,35 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package extract_content
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const schedFreq = 1 * time.Hour
+
+type Scheduler struct {
+	*jobs.PeriodicScheduler
+	jobServer *jobs.JobServer
+}
+
+func MakeScheduler(jobServer *jobs.JobServer) *Scheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.FileSettings.ExtractContent
+	}
+	return &Scheduler{
+		PeriodicScheduler: jobs.NewPeriodicScheduler(jobServer, model.JobTypeExtractContent, schedFreq, isEnabled),
+		jobServer:         jobServer,
+	}
+}
+
+func (scheduler *Scheduler) ScheduleJob(rctx request.CTX, cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
+	return scheduler.jobServer.CreateJob(rctx, model.JobTypeExtractContent, map[string]string{
+		catchupJobDataKey: "true",
+	})
+}

server/channels/jobs/extract_content/worker.go

@@ -0,0 +1,158 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package extract_content
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+const (
+	catchupJobDataKey = "catchup"
+	catchupLookback   = 24 * time.Hour
+	catchupBatchSize  = 1000
+)
+
+var ignoredFiles = map[string]bool{
+	"png": true, "jpg": true, "jpeg": true, "gif": true, "wmv": true,
+	"mpg": true, "mpeg": true, "mp3": true, "mp4": true, "ogg": true,
+	"ogv": true, "mov": true, "apk": true, "svg": true, "webm": true,
+	"mkv": true,
+}
+
+type AppIface interface {
+	ExtractContentFromFileInfo(rctx request.CTX, fileInfo *model.FileInfo) error
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface, store store.Store) *jobs.SimpleWorker {
+	const workerName = "ExtractContent"
+
+	isEnabled := func(cfg *model.Config) bool {
+		return true
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		jobServer.HandleJobPanic(logger, job)
+
+		if job.Data[catchupJobDataKey] == "true" {
+			return runCatchupExtraction(logger, job, jobServer, app, store)
+		}
+		return runRangeExtraction(logger, job, jobServer, app, store)
+	}
+	worker := jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+	return worker
+}
+
+func runCatchupExtraction(logger mlog.LoggerIFace, job *model.Job, jobServer *jobs.JobServer, app AppIface, store store.Store) error {
+	cursor := model.GetMillis() - catchupLookback.Milliseconds()
+
+	var nFiles int
+	var nErrs int
+	for {
+		opts := model.GetFileInfosOptions{
+			Since:            cursor,
+			SortBy:           model.FileinfoSortByCreated,
+			IncludeDeleted:   false,
+			OnlyEmptyContent: true,
+		}
+		fileInfos, err := store.FileInfo().GetWithOptions(0, catchupBatchSize, &opts)
+		if err != nil {
+			return err
+		}
+		if len(fileInfos) == 0 {
+			break
+		}
+
+		for _, fileInfo := range fileInfos {
+			if ignoredFiles[fileInfo.Extension] {
+				continue
+			}
+
+			err = app.ExtractContentFromFileInfo(request.EmptyContext(logger), fileInfo)
+			if err != nil {
+				logger.Warn("Failed to extract file content", mlog.Err(err), mlog.String("file_info_id", fileInfo.Id))
+				nErrs++
+			}
+			nFiles++
+		}
+
+		lastFileInfo := fileInfos[len(fileInfos)-1]
+		cursor = lastFileInfo.CreateAt + 1
+
+		if len(fileInfos) < catchupBatchSize {
+			break
+		}
+	}
+
+	job.Data["errors"] = strconv.Itoa(nErrs)
+	job.Data["processed"] = strconv.Itoa(nFiles)
+
+	if err := jobServer.UpdateInProgressJobData(job); err != nil {
+		logger.Error("Worker: Failed to update job data", mlog.Err(err))
+	}
+	return nil
+}
+
+func runRangeExtraction(logger mlog.LoggerIFace, job *model.Job, jobServer *jobs.JobServer, app AppIface, store store.Store) error {
+	var err error
+	var fromTS int64
+	var toTS int64 = model.GetMillis()
+	if fromStr, ok := job.Data["from"]; ok {
+		if fromTS, err = strconv.ParseInt(fromStr, 10, 64); err != nil {
+			return err
+		}
+		fromTS *= 1000
+	}
+	if toStr, ok := job.Data["to"]; ok {
+		if toTS, err = strconv.ParseInt(toStr, 10, 64); err != nil {
+			return err
+		}
+		toTS *= 1000
+	}
+
+	var nFiles int
+	var nErrs int
+	for {
+		opts := model.GetFileInfosOptions{
+			Since:          fromTS,
+			SortBy:         model.FileinfoSortByCreated,
+			IncludeDeleted: false,
+		}
+		fileInfos, err := store.FileInfo().GetWithOptions(0, catchupBatchSize, &opts)
+		if err != nil {
+			return err
+		}
+		if len(fileInfos) == 0 {
+			break
+		}
+		for _, fileInfo := range fileInfos {
+			if !ignoredFiles[fileInfo.Extension] {
+				err = app.ExtractContentFromFileInfo(request.EmptyContext(logger), fileInfo)
+				if err != nil {
+					logger.Warn("Failed to extract file content", mlog.Err(err), mlog.String("file_info_id", fileInfo.Id))
+					nErrs++
+				}
+				nFiles++
+			}
+		}
+		lastFileInfo := fileInfos[len(fileInfos)-1]
+		if lastFileInfo.CreateAt > toTS {
+			break
+		}
+		fromTS = lastFileInfo.CreateAt + 1
+	}
+
+	job.Data["errors"] = strconv.Itoa(nErrs)
+	job.Data["processed"] = strconv.Itoa(nFiles)
+
+	if err := jobServer.UpdateInProgressJobData(job); err != nil {
+		logger.Error("Worker: Failed to update job data", mlog.Err(err))
+	}
+	return nil
+}

server/channels/jobs/extract_content/worker_test.go

@@ -0,0 +1,258 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package extract_content
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
+	"github.com/mattermost/mattermost/server/v8/channels/utils/testutils"
+)
+
+type trackingApp struct {
+	errOn map[string]error
+	calls []string
+}
+
+func (a *trackingApp) ExtractContentFromFileInfo(_ request.CTX, fileInfo *model.FileInfo) error {
+	a.calls = append(a.calls, fileInfo.Id)
+	if err, ok := a.errOn[fileInfo.Id]; ok {
+		return err
+	}
+	return nil
+}
+
+func makeTestJobServer(t *testing.T) (*jobs.JobServer, *storetest.Store) {
+	t.Helper()
+
+	mockStore := &storetest.Store{}
+	t.Cleanup(func() {
+		mockStore.AssertExpectations(t)
+	})
+
+	jobServer := jobs.NewJobServer(
+		&testutils.StaticConfigService{},
+		mockStore,
+		nil,
+		mlog.CreateConsoleTestLogger(t),
+		nil,
+	)
+
+	return jobServer, mockStore
+}
+
+func expectJobDataUpdate(mockStore *storetest.Store) {
+	mockStore.JobStore.On("UpdateOptimistically", mock.AnythingOfType("*model.Job"), model.JobStatusInProgress).Return(nil, nil)
+}
+
+func expectWorkerJobCompletion(mockStore *storetest.Store, job *model.Job) {
+	claimed := *job
+	claimed.Status = model.JobStatusInProgress
+	mockStore.JobStore.On("UpdateStatusOptimistically", job.Id, model.JobStatusPending, model.JobStatusInProgress).Return(&claimed, nil)
+	mockStore.JobStore.On("UpdateStatus", job.Id, model.JobStatusSuccess).Return(&claimed, nil)
+}
+
+func makeFileInfo(id string, createAt int64, ext string) *model.FileInfo {
+	return &model.FileInfo{
+		Id:        id,
+		CreateAt:  createAt,
+		Extension: ext,
+		Name:      "file." + ext,
+		Path:      "path/" + id,
+	}
+}
+
+func makeFileInfoBatch(n int, startCreateAt int64) []*model.FileInfo {
+	batch := make([]*model.FileInfo, n)
+	for i := range n {
+		batch[i] = makeFileInfo(model.NewId(), startCreateAt+int64(i), "pdf")
+	}
+	return batch
+}
+
+func TestRunCatchupExtraction(t *testing.T) {
+	logger := mlog.CreateConsoleTestLogger(t)
+
+	t.Run("extracts non-ignored files and passes OnlyEmptyContent filter", func(t *testing.T) {
+		jobServer, mockStore := makeTestJobServer(t)
+		expectJobDataUpdate(mockStore)
+
+		pdf := makeFileInfo("pdf1", 1000, "pdf")
+		png := makeFileInfo("png1", 1001, "png")
+		docx := makeFileInfo("docx1", 1002, "docx")
+
+		mockStore.FileInfoStore.On("GetWithOptions", 0, catchupBatchSize, mock.MatchedBy(func(opt *model.GetFileInfosOptions) bool {
+			return opt.OnlyEmptyContent && !opt.IncludeDeleted && opt.SortBy == model.FileinfoSortByCreated
+		})).Return([]*model.FileInfo{pdf, png, docx}, nil).Once()
+
+		app := &trackingApp{}
+		job := &model.Job{Data: map[string]string{}}
+
+		err := runCatchupExtraction(logger, job, jobServer, app, mockStore)
+		require.NoError(t, err)
+
+		require.Equal(t, []string{"pdf1", "docx1"}, app.calls)
+		require.Equal(t, "2", job.Data["processed"])
+		require.Equal(t, "0", job.Data["errors"])
+	})
+
+	t.Run("empty result is a no-op", func(t *testing.T) {
+		jobServer, mockStore := makeTestJobServer(t)
+		expectJobDataUpdate(mockStore)
+
+		mockStore.FileInfoStore.On("GetWithOptions", 0, catchupBatchSize, mock.Anything).Return(nil, nil).Once()
+
+		job := &model.Job{Data: map[string]string{}}
+		err := runCatchupExtraction(logger, job, jobServer, &trackingApp{}, mockStore)
+		require.NoError(t, err)
+
+		require.Equal(t, "0", job.Data["processed"])
+		require.Equal(t, "0", job.Data["errors"])
+	})
+
+	t.Run("full batch advances cursor and fetches the next page", func(t *testing.T) {
+		jobServer, mockStore := makeTestJobServer(t)
+		expectJobDataUpdate(mockStore)
+
+		first := makeFileInfoBatch(catchupBatchSize, 1000)
+		second := []*model.FileInfo{makeFileInfo("tail1", 2000, "pdf"), makeFileInfo("tail2", 2001, "pdf")}
+
+		var firstPageOpts *model.GetFileInfosOptions
+		mockStore.FileInfoStore.On("GetWithOptions", 0, catchupBatchSize, mock.MatchedBy(func(opt *model.GetFileInfosOptions) bool {
+			return opt.OnlyEmptyContent
+		})).Run(func(args mock.Arguments) {
+			opt := args.Get(2).(*model.GetFileInfosOptions)
+			copied := *opt
+			firstPageOpts = &copied
+		}).Return(first, nil).Once()
+		mockStore.FileInfoStore.On("GetWithOptions", 0, catchupBatchSize, mock.MatchedBy(func(opt *model.GetFileInfosOptions) bool {
+			return opt.Since == 1999+1
+		})).Return(second, nil).Once()
+
+		sinceLower := model.GetMillis() - catchupLookback.Milliseconds()
+		app := &trackingApp{}
+		job := &model.Job{Data: map[string]string{}}
+
+		err := runCatchupExtraction(logger, job, jobServer, app, mockStore)
+		sinceUpper := model.GetMillis() - catchupLookback.Milliseconds()
+		require.NoError(t, err)
+
+		require.NotNil(t, firstPageOpts)
+		require.GreaterOrEqual(t, firstPageOpts.Since, sinceLower)
+		require.LessOrEqual(t, firstPageOpts.Since, sinceUpper)
+		require.Len(t, app.calls, catchupBatchSize+2)
+		require.Equal(t, "1002", job.Data["processed"])
+	})
+
+	t.Run("partial batch terminates after one fetch", func(t *testing.T) {
+		jobServer, mockStore := makeTestJobServer(t)
+		expectJobDataUpdate(mockStore)
+
+		batch := []*model.FileInfo{
+			makeFileInfo("f1", 1000, "pdf"),
+			makeFileInfo("f2", 1001, "pdf"),
+			makeFileInfo("f3", 1002, "pdf"),
+		}
+		mockStore.FileInfoStore.On("GetWithOptions", 0, catchupBatchSize, mock.Anything).Return(batch, nil).Once()
+
+		job := &model.Job{Data: map[string]string{}}
+		err := runCatchupExtraction(logger, job, jobServer, &trackingApp{}, mockStore)
+		require.NoError(t, err)
+
+		require.Equal(t, "3", job.Data["processed"])
+	})
+
+	t.Run("accumulates extraction errors in job data", func(t *testing.T) {
+		jobServer, mockStore := makeTestJobServer(t)
+		expectJobDataUpdate(mockStore)
+
+		okFile := makeFileInfo("ok", 1000, "pdf")
+		failFile := makeFileInfo("fail", 1001, "pdf")
+		mockStore.FileInfoStore.On("GetWithOptions", 0, catchupBatchSize, mock.Anything).
+			Return([]*model.FileInfo{okFile, failFile}, nil).Once()
+
+		app := &trackingApp{errOn: map[string]error{"fail": errors.New("extract failed")}}
+		job := &model.Job{Data: map[string]string{}}
+
+		err := runCatchupExtraction(logger, job, jobServer, app, mockStore)
+		require.NoError(t, err)
+
+		require.Equal(t, "2", job.Data["processed"])
+		require.Equal(t, "1", job.Data["errors"])
+	})
+
+	t.Run("store error propagates", func(t *testing.T) {
+		jobServer, mockStore := makeTestJobServer(t)
+
+		wantErr := errors.New("store failed")
+		mockStore.FileInfoStore.On("GetWithOptions", 0, catchupBatchSize, mock.Anything).Return(nil, wantErr).Once()
+
+		job := &model.Job{Data: map[string]string{}}
+		err := runCatchupExtraction(logger, job, jobServer, &trackingApp{}, mockStore)
+		require.ErrorIs(t, err, wantErr)
+	})
+}
+
+func TestWorkerDispatch(t *testing.T) {
+	t.Run("catchup job uses OnlyEmptyContent filter", func(t *testing.T) {
+		jobServer, mockStore := makeTestJobServer(t)
+		expectJobDataUpdate(mockStore)
+
+		mockStore.FileInfoStore.On("GetWithOptions", 0, catchupBatchSize, mock.MatchedBy(func(opt *model.GetFileInfosOptions) bool {
+			return opt.OnlyEmptyContent
+		})).Return(nil, nil).Once()
+
+		worker := MakeWorker(jobServer, &trackingApp{}, mockStore)
+		job := &model.Job{Id: model.NewId(), Data: map[string]string{catchupJobDataKey: "true"}}
+		expectWorkerJobCompletion(mockStore, job)
+
+		worker.DoJob(job)
+	})
+
+	t.Run("range job does not use OnlyEmptyContent filter", func(t *testing.T) {
+		jobServer, mockStore := makeTestJobServer(t)
+		expectJobDataUpdate(mockStore)
+
+		mockStore.FileInfoStore.On("GetWithOptions", 0, catchupBatchSize, mock.MatchedBy(func(opt *model.GetFileInfosOptions) bool {
+			return !opt.OnlyEmptyContent
+		})).Return(nil, nil).Once()
+
+		worker := MakeWorker(jobServer, &trackingApp{}, mockStore)
+		job := &model.Job{Id: model.NewId(), Data: map[string]string{}}
+		expectWorkerJobCompletion(mockStore, job)
+
+		worker.DoJob(job)
+	})
+}
+
+func TestSchedulerScheduleJob(t *testing.T) {
+	logger := mlog.CreateConsoleTestLogger(t)
+	jobServer, mockStore := makeTestJobServer(t)
+
+	app := &trackingApp{}
+	jobServer.RegisterJobType(model.JobTypeExtractContent, MakeWorker(jobServer, app, mockStore), nil)
+
+	savedJob := &model.Job{
+		Id:   model.NewId(),
+		Type: model.JobTypeExtractContent,
+		Data: map[string]string{catchupJobDataKey: "true"},
+	}
+	mockStore.JobStore.On("Save", mock.MatchedBy(func(job *model.Job) bool {
+		return job.Type == model.JobTypeExtractContent && job.Data[catchupJobDataKey] == "true"
+	})).Return(savedJob, nil)
+
+	scheduler := MakeScheduler(jobServer)
+	job, appErr := scheduler.ScheduleJob(request.EmptyContext(logger), &model.Config{}, false, nil)
+	require.Nil(t, appErr)
+	require.NotNil(t, job)
+	require.Equal(t, "true", job.Data[catchupJobDataKey])
+}

server/channels/jobs/helper_test.go

@@ -0,0 +1,358 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs_test
+
+import (
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/app"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/mattermost/mattermost/server/v8/config"
+)
+
+type TestHelper struct {
+	App        *app.App
+	Context    *request.Context
+	Server     *app.Server
+	BasicTeam  *model.Team
+	BasicUser  *model.User
+	BasicUser2 *model.User
+	Store      store.Store
+
+	SystemAdminUser   *model.User
+	LogBuffer         *mlog.Buffer
+	TestLogger        *mlog.Logger
+	IncludeCacheLayer bool
+	ConfigStore       *config.Store
+
+	tempWorkspace string
+}
+
+func setupTestHelper(tb testing.TB, dbStore store.Store, sqlSettings *model.SqlSettings, enterprise bool, includeCacheLayer bool,
+	updateCfg func(cfg *model.Config), options []app.Option,
+) *TestHelper {
+	tempWorkspace, err := os.MkdirTemp("", "jobstest")
+	require.NoError(tb, err)
+
+	configStore := config.NewTestMemoryStore()
+	memoryConfig := configStore.Get()
+	memoryConfig.SqlSettings = *sqlSettings
+	*memoryConfig.PluginSettings.Directory = filepath.Join(tempWorkspace, "plugins")
+	*memoryConfig.PluginSettings.ClientDirectory = filepath.Join(tempWorkspace, "webapp")
+	*memoryConfig.PluginSettings.AutomaticPrepackagedPlugins = false
+	*memoryConfig.LogSettings.EnableSentry = false // disable error reporting during tests
+
+	// Check for environment variable override for console log level (useful for debugging tests)
+	consoleLevel := os.Getenv("MM_LOGSETTINGS_CONSOLELEVEL")
+	if consoleLevel == "" {
+		consoleLevel = mlog.LvlStdLog.Name
+	}
+	*memoryConfig.LogSettings.ConsoleLevel = consoleLevel
+
+	*memoryConfig.AnnouncementSettings.AdminNoticesEnabled = false
+	*memoryConfig.AnnouncementSettings.UserNoticesEnabled = false
+
+	if updateCfg != nil {
+		updateCfg(memoryConfig)
+	}
+
+	_, _, err = configStore.Set(memoryConfig)
+	require.NoError(tb, err)
+
+	buffer := &mlog.Buffer{}
+
+	options = append(options, app.ConfigStore(configStore))
+	if includeCacheLayer {
+		// Adds the cache layer to the test store
+		options = append(options, app.StoreOverrideWithCache(dbStore))
+	} else {
+		options = append(options, app.StoreOverride(dbStore))
+	}
+
+	testLogger, err := mlog.NewLogger()
+	require.NoError(tb, err)
+	logCfg, err := config.MloggerConfigFromLoggerConfig(&memoryConfig.LogSettings, nil, config.GetLogFileLocation)
+	require.NoError(tb, err)
+	err = testLogger.ConfigureTargets(logCfg, nil)
+	require.NoError(tb, err, "failed to configure test logger")
+	err = mlog.AddWriterTarget(testLogger, buffer, true, mlog.StdAll...)
+	require.NoError(tb, err, "failed to add writer target to test logger")
+	// lock logger config so server init cannot override it during testing.
+	testLogger.LockConfiguration()
+	options = append(options, app.SetLogger(testLogger))
+
+	s, err := app.NewServer(options...)
+	require.NoError(tb, err)
+
+	th := &TestHelper{
+		App:               app.New(app.ServerConnector(s.Channels())),
+		Context:           request.EmptyContext(testLogger),
+		Server:            s,
+		LogBuffer:         buffer,
+		TestLogger:        testLogger,
+		IncludeCacheLayer: includeCacheLayer,
+		ConfigStore:       configStore,
+		Store:             dbStore,
+		tempWorkspace:     tempWorkspace,
+	}
+
+	prevListenAddress := *th.App.Config().ServiceSettings.ListenAddress
+	th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = "localhost:0" })
+	err = th.Server.Start()
+	require.NoError(tb, err)
+
+	th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = prevListenAddress })
+
+	th.App.Srv().Store().MarkSystemRanUnitTests()
+
+	tb.Cleanup(func() {
+		if th.IncludeCacheLayer {
+			// Clean all the caches
+			appErr := th.App.Srv().InvalidateAllCaches()
+			require.Nil(tb, appErr)
+		}
+		done := make(chan bool)
+		go func() {
+			th.Server.Shutdown()
+			close(done)
+		}()
+
+		select {
+		case <-done:
+		case <-time.After(30 * time.Second):
+			// panic instead of fatal to terminate all tests in this package, otherwise the
+			// still running App could spuriously fail subsequent tests.
+			panic("failed to shutdown App within 30 seconds")
+		}
+
+		if th.tempWorkspace != "" {
+			os.RemoveAll(th.tempWorkspace)
+		}
+	})
+
+	return th
+}
+
+func Setup(tb testing.TB, options ...app.Option) *TestHelper {
+	return SetupWithUpdateCfg(tb, nil, options...)
+}
+
+func SetupWithUpdateCfg(tb testing.TB, updateCfg func(cfg *model.Config), options ...app.Option) *TestHelper {
+	tb.Helper()
+	if testing.Short() {
+		tb.SkipNow()
+	}
+
+	var dbStore store.Store
+	var dbSettings *model.SqlSettings
+	if mainHelper.Options.RunParallel {
+		dbStore, _, dbSettings, _ = mainHelper.GetNewStores(tb)
+		tb.Cleanup(func() {
+			dbStore.Close()
+		})
+	} else {
+		dbStore = mainHelper.GetStore()
+		dbSettings = mainHelper.GetSQLSettings()
+		dbStore.DropAllTables()
+		dbStore.MarkSystemRanUnitTests()
+		mainHelper.PreloadMigrations()
+	}
+
+	th := setupTestHelper(tb, dbStore, dbSettings, false, true, updateCfg, options)
+	return th
+}
+
+func (th *TestHelper) InitBasic(tb testing.TB) *TestHelper {
+	tb.Helper()
+
+	th.SystemAdminUser = th.CreateUser(tb)
+	_, appErr := th.App.UpdateUserRoles(th.Context, th.SystemAdminUser.Id, model.SystemUserRoleId+" "+model.SystemAdminRoleId, false)
+	require.Nil(tb, appErr)
+	th.SystemAdminUser, appErr = th.App.GetUser(th.Context, th.SystemAdminUser.Id)
+	require.Nil(tb, appErr)
+
+	th.BasicUser = th.CreateUser(tb)
+	th.BasicUser, appErr = th.App.GetUser(th.Context, th.BasicUser.Id)
+	require.Nil(tb, appErr)
+
+	th.BasicUser2 = th.CreateUser(tb)
+	th.BasicUser2, appErr = th.App.GetUser(th.Context, th.BasicUser2.Id)
+	require.Nil(tb, appErr)
+
+	th.BasicTeam = th.CreateTeam(tb)
+
+	return th
+}
+
+func (th *TestHelper) CreateTeam(tb testing.TB) *model.Team {
+	tb.Helper()
+
+	id := model.NewId()
+	team := &model.Team{
+		DisplayName: "dn_" + id,
+		Name:        "name" + id,
+		Email:       "success+" + id + "@simulator.amazonses.com",
+		Type:        model.TeamOpen,
+	}
+
+	team, err := th.App.CreateTeam(th.Context, team)
+	require.Nil(tb, err)
+	return team
+}
+
+func (th *TestHelper) CreateUser(tb testing.TB) *model.User {
+	tb.Helper()
+	return th.CreateUserOrGuest(tb, false)
+}
+
+func (th *TestHelper) CreateUserOrGuest(tb testing.TB, guest bool) *model.User {
+	tb.Helper()
+	id := model.NewId()
+
+	user := &model.User{
+		Email:         "success+" + id + "@simulator.amazonses.com",
+		Username:      "un_" + id,
+		Nickname:      "nn_" + id,
+		Password:      model.NewTestPassword(),
+		EmailVerified: true,
+	}
+
+	var err *model.AppError
+	if guest {
+		user, err = th.App.CreateGuest(th.Context, user)
+	} else {
+		user, err = th.App.CreateUser(th.Context, user)
+	}
+	require.Nil(tb, err)
+	return user
+}
+
+func (th *TestHelper) ShutdownApp() {
+	done := make(chan bool)
+	go func() {
+		th.Server.Shutdown()
+		close(done)
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(30 * time.Second):
+		// panic instead of fatal to terminate all tests in this package, otherwise the
+		// still running App could spuriously fail subsequent tests.
+		panic("failed to shutdown App within 30 seconds")
+	}
+}
+
+func (th *TestHelper) SetupBatchWorker(tb testing.TB, worker *jobs.BatchWorker) *model.Job {
+	tb.Helper()
+
+	jobId := model.NewId()
+	th.Server.Jobs.RegisterJobType(jobId, worker, nil)
+
+	jobData := make(model.StringMap)
+	jobData["batch_number"] = "1"
+	job, appErr := th.Server.Jobs.CreateJob(th.Context, jobId, jobData)
+	require.Nil(tb, appErr)
+
+	done := make(chan bool)
+	go func() {
+		defer close(done)
+		worker.Run()
+	}()
+
+	// When ending the test, ensure we wait for the worker to finish.
+	tb.Cleanup(func() {
+		waitDone(tb, done, "worker did not stop running")
+	})
+
+	// Give the worker time to start running
+	time.Sleep(500 * time.Millisecond)
+
+	return job
+}
+
+func (th *TestHelper) WaitForJobStatus(tb testing.TB, job *model.Job, status string) {
+	tb.Helper()
+
+	require.Eventuallyf(tb, func() bool {
+		actualJob, appErr := th.Server.Jobs.GetJob(th.Context, job.Id)
+		require.Nil(tb, appErr)
+		require.Equal(tb, job.Id, actualJob.Id)
+
+		return actualJob.Status == status
+	}, 5*time.Second, 250*time.Millisecond, "job never transitioned to %s", status)
+}
+
+func (th *TestHelper) WaitForBatchNumber(tb testing.TB, job *model.Job, batchNumber int) {
+	tb.Helper()
+
+	require.Eventuallyf(tb, func() bool {
+		actualJob, appErr := th.Server.Jobs.GetJob(th.Context, job.Id)
+		require.Nil(tb, appErr)
+		require.Equal(tb, job.Id, actualJob.Id)
+
+		finalBatchNumber, err := strconv.Atoi(actualJob.Data["batch_number"])
+		require.NoError(tb, err)
+		return finalBatchNumber == batchNumber
+	}, 5*time.Second, 250*time.Millisecond, "job did not stop at batch %d", batchNumber)
+}
+
+func waitDone(tb testing.TB, done chan bool, msg string) {
+	tb.Helper()
+
+	require.Eventually(tb, func() bool {
+		select {
+		case <-done:
+			return true
+		default:
+			return false
+		}
+	}, 5*time.Second, 100*time.Millisecond, msg)
+}
+
+func (th *TestHelper) SetupWorkers(tb testing.TB) {
+	tb.Helper()
+
+	err := th.App.Srv().Jobs.StartWorkers()
+	require.NoError(tb, err)
+}
+
+func (th *TestHelper) RunJob(tb testing.TB, jobType string, jobData map[string]string) *model.Job {
+	tb.Helper()
+
+	job, appErr := th.Server.Jobs.CreateJob(th.Context, jobType, jobData)
+	require.Nil(tb, appErr)
+
+	// poll until completion
+	th.checkJobStatus(tb, job.Id, model.JobStatusSuccess)
+	job, appErr = th.Server.Jobs.GetJob(th.Context, job.Id)
+	require.Nil(tb, appErr)
+
+	return job
+}
+
+func (th *TestHelper) checkJobStatus(tb testing.TB, jobId string, status string) {
+	tb.Helper()
+
+	require.Eventuallyf(tb, func() bool {
+		// it's ok if there's an error, it might take awhile for the job to finish.
+		job, appErr := th.Server.Jobs.GetJob(th.Context, jobId)
+		assert.Nil(tb, appErr)
+		if jobId == job.Id {
+			return job.Status == status
+		}
+		return false
+	}, 15*time.Second, 100*time.Millisecond, "expected job's status to be %s", status)
+}

server/channels/jobs/hosted_purchase_screening/scheduler.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package hosted_purchase_screening
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const schedFreq = 24 * time.Hour
+
+func MakeScheduler(jobServer *jobs.JobServer, license *model.License) *jobs.PeriodicScheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return model.BuildEnterpriseReady == "true" && license == nil
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeHostedPurchaseScreening, schedFreq, isEnabled)
+}

server/channels/jobs/hosted_purchase_screening/worker.go

@@ -0,0 +1,54 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package hosted_purchase_screening
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const (
+	// 3 days matches the expecation given in portal purchase flow.
+	waitForScreeningDuration = 3 * 24 * time.Hour
+)
+
+type ScreenTimeStore interface {
+	GetByName(string) (*model.System, error)
+	PermanentDeleteByName(name string) (*model.System, error)
+}
+
+func MakeWorker(jobServer *jobs.JobServer, license *model.License, screenTimeStore ScreenTimeStore) *jobs.SimpleWorker {
+	const workerName = "HostedPurchaseScreening"
+
+	isEnabled := func(_ *model.Config) bool {
+		return !license.IsCloud()
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		now := time.Now()
+		screenTimeValue, err := screenTimeStore.GetByName(model.SystemHostedPurchaseNeedsScreening)
+		if err != nil {
+			return err
+		}
+		screenTime, err := strconv.ParseInt(screenTimeValue.Value, 10, 64)
+		if err != nil {
+			return err
+		}
+
+		if now.After(time.UnixMilli(screenTime).Add(waitForScreeningDuration)) {
+			_, err = screenTimeStore.PermanentDeleteByName(model.SystemHostedPurchaseNeedsScreening)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/import_delete/scheduler.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package import_delete
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const schedFreq = 24 * time.Hour
+
+func MakeScheduler(jobServer *jobs.JobServer) *jobs.PeriodicScheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.ImportSettings.Directory != "" && *cfg.ImportSettings.RetentionDays > 0
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeImportDelete, schedFreq, isEnabled)
+}

server/channels/jobs/import_delete/worker.go

@@ -0,0 +1,108 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package import_delete
+
+import (
+	"errors"
+	"path/filepath"
+	"time"
+
+	"github.com/wiggin77/merror"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/configservice"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+type AppIface interface {
+	configservice.ConfigService
+	ListDirectory(path string) ([]string, *model.AppError)
+	FileModTime(path string) (time.Time, *model.AppError)
+	RemoveFile(path string) *model.AppError
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface, s store.Store) *jobs.SimpleWorker {
+	const workerName = "ImportDelete"
+
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.ImportSettings.Directory != "" && *cfg.ImportSettings.RetentionDays > 0
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		rctx := request.EmptyContext(logger)
+
+		importPath := *app.Config().ImportSettings.Directory
+		retentionTime := time.Duration(*app.Config().ImportSettings.RetentionDays) * 24 * time.Hour
+		imports, appErr := app.ListDirectory(importPath)
+		if appErr != nil {
+			return appErr
+		}
+
+		multipleErrors := merror.New()
+		for i := range imports {
+			filename := filepath.Base(imports[i])
+			modTime, appErr := app.FileModTime(filepath.Join(importPath, filename))
+			if appErr != nil {
+				logger.Debug("Worker: Failed to get file modification time",
+					mlog.Err(appErr), mlog.String("import", imports[i]))
+				multipleErrors.Append(appErr)
+				continue
+			}
+
+			if time.Now().After(modTime.Add(retentionTime)) {
+				// expected format if uploaded through the API is
+				// ${uploadID}_${filename}${model.IncompleteUploadSuffix}
+				minLen := 26 + 1 + len(model.IncompleteUploadSuffix)
+
+				// check if it's an incomplete upload and attempt to delete its session.
+				if len(filename) > minLen && filepath.Ext(filename) == model.IncompleteUploadSuffix {
+					uploadID := filename[:26]
+					if storeErr := s.UploadSession().Delete(uploadID); storeErr != nil {
+						logger.Debug("Worker: Failed to delete UploadSession",
+							mlog.Err(storeErr), mlog.String("upload_id", uploadID))
+						multipleErrors.Append(storeErr)
+						continue
+					}
+				} else {
+					// check if fileinfo exists and if so delete it.
+					filePath := filepath.Join(imports[i])
+					info, storeErr := s.FileInfo().GetByPath(filePath)
+					var nfErr *store.ErrNotFound
+					if storeErr != nil && !errors.As(storeErr, &nfErr) {
+						logger.Debug("Worker: Failed to get FileInfo",
+							mlog.Err(storeErr), mlog.String("path", filePath))
+						multipleErrors.Append(storeErr)
+						continue
+					} else if storeErr == nil {
+						if storeErr = s.FileInfo().PermanentDelete(rctx, info.Id); storeErr != nil {
+							logger.Debug("Worker: Failed to delete FileInfo",
+								mlog.Err(storeErr), mlog.String("file_id", info.Id))
+							multipleErrors.Append(storeErr)
+							continue
+						}
+					}
+				}
+
+				// remove file data from storage.
+				if appErr := app.RemoveFile(imports[i]); appErr != nil {
+					logger.Debug("Worker: Failed to remove file",
+						mlog.Err(appErr), mlog.String("import", imports[i]))
+					multipleErrors.Append(appErr)
+					continue
+				}
+			}
+		}
+
+		if err := multipleErrors.ErrorOrNil(); err != nil {
+			logger.Warn("Worker: errors occurred", mlog.Err(err))
+		}
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/import_process/worker.go

@@ -0,0 +1,153 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package import_process
+
+import (
+	"archive/zip"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/configservice"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/app/imports"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+)
+
+type AppIface interface {
+	configservice.ConfigService
+	RemoveFile(path string) *model.AppError
+	FileExists(path string) (bool, *model.AppError)
+	FileSize(path string) (int64, *model.AppError)
+	FileReader(path string) (filestore.ReadCloseSeeker, *model.AppError)
+	BulkImportWithPath(rctx request.CTX, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun, extractContent bool, workers int, importPath string) (int, *model.AppError)
+	Log() *mlog.Logger
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface) *jobs.SimpleWorker {
+	const workerName = "ImportProcess"
+
+	appContext := request.EmptyContext(jobServer.Logger())
+	isEnabled := func(cfg *model.Config) bool {
+		return true
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		importFileName, ok := job.Data["import_file"]
+		if !ok {
+			return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.missing_file", nil, "", http.StatusBadRequest)
+		}
+
+		var importFilePath string
+		var importFileSize int64
+		var importFile filestore.ReadCloseSeeker
+		if job.Data["local_mode"] == "true" {
+			// We simply read the file from the local filesystem.
+			info, err := os.Stat(importFileName)
+			if errors.Is(err, os.ErrNotExist) {
+				return fmt.Errorf("file %s doesn't exist.", importFileName)
+			}
+
+			importFileSize = info.Size()
+
+			importFile, err = os.Open(importFileName)
+			if err != nil {
+				return err
+			}
+			defer importFile.Close()
+		} else {
+			importFilePath = filepath.Join(*app.Config().ImportSettings.Directory, importFileName)
+			if ok, err := app.FileExists(importFilePath); err != nil {
+				return err
+			} else if !ok {
+				return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.file_exists", nil, "", http.StatusBadRequest)
+			}
+
+			var appErr *model.AppError
+			importFileSize, appErr = app.FileSize(importFilePath)
+			if appErr != nil {
+				return appErr
+			}
+
+			importFile, appErr = app.FileReader(importFilePath)
+			if appErr != nil {
+				return appErr
+			}
+			defer importFile.Close()
+
+			// The import is a long running operation, try to cancel any timeouts attached to the reader.
+			type TimeoutCanceler interface{ CancelTimeout() bool }
+			if tc, ok := importFile.(TimeoutCanceler); ok {
+				if !tc.CancelTimeout() {
+					appContext.Logger().Warn("Could not cancel the timeout for the file reader. The import may fail due to a timeout.")
+				}
+			}
+		}
+
+		importZipReader, err := zip.NewReader(importFile.(io.ReaderAt), importFileSize)
+		if err != nil {
+			return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+
+		// find JSONL import file.
+		var jsonZipFile *zip.File
+		for _, f := range importZipReader.File {
+			if imports.IsRootJsonlFile(f.Name) {
+				jsonZipFile = f
+				break
+			}
+		}
+		if jsonZipFile == nil {
+			return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.missing_jsonl", nil, "jsonFile was nil", http.StatusBadRequest)
+		}
+
+		// avoid "zip slip"
+		if strings.Contains(jsonZipFile.Name, "..") {
+			return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, "jsonFilePath contains path traversal", http.StatusForbidden)
+		}
+
+		jsonFile, err := jsonZipFile.Open()
+		if err != nil {
+			return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+		defer jsonFile.Close()
+
+		extractContent := job.Data["extract_content"] == "true"
+
+		numWorkers := runtime.NumCPU()
+		if workersStr, ok := job.Data["workers"]; ok {
+			if n, err := strconv.Atoi(workersStr); err == nil && n > 0 {
+				numWorkers = n
+			}
+		}
+
+		// do the actual import.
+		lineNumber, appErr := app.BulkImportWithPath(appContext, jsonFile, importZipReader, false, extractContent, numWorkers, model.ExportDataDir)
+		if appErr != nil {
+			job.Data["line_number"] = strconv.Itoa(lineNumber)
+			return appErr
+		}
+
+		// No need to remove the file in local mode.
+		if job.Data["local_mode"] != "true" {
+			// remove import file when done.
+			if appErr := app.RemoveFile(importFilePath); appErr != nil {
+				return appErr
+			}
+		}
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/jobs.go

@@ -0,0 +1,429 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"runtime/pprof"
+	"strings"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+const (
+	CancelWatcherPollingInterval = 5000
+)
+
+// JobLoggerFields returns the logger annotations reflecting the given job metadata.
+func JobLoggerFields(job *model.Job) []mlog.Field {
+	if job == nil {
+		return nil
+	}
+
+	return []mlog.Field{
+		mlog.String("job_id", job.Id),
+		mlog.String("job_type", job.Type),
+		mlog.Millis("job_create_at", job.CreateAt),
+	}
+}
+
+func (srv *JobServer) CreateJob(rctx request.CTX, jobType string, jobData map[string]string) (*model.Job, *model.AppError) {
+	job, appErr := srv._createJob(rctx, jobType, jobData)
+	if appErr != nil {
+		return nil, appErr
+	}
+
+	if _, err := srv.Store.Job().Save(job); err != nil {
+		return nil, model.NewAppError("CreateJob", "app.job.save.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	return job, nil
+}
+
+func (srv *JobServer) CreateJobOnce(rctx request.CTX, jobType string, jobData map[string]string) (*model.Job, *model.AppError) {
+	job, appErr := srv._createJob(rctx, jobType, jobData)
+	if appErr != nil {
+		return nil, appErr
+	}
+
+	if _, err := srv.Store.Job().SaveOnce(job); err != nil {
+		return nil, model.NewAppError("CreateJob", "app.job.save.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	return job, nil
+}
+
+func (srv *JobServer) CreateJobOnceByTypeAndData(rctx request.CTX, jobType string, jobData map[string]string, data map[string]string) (*model.Job, *model.AppError) {
+	job, appErr := srv._createJob(rctx, jobType, jobData)
+	if appErr != nil {
+		return nil, appErr
+	}
+
+	savedJob, err := srv.Store.Job().SaveOnceByTypeAndData(job, data)
+	if err != nil {
+		return nil, model.NewAppError("CreateJob", "app.job.save.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	return savedJob, nil
+}
+
+func (srv *JobServer) _createJob(rctx request.CTX, jobType string, jobData map[string]string) (*model.Job, *model.AppError) {
+	job := model.Job{
+		Id:       model.NewId(),
+		Type:     jobType,
+		CreateAt: model.GetMillis(),
+		Status:   model.JobStatusPending,
+		Data:     jobData,
+	}
+
+	if err := job.IsValid(); err != nil {
+		return nil, err
+	}
+
+	if srv.workers.Get(job.Type) == nil {
+		return nil, model.NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+job.Id, http.StatusBadRequest)
+	}
+
+	return &job, nil
+}
+
+func (srv *JobServer) GetJob(rctx request.CTX, id string) (*model.Job, *model.AppError) {
+	job, err := srv.Store.Job().Get(rctx, id)
+	if err != nil {
+		var nfErr *store.ErrNotFound
+		switch {
+		case errors.As(err, &nfErr):
+			return nil, model.NewAppError("GetJob", "app.job.get.app_error", nil, "", http.StatusNotFound).Wrap(err)
+		default:
+			return nil, model.NewAppError("GetJob", "app.job.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+	}
+
+	return job, nil
+}
+
+func (srv *JobServer) ClaimJob(job *model.Job) (*model.Job, *model.AppError) {
+	newJob, err := srv.Store.Job().UpdateStatusOptimistically(job.Id, model.JobStatusPending, model.JobStatusInProgress)
+	if err != nil {
+		return nil, model.NewAppError("ClaimJob", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	if newJob != nil {
+		if srv.metrics != nil {
+			srv.metrics.IncrementJobActive(newJob.Type)
+		}
+		srv.publishJobStatus(newJob, model.JobStatusInProgress)
+	}
+
+	return newJob, nil
+}
+
+func (srv *JobServer) publishJobStatus(job *model.Job, status string) {
+	if srv.publish == nil {
+		return
+	}
+	jobCopy := *job
+	jobCopy.Status = status
+	jobJSON, err := json.Marshal(&jobCopy)
+	if err != nil {
+		srv.logger.Warn("Failed to marshal job for WebSocket event", mlog.Err(err))
+		return
+	}
+	message := model.NewWebSocketEvent(model.WebsocketEventJobUpdated, "", "", "", nil, "")
+	message.Add("job", string(jobJSON))
+
+	var permission *model.Permission
+	switch job.Type {
+	case model.JobTypeDataRetention:
+		permission = model.PermissionReadDataRetentionJob
+	case model.JobTypeMessageExport:
+		permission = model.PermissionReadComplianceExportJob
+	case model.JobTypeElasticsearchPostIndexing:
+		permission = model.PermissionReadElasticsearchPostIndexingJob
+	case model.JobTypeElasticsearchPostAggregation:
+		permission = model.PermissionReadElasticsearchPostAggregationJob
+	case model.JobTypeLdapSync:
+		permission = model.PermissionReadLdapSyncJob
+	case
+		model.JobTypeMigrations,
+		model.JobTypePlugins,
+		model.JobTypeProductNotices,
+		model.JobTypeExpiryNotify,
+		model.JobTypeActiveUsers,
+		model.JobTypeImportProcess,
+		model.JobTypeImportDelete,
+		model.JobTypeExportProcess,
+		model.JobTypeExportDelete,
+		model.JobTypeCloud,
+		model.JobTypeMobileSessionMetadata,
+		model.JobTypeExtractContent,
+		model.JobTypeCleanupExpiredAccessTokens:
+		permission = model.PermissionReadJobs
+	case model.JobTypeAccessControlSync:
+		permission = model.PermissionManageSystem
+	case model.JobTypeAccessControlTeamSync:
+		permission = model.PermissionManageTeamAccessRules
+	}
+	// TODO: ContainsSensitiveData is set unconditionally as a temporary workaround for mixed-version
+	// cluster rollouts. Older nodes don't know about RequiredPermissions and silently drop it when
+	// decoding the cluster message, so this makes them fail closed (sysadmin-only) instead of
+	// broadcasting the unfiltered job to everyone. ShouldSendEvent on upgraded nodes ignores
+	// ContainsSensitiveData whenever RequiredPermissions is set. Remove once permission-scoped
+	// WebSocket delivery no longer needs to interoperate with pre-MM-69403 nodes.
+	message.GetBroadcast().ContainsSensitiveData = true
+	if permission != nil {
+		message.GetBroadcast().RequiredPermissions = []string{permission.Id}
+	}
+	srv.publish(message)
+}
+
+func (srv *JobServer) SetJobProgress(job *model.Job, progress int64) *model.AppError {
+	job.Status = model.JobStatusInProgress
+	job.Progress = progress
+
+	ret, err := srv.Store.Job().UpdateOptimistically(job, model.JobStatusInProgress)
+	if err != nil {
+		return model.NewAppError("SetJobProgress", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	if ret != nil {
+		srv.publishJobStatus(ret, model.JobStatusInProgress)
+	}
+	return nil
+}
+
+func (srv *JobServer) SetJobWarning(job *model.Job) *model.AppError {
+	ret, err := srv.Store.Job().UpdateStatus(job.Id, model.JobStatusWarning)
+	if err != nil {
+		return model.NewAppError("SetJobWarning", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	if srv.metrics != nil {
+		srv.metrics.DecrementJobActive(job.Type)
+	}
+
+	srv.publishJobStatus(ret, model.JobStatusWarning)
+	return nil
+}
+
+func (srv *JobServer) SetJobSuccess(job *model.Job) *model.AppError {
+	ret, err := srv.Store.Job().UpdateStatus(job.Id, model.JobStatusSuccess)
+	if err != nil {
+		return model.NewAppError("SetJobSuccess", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	if srv.metrics != nil {
+		srv.metrics.DecrementJobActive(ret.Type)
+	}
+
+	srv.publishJobStatus(ret, model.JobStatusSuccess)
+	return nil
+}
+
+func (srv *JobServer) SetJobError(job *model.Job, jobError *model.AppError) *model.AppError {
+	if jobError == nil {
+		ret, err := srv.Store.Job().UpdateStatus(job.Id, model.JobStatusError)
+		if err != nil {
+			return model.NewAppError("SetJobError", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+
+		if srv.metrics != nil {
+			srv.metrics.DecrementJobActive(ret.Type)
+		}
+
+		srv.publishJobStatus(ret, model.JobStatusError)
+		return nil
+	}
+
+	job.Status = model.JobStatusError
+	job.Progress = -1
+	if job.Data == nil {
+		job.Data = make(map[string]string)
+	}
+	job.Data["error"] = jobError.Message
+	if jobError.DetailedError != "" {
+		job.Data["error"] += " — " + jobError.DetailedError
+	}
+	if wrapped := jobError.Unwrap(); wrapped != nil {
+		job.Data["error"] += " — " + wrapped.Error()
+	}
+	ret, err := srv.Store.Job().UpdateOptimistically(job, model.JobStatusInProgress)
+	if err != nil {
+		return model.NewAppError("SetJobError", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	if ret != nil && srv.metrics != nil {
+		srv.metrics.DecrementJobActive(job.Type)
+	}
+
+	if ret == nil {
+		ret, err = srv.Store.Job().UpdateOptimistically(job, model.JobStatusCancelRequested)
+		if err != nil {
+			return model.NewAppError("SetJobError", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+		if ret == nil {
+			return model.NewAppError("SetJobError", "jobs.set_job_error.update.error", nil, "id="+job.Id, http.StatusInternalServerError)
+		}
+	}
+
+	srv.publishJobStatus(ret, ret.Status)
+	return nil
+}
+
+func (srv *JobServer) SetJobCanceled(job *model.Job) *model.AppError {
+	ret, err := srv.Store.Job().UpdateStatus(job.Id, model.JobStatusCanceled)
+	if err != nil {
+		return model.NewAppError("SetJobCanceled", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	if srv.metrics != nil {
+		srv.metrics.DecrementJobActive(ret.Type)
+	}
+
+	srv.publishJobStatus(ret, model.JobStatusCanceled)
+	return nil
+}
+
+func (srv *JobServer) SetJobPending(job *model.Job) *model.AppError {
+	ret, err := srv.Store.Job().UpdateStatus(job.Id, model.JobStatusPending)
+	if err != nil {
+		return model.NewAppError("SetJobPending", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	if srv.metrics != nil {
+		srv.metrics.DecrementJobActive(ret.Type)
+	}
+
+	srv.publishJobStatus(ret, model.JobStatusPending)
+	return nil
+}
+
+func (srv *JobServer) UpdateInProgressJobData(job *model.Job) *model.AppError {
+	job.Status = model.JobStatusInProgress
+	job.LastActivityAt = model.GetMillis()
+	if _, err := srv.Store.Job().UpdateOptimistically(job, model.JobStatusInProgress); err != nil {
+		return model.NewAppError("UpdateInProgressJobData", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+// HandleJobPanic is used to handle panics during the execution of a job. It logs the panic and sets the status for the job.
+// After handling, the method repanics! This method is supposed to be `defer`'d at the start of the job.
+func (srv *JobServer) HandleJobPanic(logger mlog.LoggerIFace, job *model.Job) {
+	r := recover()
+	if r == nil {
+		return
+	}
+
+	sb := &strings.Builder{}
+	err := pprof.Lookup("goroutine").WriteTo(sb, 2)
+	if err != nil {
+		logger.Error("Error fetching goroutine stack", mlog.Err(err))
+	}
+	logger.Error("Unhandled panic in job", mlog.Any("panic", r), mlog.Any("job", job), mlog.String("stack", sb.String()))
+
+	rerr, ok := r.(error)
+	if !ok {
+		rerr = fmt.Errorf("job panic: %v", r)
+	}
+
+	appErr := srv.SetJobError(job, model.NewAppError("HandleJobPanic", "app.job.update.app_error", nil, "", http.StatusInternalServerError))
+	if appErr != nil {
+		appErr = appErr.Wrap(rerr)
+		logger.Error("Failed to set the job status to 'failed'", mlog.Err(appErr), mlog.Any("job", job))
+	}
+
+	panic(r)
+}
+
+func (srv *JobServer) RequestCancellation(rctx request.CTX, jobId string) *model.AppError {
+	newJob, err := srv.Store.Job().UpdateStatusOptimistically(jobId, model.JobStatusPending, model.JobStatusCanceled)
+	if err != nil {
+		return model.NewAppError("RequestCancellation", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	if newJob != nil {
+		if srv.metrics != nil {
+			srv.metrics.DecrementJobActive(newJob.Type)
+		}
+		srv.publishJobStatus(newJob, model.JobStatusCanceled)
+		return nil
+	}
+
+	newJob, err = srv.Store.Job().UpdateStatusOptimistically(jobId, model.JobStatusInProgress, model.JobStatusCancelRequested)
+	if err != nil {
+		return model.NewAppError("RequestCancellation", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	if newJob != nil {
+		srv.publishJobStatus(newJob, model.JobStatusCancelRequested)
+		return nil
+	}
+
+	return model.NewAppError("RequestCancellation", "jobs.request_cancellation.status.error", nil, "id="+jobId, http.StatusInternalServerError)
+}
+
+func (srv *JobServer) CancellationWatcher(rctx request.CTX, jobId string, cancelChan chan struct{}) {
+	for {
+		select {
+		case <-rctx.Context().Done():
+			rctx.Logger().Debug("CancellationWatcher for Job Aborting as job has finished.", mlog.String("job_id", jobId))
+			return
+		case <-time.After(CancelWatcherPollingInterval * time.Millisecond):
+			rctx.Logger().Debug("CancellationWatcher for Job started polling.", mlog.String("job_id", jobId))
+			jobStatus, err := srv.Store.Job().Get(rctx, jobId)
+			if err != nil {
+				rctx.Logger().Warn("Error getting job", mlog.String("job_id", jobId), mlog.Err(err))
+				continue
+			}
+			if jobStatus.Status == model.JobStatusCancelRequested {
+				close(cancelChan)
+				return
+			}
+		}
+	}
+}
+
+func GenerateNextStartDateTime(now time.Time, nextStartTime time.Time) *time.Time {
+	nextTime := time.Date(now.Year(), now.Month(), now.Day(), nextStartTime.Hour(), nextStartTime.Minute(), 0, 0, time.Local)
+
+	if !now.Before(nextTime) {
+		nextTime = nextTime.AddDate(0, 0, 1)
+	}
+
+	return &nextTime
+}
+
+func (srv *JobServer) CheckForPendingJobsByType(jobType string) (bool, *model.AppError) {
+	count, err := srv.Store.Job().GetCountByStatusAndType(model.JobStatusPending, jobType)
+	if err != nil {
+		return false, model.NewAppError("CheckForPendingJobsByType", "app.job.get_count_by_status_and_type.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return count > 0, nil
+}
+
+func (srv *JobServer) GetJobsByTypeAndStatus(rctx request.CTX, jobType string, status string) ([]*model.Job, *model.AppError) {
+	jobs, err := srv.Store.Job().GetAllByTypeAndStatus(rctx, jobType, status)
+	if err != nil {
+		return nil, model.NewAppError("GetJobsByTypeAndStatus", "app.job.get_all_jobs_by_type_and_status.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	return jobs, nil
+}
+
+func (srv *JobServer) GetLastSuccessfulJobByType(jobType string) (*model.Job, *model.AppError) {
+	statuses := []string{model.JobStatusSuccess}
+	if jobType == model.JobTypeMessageExport {
+		statuses = []string{model.JobStatusWarning, model.JobStatusSuccess}
+	}
+	job, err := srv.Store.Job().GetNewestJobByStatusesAndType(statuses, jobType)
+	var nfErr *store.ErrNotFound
+	if err != nil && !errors.As(err, &nfErr) {
+		return nil, model.NewAppError("GetLastSuccessfulJobByType", "app.job.get_newest_job_by_status_and_type.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return job, nil
+}

server/channels/jobs/jobs_watcher.go

@@ -0,0 +1,86 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"math/rand"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+)
+
+// Default polling interval for jobs termination.
+// (Defining as `var` rather than `const` allows tests to lower the interval.)
+var DefaultWatcherPollingInterval = 15000
+
+type Watcher struct {
+	srv     *JobServer
+	workers *Workers
+
+	stop            chan struct{}
+	stopped         chan struct{}
+	pollingInterval int
+}
+
+func (srv *JobServer) MakeWatcher(workers *Workers, pollingInterval int) *Watcher {
+	return &Watcher{
+		stop:            make(chan struct{}),
+		stopped:         make(chan struct{}),
+		pollingInterval: pollingInterval,
+		workers:         workers,
+		srv:             srv,
+	}
+}
+
+func (watcher *Watcher) Start() {
+	mlog.Debug("Watcher Started")
+	// Delay for some random number of milliseconds before starting to ensure that multiple
+	// instances of the jobserver  don't poll at a time too close to each other.
+	rand.Seed(time.Now().UTC().UnixNano())
+	<-time.After(time.Duration(rand.Intn(watcher.pollingInterval)) * time.Millisecond)
+
+	defer func() {
+		mlog.Debug("Watcher Finished")
+		close(watcher.stopped)
+	}()
+
+	for {
+		select {
+		case <-watcher.stop:
+			mlog.Debug("Watcher: Received stop signal")
+			return
+		case <-time.After(time.Duration(watcher.pollingInterval) * time.Millisecond):
+			watcher.PollAndNotify()
+		}
+	}
+}
+
+func (watcher *Watcher) Stop() {
+	mlog.Debug("Watcher Stopping")
+	close(watcher.stop)
+	<-watcher.stopped
+
+	watcher.stop = make(chan struct{})
+	watcher.stopped = make(chan struct{})
+}
+
+func (watcher *Watcher) PollAndNotify() {
+	jobs, err := watcher.srv.Store.Job().GetAllByStatus(request.EmptyContext(watcher.srv.logger), model.JobStatusPending)
+	if err != nil {
+		mlog.Error("Error occurred getting all pending statuses.", mlog.Err(err))
+		return
+	}
+
+	for _, job := range jobs {
+		worker := watcher.workers.Get(job.Type)
+		if worker != nil {
+			select {
+			case worker.JobChannel() <- *job:
+			default:
+			}
+		}
+	}
+}

server/channels/jobs/last_accessible_file/scheduler.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package last_accessible_file
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const schedFreq = 2 * time.Hour
+
+func MakeScheduler(jobServer *jobs.JobServer, license *model.License) *jobs.PeriodicScheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		enabled := license != nil && *license.Features.Cloud
+		mlog.Debug("Scheduler: isEnabled: "+strconv.FormatBool(enabled), mlog.String("scheduler", model.JobTypeLastAccessibleFile))
+		return enabled
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeLastAccessibleFile, schedFreq, isEnabled)
+}

server/channels/jobs/last_accessible_file/worker.go

@@ -0,0 +1,29 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package last_accessible_file
+
+import (
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+type AppIface interface {
+	ComputeLastAccessibleFileTime() error
+}
+
+func MakeWorker(jobServer *jobs.JobServer, license *model.License, app AppIface) *jobs.SimpleWorker {
+	const workerName = "LastAccessibleFile"
+
+	isEnabled := func(_ *model.Config) bool {
+		return license != nil && *license.Features.Cloud
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		return app.ComputeLastAccessibleFileTime()
+	}
+	worker := jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/last_accessible_post/scheduler.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package last_accessible_post
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const schedFreq = 30 * time.Minute
+
+func MakeScheduler(jobServer *jobs.JobServer, license *model.License) *jobs.PeriodicScheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		// Enable for any license with post history limits (i.e. Entry SKU)
+		enabled := license != nil && license.Limits != nil && license.Limits.PostHistory > 0
+		mlog.Debug("Scheduler: isEnabled: "+strconv.FormatBool(enabled), mlog.String("scheduler", model.JobTypeLastAccessiblePost))
+		return enabled
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeLastAccessiblePost, schedFreq, isEnabled)
+}

server/channels/jobs/last_accessible_post/worker.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package last_accessible_post
+
+import (
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+type AppIface interface {
+	ComputeLastAccessiblePostTime() error
+}
+
+func MakeWorker(jobServer *jobs.JobServer, license *model.License, app AppIface) *jobs.SimpleWorker {
+	const workerName = "LastAccessiblePost"
+
+	isEnabled := func(_ *model.Config) bool {
+		// Enable for any license with post history limits (i.e. Entry SKU)
+		return license != nil && license.Limits != nil && license.Limits.PostHistory > 0
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		return app.ComputeLastAccessiblePostTime()
+	}
+	worker := jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/main_test.go

@@ -0,0 +1,46 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs_test
+
+import (
+	"flag"
+	"os"
+	"strconv"
+	"testing"
+
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/testlib"
+)
+
+var mainHelper *testlib.MainHelper
+
+func TestMain(m *testing.M) {
+	var parallelism int
+	if f := flag.Lookup("test.parallel"); f != nil {
+		parallelism, _ = strconv.Atoi(f.Value.String())
+	}
+	runParallel := os.Getenv("ENABLE_FULLY_PARALLEL_TESTS") == "true" && parallelism > 1
+	if runParallel {
+		mlog.Info("Fully parallel tests enabled", mlog.Int("parallelism", parallelism))
+	}
+
+	oldWatcherPollingInterval := jobs.DefaultWatcherPollingInterval
+	jobs.DefaultWatcherPollingInterval = 100
+	defer func() {
+		jobs.DefaultWatcherPollingInterval = oldWatcherPollingInterval
+	}()
+
+	options := testlib.HelperOptions{
+		EnableStore:     true,
+		EnableResources: true,
+		RunParallel:     runParallel,
+		Parallelism:     parallelism,
+	}
+
+	mainHelper = testlib.NewMainHelperWithOptions(&options)
+	defer mainHelper.Close()
+
+	mainHelper.Main(m)
+}

server/channels/jobs/migrations/advanced_permissions_phase_2.go

@@ -0,0 +1,112 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+)
+
+type AdvancedPermissionsPhase2Progress struct {
+	CurrentTable  string `json:"current_table"`
+	LastTeamID    string `json:"last_team_id"`
+	LastChannelID string `json:"last_channel_id"`
+	LastUserID    string `json:"last_user"`
+}
+
+func (p *AdvancedPermissionsPhase2Progress) ToJSON() string {
+	b, _ := json.Marshal(p)
+	return string(b)
+}
+
+func AdvancedPermissionsPhase2ProgressFromJSON(data io.Reader) *AdvancedPermissionsPhase2Progress {
+	var o *AdvancedPermissionsPhase2Progress
+	err := json.NewDecoder(data).Decode(&o)
+	if err != nil {
+		mlog.Warn("Error decoding advanced permissions phase 2 progress", mlog.Err(err))
+	}
+	return o
+}
+
+func (p *AdvancedPermissionsPhase2Progress) IsValid() bool {
+	if !model.IsValidId(p.LastChannelID) {
+		return false
+	}
+
+	if !model.IsValidId(p.LastTeamID) {
+		return false
+	}
+
+	if !model.IsValidId(p.LastUserID) {
+		return false
+	}
+
+	switch p.CurrentTable {
+	case "TeamMembers":
+	case "ChannelMembers":
+	default:
+		return false
+	}
+
+	return true
+}
+
+func (worker *Worker) runAdvancedPermissionsPhase2Migration(lastDone string) (bool, string, *model.AppError) {
+	var progress *AdvancedPermissionsPhase2Progress
+	if lastDone == "" {
+		// Haven't started the migration yet.
+		progress = &AdvancedPermissionsPhase2Progress{
+			CurrentTable:  "TeamMembers",
+			LastChannelID: strings.Repeat("0", 26),
+			LastTeamID:    strings.Repeat("0", 26),
+			LastUserID:    strings.Repeat("0", 26),
+		}
+	} else {
+		err := json.NewDecoder(strings.NewReader(lastDone)).Decode(&progress)
+		if err != nil {
+			return false, "", model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "migrations.worker.run_advanced_permissions_phase_2_migration.invalid_progress", map[string]any{"lastDone": lastDone}, "", http.StatusInternalServerError).Wrap(err)
+		}
+		if !progress.IsValid() {
+			return false, "", model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "migrations.worker.run_advanced_permissions_phase_2_migration.invalid_progress", map[string]any{"progress": progress.ToJSON()}, "", http.StatusInternalServerError)
+		}
+	}
+
+	if progress.CurrentTable == "TeamMembers" {
+		// Run a TeamMembers migration batch.
+		result, err := worker.store.Team().MigrateTeamMembers(progress.LastTeamID, progress.LastUserID)
+		if err != nil {
+			return false, progress.ToJSON(), model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "app.team.migrate_team_members.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+		if result == nil {
+			// We haven't progressed. That means that we've reached the end of this stage of the migration, and should now advance to the next stage.
+			progress.LastUserID = strings.Repeat("0", 26)
+			progress.CurrentTable = "ChannelMembers"
+			return false, progress.ToJSON(), nil
+		}
+
+		progress.LastTeamID = result["TeamId"]
+		progress.LastUserID = result["UserId"]
+	} else if progress.CurrentTable == "ChannelMembers" {
+		// Run a ChannelMembers migration batch.
+		data, err := worker.store.Channel().MigrateChannelMembers(progress.LastChannelID, progress.LastUserID)
+		if err != nil {
+			return false, progress.ToJSON(), model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "app.channel.migrate_channel_members.select.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+		if data == nil {
+			// We haven't progressed. That means we've reached the end of this final stage of the migration.
+
+			return true, progress.ToJSON(), nil
+		}
+
+		progress.LastChannelID = data["ChannelId"]
+		progress.LastUserID = data["UserId"]
+	}
+
+	return false, progress.ToJSON(), nil
+}

server/channels/jobs/migrations/helper_test.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/stretchr/testify/require"
+)
+
+func Setup(tb testing.TB) store.Store {
+	store := mainHelper.GetStore()
+	store.DropAllTables()
+	return store
+}
+
+func deleteAllJobsByTypeAndMigrationKey(t *testing.T, store store.Store, jobType string, migrationKey string) {
+	ctx := request.TestContext(t)
+	jobs, err := store.Job().GetAllByType(ctx, model.JobTypeMigrations)
+	require.NoError(t, err)
+
+	for _, job := range jobs {
+		if key, ok := job.Data[JobDataKeyMigration]; ok && key == migrationKey {
+			_, err = store.Job().Delete(job.Id)
+			require.NoError(t, err)
+		}
+	}
+}

server/channels/jobs/migrations/main_test.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost/server/v8/channels/testlib"
+)
+
+var mainHelper *testlib.MainHelper
+
+func TestMain(m *testing.M) {
+	var options = testlib.HelperOptions{
+		EnableStore:     true,
+		EnableResources: true,
+	}
+
+	mainHelper = testlib.NewMainHelperWithOptions(&options)
+	defer mainHelper.Close()
+
+	mainHelper.Main(m)
+}

server/channels/jobs/migrations/migrations.go

@@ -0,0 +1,55 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"net/http"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+const (
+	MigrationStateUnscheduled = "unscheduled"
+	MigrationStateInProgress  = "in_progress"
+	MigrationStateCompleted   = "completed"
+
+	JobDataKeyMigration         = "migration_key"
+	JobDataKeyMigrationLastDone = "last_done"
+)
+
+func MakeMigrationsList() []string {
+	return []string{
+		model.MigrationKeyAdvancedPermissionsPhase2,
+	}
+}
+
+func GetMigrationState(rctx request.CTX, migration string, store store.Store) (string, *model.Job, *model.AppError) {
+	if _, err := store.System().GetByName(migration); err == nil {
+		return MigrationStateCompleted, nil, nil
+	}
+
+	jobs, err := store.Job().GetAllByType(rctx, model.JobTypeMigrations)
+	if err != nil {
+		return "", nil, model.NewAppError("GetMigrationState", "app.job.get_all.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	for _, job := range jobs {
+		if key, ok := job.Data[JobDataKeyMigration]; ok {
+			if key != migration {
+				continue
+			}
+
+			switch job.Status {
+			case model.JobStatusInProgress, model.JobStatusPending:
+				return MigrationStateInProgress, job, nil
+			default:
+				return MigrationStateUnscheduled, job, nil
+			}
+		}
+	}
+
+	return MigrationStateUnscheduled, nil, nil
+}

server/channels/jobs/migrations/migrations_test.go

@@ -0,0 +1,105 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+)
+
+func TestGetMigrationState(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+	}
+	store := Setup(t)
+	ctx := request.TestContext(t)
+
+	migrationKey := model.NewId()
+
+	deleteAllJobsByTypeAndMigrationKey(t, store, model.JobTypeMigrations, migrationKey)
+
+	// Test with no job yet.
+	state, job, err := GetMigrationState(ctx, migrationKey, store)
+	assert.Nil(t, err)
+	assert.Nil(t, job)
+	assert.Equal(t, "unscheduled", state)
+
+	// Test with the system table showing the migration as done.
+	system := model.System{
+		Name:  migrationKey,
+		Value: "true",
+	}
+	nErr := store.System().Save(&system)
+	assert.NoError(t, nErr)
+
+	state, job, err = GetMigrationState(ctx, migrationKey, store)
+	assert.Nil(t, err)
+	assert.Nil(t, job)
+	assert.Equal(t, "completed", state)
+
+	_, nErr = store.System().PermanentDeleteByName(migrationKey)
+	assert.NoError(t, nErr)
+
+	// Test with a job scheduled in "pending" state.
+	j1 := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: model.GetMillis(),
+		Data: map[string]string{
+			JobDataKeyMigration: migrationKey,
+		},
+		Status: model.JobStatusPending,
+		Type:   model.JobTypeMigrations,
+	}
+
+	j1, nErr = store.Job().Save(j1)
+	require.NoError(t, nErr)
+
+	state, job, err = GetMigrationState(ctx, migrationKey, store)
+	assert.Nil(t, err)
+	assert.Equal(t, j1.Id, job.Id)
+	assert.Equal(t, "in_progress", state)
+
+	// Test with a job scheduled in "in progress" state.
+	j2 := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: j1.CreateAt + 1,
+		Data: map[string]string{
+			JobDataKeyMigration: migrationKey,
+		},
+		Status: model.JobStatusInProgress,
+		Type:   model.JobTypeMigrations,
+	}
+
+	j2, nErr = store.Job().Save(j2)
+	require.NoError(t, nErr)
+
+	state, job, err = GetMigrationState(ctx, migrationKey, store)
+	assert.Nil(t, err)
+	assert.Equal(t, j2.Id, job.Id)
+	assert.Equal(t, "in_progress", state)
+
+	// Test with a job scheduled in "error" state.
+	j3 := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: j2.CreateAt + 1,
+		Data: map[string]string{
+			JobDataKeyMigration: migrationKey,
+		},
+		Status: model.JobStatusError,
+		Type:   model.JobTypeMigrations,
+	}
+
+	j3, nErr = store.Job().Save(j3)
+	require.NoError(t, nErr)
+
+	state, job, err = GetMigrationState(ctx, migrationKey, store)
+	assert.Nil(t, err)
+	assert.Equal(t, j3.Id, job.Id)
+	assert.Equal(t, "unscheduled", state)
+}

server/channels/jobs/migrations/scheduler.go

@@ -0,0 +1,110 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+const (
+	MigrationJobWedgedTimeoutMilliseconds = 3600000 // 1 hour
+)
+
+type Scheduler struct {
+	jobServer              *jobs.JobServer
+	store                  store.Store
+	allMigrationsCompleted bool
+}
+
+var _ jobs.Scheduler = (*Scheduler)(nil)
+
+func MakeScheduler(jobServer *jobs.JobServer, store store.Store) *Scheduler {
+	return &Scheduler{jobServer, store, false}
+}
+
+func (scheduler *Scheduler) Enabled(_ *model.Config) bool {
+	return true
+}
+
+//nolint:unparam
+func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
+	if scheduler.allMigrationsCompleted {
+		return nil
+	}
+
+	nextTime := time.Now().Add(60 * time.Second)
+	return &nextTime
+}
+
+//nolint:unparam
+func (scheduler *Scheduler) ScheduleJob(rctx request.CTX, cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
+	rctx.Logger().Debug("Scheduling Job", mlog.String("scheduler", model.JobTypeMigrations))
+
+	// Work through the list of migrations in order. Schedule the first one that isn't done (assuming it isn't in progress already).
+	for _, key := range MakeMigrationsList() {
+		state, job, err := GetMigrationState(rctx, key, scheduler.store)
+		if err != nil {
+			rctx.Logger().Error("Failed to determine status of migration: ", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key), mlog.Err(err))
+			return nil, nil
+		}
+
+		logger := rctx.Logger().With(jobs.JobLoggerFields(job)...)
+
+		if state == MigrationStateCompleted {
+			// This migration is done. Continue to check the next.
+			continue
+		}
+
+		if state == MigrationStateInProgress {
+			// Check the migration job isn't wedged.
+			if job != nil && job.LastActivityAt < model.GetMillis()-MigrationJobWedgedTimeoutMilliseconds && job.CreateAt < model.GetMillis()-MigrationJobWedgedTimeoutMilliseconds {
+				logger.Warn("Job appears to be wedged. Rescheduling another instance.", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key))
+				if err := scheduler.jobServer.SetJobError(job, nil); err != nil {
+					logger.Error("Worker: Failed to set job error", mlog.String("scheduler", model.JobTypeMigrations), mlog.Err(err))
+				}
+				return scheduler.createJob(rctx, key, job)
+			}
+
+			return nil, nil
+		}
+
+		if state == MigrationStateUnscheduled {
+			logger.Debug("Scheduling a new job for migration.", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key))
+			return scheduler.createJob(rctx, key, job)
+		}
+
+		logger.Error("Unknown migration state. Not doing anything.", mlog.String("migration_state", state))
+		return nil, nil
+	}
+
+	// If we reached here, then there aren't any migrations left to run.
+	scheduler.allMigrationsCompleted = true
+	rctx.Logger().Debug("All migrations are complete.", mlog.String("scheduler", model.JobTypeMigrations))
+
+	return nil, nil
+}
+
+func (scheduler *Scheduler) createJob(rctx request.CTX, migrationKey string, lastJob *model.Job) (*model.Job, *model.AppError) {
+	var lastDone string
+	if lastJob != nil {
+		lastDone = lastJob.Data[JobDataKeyMigrationLastDone]
+	}
+
+	data := map[string]string{
+		JobDataKeyMigration:         migrationKey,
+		JobDataKeyMigrationLastDone: lastDone,
+	}
+
+	job, err := scheduler.jobServer.CreateJob(rctx, model.JobTypeMigrations, data)
+	if err != nil {
+		return nil, err
+	}
+	return job, nil
+}

server/channels/jobs/migrations/worker.go

@@ -0,0 +1,188 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"context"
+	"net/http"
+	"sync/atomic"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+const (
+	TimeBetweenBatches = 100
+)
+
+type Worker struct {
+	name      string
+	stop      chan struct{}
+	stopped   chan bool
+	jobs      chan model.Job
+	jobServer *jobs.JobServer
+	logger    mlog.LoggerIFace
+	store     store.Store
+	closed    atomic.Int32
+}
+
+func MakeWorker(jobServer *jobs.JobServer, store store.Store) *Worker {
+	const workerName = "Migrations"
+	worker := Worker{
+		name:      workerName,
+		stop:      make(chan struct{}),
+		stopped:   make(chan bool, 1),
+		jobs:      make(chan model.Job),
+		jobServer: jobServer,
+		logger:    jobServer.Logger().With(mlog.String("worker_name", workerName)),
+		store:     store,
+	}
+
+	return &worker
+}
+
+func (worker *Worker) Run() {
+	// Set to open if closed before. We are not bothered about multiple opens.
+	if worker.closed.CompareAndSwap(1, 0) {
+		worker.stop = make(chan struct{})
+	}
+	worker.logger.Debug("Worker started")
+
+	defer func() {
+		worker.logger.Debug("Worker finished")
+		worker.stopped <- true
+	}()
+
+	for {
+		select {
+		case <-worker.stop:
+			worker.logger.Debug("Worker received stop signal")
+			return
+		case job := <-worker.jobs:
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *Worker) Stop() {
+	// Set to close, and if already closed before, then return.
+	if !worker.closed.CompareAndSwap(0, 1) {
+		return
+	}
+	worker.logger.Debug("Worker stopping")
+	close(worker.stop)
+	<-worker.stopped
+}
+
+func (worker *Worker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+func (worker *Worker) IsEnabled(_ *model.Config) bool {
+	return true
+}
+
+func (worker *Worker) DoJob(job *model.Job) {
+	logger := worker.logger.With(jobs.JobLoggerFields(job)...)
+	logger.Debug("Worker: Received a new candidate job.")
+
+	defer worker.jobServer.HandleJobPanic(logger, job)
+
+	var appErr *model.AppError
+	job, appErr = worker.jobServer.ClaimJob(job)
+	if appErr != nil {
+		logger.Warn("Worker experienced an error while trying to claim job", mlog.Err(appErr))
+		return
+	} else if job == nil {
+		return
+	}
+
+	var cancelContext request.CTX = request.EmptyContext(worker.logger)
+	cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
+	cancelWatcherChan := make(chan struct{}, 1)
+	cancelContext = cancelContext.WithContext(cancelCtx)
+	go worker.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan)
+	defer cancelCancelWatcher()
+
+	for {
+		select {
+		case <-cancelWatcherChan:
+			logger.Debug("Worker: Job has been canceled via CancellationWatcher")
+			worker.setJobCanceled(logger, job)
+			return
+
+		case <-worker.stop:
+			logger.Debug("Worker: Job has been canceled via Worker Stop")
+			worker.setJobCanceled(logger, job)
+			return
+
+		case <-time.After(TimeBetweenBatches * time.Millisecond):
+			done, progress, err := worker.runMigration(job.Data[JobDataKeyMigration], job.Data[JobDataKeyMigrationLastDone])
+			if err != nil {
+				logger.Error("Worker: Failed to run migration", mlog.Err(err))
+				worker.setJobError(logger, job, err)
+				return
+			} else if done {
+				logger.Info("Worker: Job is complete")
+				worker.setJobSuccess(logger, job)
+				return
+			} else {
+				job.Data[JobDataKeyMigrationLastDone] = progress
+				if err := worker.jobServer.UpdateInProgressJobData(job); err != nil {
+					logger.Error("Worker: Failed to update migration status data for job", mlog.Err(err))
+					worker.setJobError(logger, job, err)
+					return
+				}
+			}
+		}
+	}
+}
+
+func (worker *Worker) setJobSuccess(logger mlog.LoggerIFace, job *model.Job) {
+	if err := worker.jobServer.SetJobSuccess(job); err != nil {
+		logger.Error("Worker: Failed to set success for job", mlog.Err(err))
+		worker.setJobError(logger, job, err)
+	}
+}
+
+func (worker *Worker) setJobError(logger mlog.LoggerIFace, job *model.Job, appError *model.AppError) {
+	if err := worker.jobServer.SetJobError(job, appError); err != nil {
+		logger.Error("Worker: Failed to set job error", mlog.Err(err))
+	}
+}
+
+func (worker *Worker) setJobCanceled(logger mlog.LoggerIFace, job *model.Job) {
+	if err := worker.jobServer.SetJobCanceled(job); err != nil {
+		logger.Error("Worker: Failed to mark job as canceled", mlog.Err(err))
+	}
+}
+
+// Return parameters:
+// - whether the migration is completed on this run (true) or still incomplete (false).
+// - the updated lastDone string for the migration.
+// - any error which may have occurred while running the migration.
+func (worker *Worker) runMigration(key string, lastDone string) (bool, string, *model.AppError) {
+	var done bool
+	var progress string
+	var err *model.AppError
+
+	switch key {
+	case model.MigrationKeyAdvancedPermissionsPhase2:
+		done, progress, err = worker.runAdvancedPermissionsPhase2Migration(lastDone)
+	default:
+		return false, "", model.NewAppError("MigrationsWorker.runMigration", "migrations.worker.run_migration.unknown_key", map[string]any{"key": key}, "", http.StatusInternalServerError)
+	}
+
+	if done {
+		if nErr := worker.store.System().Save(&model.System{Name: key, Value: "true"}); nErr != nil {
+			return false, "", model.NewAppError("runMigration", "migrations.system.save.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
+		}
+	}
+
+	return done, progress, err
+}

server/channels/jobs/mobile_session_metadata/scheduler.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package mobile_session_metadata
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const schedFreq = 24 * time.Hour
+
+func MakeScheduler(jobServer *jobs.JobServer) *jobs.PeriodicScheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.MetricsSettings.EnableClientMetrics
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeMobileSessionMetadata, schedFreq, isEnabled)
+}

server/channels/jobs/mobile_session_metadata/worker.go

@@ -0,0 +1,42 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package mobile_session_metadata
+
+import (
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/mattermost/mattermost/server/v8/einterfaces"
+)
+
+func MakeWorker(jobServer *jobs.JobServer, store store.Store, getMetrics func() einterfaces.MetricsInterface) *jobs.SimpleWorker {
+	const workerName = "MobileSessionMetadata"
+
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.MetricsSettings.EnableClientMetrics
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		metrics := getMetrics()
+		if metrics == nil {
+			return nil
+		}
+
+		versions, err := store.Session().GetMobileSessionMetadata()
+		if err != nil {
+			return err
+		}
+
+		metrics.ClearMobileClientSessionMetadata()
+		for _, v := range versions {
+			metrics.ObserveMobileClientSessionMetadata(v.Version, v.Platform, v.Count, v.NotificationDisabled)
+		}
+
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/notify_admin/install_plugin_scheduler.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package notify_admin
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const installPluginSchedFreq = 24 * time.Hour
+
+func MakeInstallPluginScheduler(jobServer *jobs.JobServer, license *model.License, jobType string) *jobs.PeriodicScheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		enabled := jobType == model.JobTypeInstallPluginNotifyAdmin
+		mlog.Debug("Scheduler: isEnabled: "+strconv.FormatBool(enabled), mlog.String("scheduler", jobType))
+		return enabled
+	}
+	return jobs.NewPeriodicScheduler(jobServer, jobType, installPluginSchedFreq, isEnabled)
+}

server/channels/jobs/notify_admin/scheduler.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package notify_admin
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const schedFreq = 24 * time.Hour
+
+func MakeScheduler(jobServer *jobs.JobServer, license *model.License, jobType string) *jobs.PeriodicScheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		enabled := license != nil && *license.Features.Cloud
+		mlog.Debug("Scheduler: isEnabled: "+strconv.FormatBool(enabled), mlog.String("scheduler", jobType))
+		return enabled
+	}
+	return jobs.NewPeriodicScheduler(jobServer, jobType, schedFreq, isEnabled)
+}

server/channels/jobs/notify_admin/worker.go

@@ -0,0 +1,74 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package notify_admin
+
+import (
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const (
+	UpgradeNotifyJobName = "UpgradeNotifyAdmin"
+	TrialNotifyJobName   = "TrialNotifyAdmin"
+	InstallNotifyJobName = "InstallNotifyAdmin"
+)
+
+type AppIface interface {
+	DoCheckForAdminNotifications(trial bool) *model.AppError
+}
+
+func MakeUpgradeNotifyWorker(jobServer *jobs.JobServer, license *model.License, app AppIface) *jobs.SimpleWorker {
+	isEnabled := func(_ *model.Config) bool {
+		return license != nil && license.Features != nil && *license.Features.Cloud
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		appErr := app.DoCheckForAdminNotifications(false)
+		if appErr != nil {
+			return appErr
+		}
+
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(UpgradeNotifyJobName, jobServer, execute, isEnabled)
+	return worker
+}
+
+func MakeTrialNotifyWorker(jobServer *jobs.JobServer, license *model.License, app AppIface) *jobs.SimpleWorker {
+	isEnabled := func(_ *model.Config) bool {
+		return license != nil && license.Features != nil && *license.Features.Cloud
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		appErr := app.DoCheckForAdminNotifications(true)
+		if appErr != nil {
+			return appErr
+		}
+
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(TrialNotifyJobName, jobServer, execute, isEnabled)
+	return worker
+}
+
+func MakeInstallPluginNotifyWorker(jobServer *jobs.JobServer, app AppIface) *jobs.SimpleWorker {
+	isEnabled := func(_ *model.Config) bool {
+		return true
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		appErr := app.DoCheckForAdminNotifications(false)
+		if appErr != nil {
+			return appErr
+		}
+
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(InstallNotifyJobName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/notify_expiring_access_tokens/scheduler.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package notify_expiring_access_tokens
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const schedFreq = 1 * time.Hour
+
+func MakeScheduler(jobServer *jobs.JobServer) *jobs.PeriodicScheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.ServiceSettings.EnableUserAccessTokens
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeNotifyExpiringAccessTokens, schedFreq, isEnabled)
+}

server/channels/jobs/notify_expiring_access_tokens/worker.go

@@ -0,0 +1,31 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+// Package notify_expiring_access_tokens warns the owners of model.UserAccessToken
+// ("Personal Access Token" in config and to users, e.g.
+// ServiceSettings.MaximumPersonalAccessTokenLifetimeDays) as their tokens approach
+// expiry.
+package notify_expiring_access_tokens
+
+import (
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+// MakeWorker creates a worker that warns the owners of personal access tokens
+// that are approaching expiry. The actual notification logic lives in the app
+// layer and is passed in as notifyExpiring, mirroring the expirynotify job.
+func MakeWorker(jobServer *jobs.JobServer, notifyExpiring func() error) *jobs.SimpleWorker {
+	const workerName = "NotifyExpiringAccessTokens"
+
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.ServiceSettings.EnableUserAccessTokens
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		return notifyExpiring()
+	}
+	return jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+}

server/channels/jobs/plugins/scheduler.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package plugins
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const schedFreq = 24 * time.Hour
+
+func MakeScheduler(jobServer *jobs.JobServer) *jobs.PeriodicScheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return true
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypePlugins, schedFreq, isEnabled)
+}

server/channels/jobs/plugins/worker.go

@@ -0,0 +1,108 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package plugins
+
+import (
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+type AppIface interface {
+	DeleteAllExpiredPluginKeys() *model.AppError
+}
+
+type Worker struct {
+	name      string
+	stop      chan bool
+	stopped   chan bool
+	jobs      chan model.Job
+	jobServer *jobs.JobServer
+	logger    mlog.LoggerIFace
+	app       AppIface
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface) *Worker {
+	const workerName = "Plugins"
+	worker := Worker{
+		name:      workerName,
+		stop:      make(chan bool, 1),
+		stopped:   make(chan bool, 1),
+		jobs:      make(chan model.Job),
+		jobServer: jobServer,
+		logger:    jobServer.Logger().With(mlog.String("worker_name", workerName)),
+		app:       app,
+	}
+
+	return &worker
+}
+
+func (worker *Worker) Run() {
+	worker.logger.Debug("Worker started")
+
+	defer func() {
+		worker.logger.Debug("Worker finished")
+		worker.stopped <- true
+	}()
+
+	for {
+		select {
+		case <-worker.stop:
+			worker.logger.Debug("Worker received stop signal")
+			return
+		case job := <-worker.jobs:
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *Worker) Stop() {
+	worker.logger.Debug("Worker stopping")
+	worker.stop <- true
+	<-worker.stopped
+}
+
+func (worker *Worker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+func (worker *Worker) IsEnabled(cfg *model.Config) bool {
+	return true
+}
+
+func (worker *Worker) DoJob(job *model.Job) {
+	logger := worker.logger.With(jobs.JobLoggerFields(job)...)
+	logger.Debug("Worker: Received a new candidate job.")
+
+	var appErr *model.AppError
+	job, appErr = worker.jobServer.ClaimJob(job)
+	if appErr != nil {
+		logger.Warn("Worker experienced an error while trying to claim job", mlog.Err(appErr))
+		return
+	} else if job == nil {
+		return
+	}
+
+	if err := worker.app.DeleteAllExpiredPluginKeys(); err != nil {
+		logger.Error("Worker: Failed to delete expired keys", mlog.Err(err))
+		worker.setJobError(logger, job, err)
+		return
+	}
+
+	logger.Info("Worker: Job is complete")
+	worker.setJobSuccess(logger, job)
+}
+
+func (worker *Worker) setJobSuccess(logger mlog.LoggerIFace, job *model.Job) {
+	if err := worker.jobServer.SetJobSuccess(job); err != nil {
+		logger.Error("Worker: Failed to set success for job", mlog.Err(err))
+		worker.setJobError(logger, job, err)
+	}
+}
+
+func (worker *Worker) setJobError(logger mlog.LoggerIFace, job *model.Job, appError *model.AppError) {
+	if err := worker.jobServer.SetJobError(job, appError); err != nil {
+		logger.Error("Worker: Failed to set job error", mlog.Err(err))
+	}
+}

server/channels/jobs/post_persistent_notifications/scheduler.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package post_persistent_notifications
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+type Scheduler struct {
+	*jobs.PeriodicScheduler
+}
+
+func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, _ time.Time, _ bool, _ *model.Job) *time.Time {
+	nextTime := time.Now().Add((time.Duration(*cfg.ServiceSettings.PersistentNotificationIntervalMinutes) * time.Minute) / 2)
+	return &nextTime
+}
+
+func MakeScheduler(jobServer *jobs.JobServer, licenseFunc func() *model.License) *Scheduler {
+	enabledFunc := func(_ *model.Config) bool {
+		return model.MinimumProfessionalLicense(licenseFunc())
+	}
+	return &Scheduler{jobs.NewPeriodicScheduler(jobServer, model.JobTypePostPersistentNotifications, 0, enabledFunc)}
+}

server/channels/jobs/post_persistent_notifications/worker.go

@@ -0,0 +1,29 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package post_persistent_notifications
+
+import (
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+type AppIface interface {
+	SendPersistentNotifications() error
+	IsPersistentNotificationsEnabled() bool
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface) *jobs.SimpleWorker {
+	const workerName = "PostPersistentNotifications"
+
+	isEnabled := func(_ *model.Config) bool {
+		return app.IsPersistentNotificationsEnabled()
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+		return app.SendPersistentNotifications()
+	}
+	worker := jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/product_notices/scheduler.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package product_notices
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+type Scheduler struct {
+	*jobs.PeriodicScheduler
+}
+
+func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, _ time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
+	nextTime := time.Now().Add(time.Duration(*cfg.AnnouncementSettings.NoticesFetchFrequency) * time.Second)
+	return &nextTime
+}
+
+func MakeScheduler(jobServer *jobs.JobServer) *Scheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.AnnouncementSettings.AdminNoticesEnabled || *cfg.AnnouncementSettings.UserNoticesEnabled
+	}
+	return &Scheduler{PeriodicScheduler: jobs.NewPeriodicScheduler(jobServer, model.JobTypeProductNotices, 0, isEnabled)}
+}

server/channels/jobs/product_notices/worker.go

@@ -0,0 +1,33 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package product_notices
+
+import (
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+type AppIface interface {
+	UpdateProductNotices() *model.AppError
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface) *jobs.SimpleWorker {
+	const workerName = "ProductNotices"
+
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.AnnouncementSettings.AdminNoticesEnabled || *cfg.AnnouncementSettings.UserNoticesEnabled
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		if err := app.UpdateProductNotices(); err != nil {
+			logger.Error("Worker: Failed to fetch product notices", mlog.Err(err))
+			return err
+		}
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/recap/worker.go

@@ -0,0 +1,141 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package recap
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+type AppIface interface {
+	ProcessRecapChannelWithOptions(rctx request.CTX, recapID, channelID, userID, agentID string, options model.RecapProcessingOptions) (*model.RecapChannelResult, *model.AppError)
+	Publish(message *model.WebSocketEvent)
+}
+
+func MakeWorker(jobServer *jobs.JobServer, storeInstance store.Store, appInstance AppIface) *jobs.SimpleWorker {
+	isEnabled := func(cfg *model.Config) bool {
+		return cfg.AIRecapsEnabled()
+	}
+
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+		return processRecapJob(logger, job, storeInstance, appInstance, func(progress int64) {
+			_ = jobServer.SetJobProgress(job, progress)
+		})
+	}
+
+	return jobs.NewSimpleWorker("Recap", jobServer, execute, isEnabled)
+}
+
+func processRecapJob(logger mlog.LoggerIFace, job *model.Job, storeInstance store.Store, appInstance AppIface, setProgress func(int64)) error {
+	recapID := job.Data["recap_id"]
+	userID := job.Data["user_id"]
+	channelIDs := strings.Split(job.Data["channel_ids"], ",")
+	agentID := job.Data["agent_id"]
+	options := model.RecapProcessingOptions{
+		TimePeriod:         job.Data["time_period"],
+		CustomInstructions: job.Data["custom_instructions"],
+	}
+
+	logger.Info("Starting recap job",
+		mlog.String("recap_id", recapID),
+		mlog.String("agent_id", agentID),
+		mlog.Int("channel_count", len(channelIDs)))
+
+	// Update status to processing
+	_ = storeInstance.Recap().UpdateRecapStatus(recapID, model.RecapStatusProcessing)
+	publishRecapUpdate(appInstance, recapID, userID)
+
+	totalMessages := 0
+	successfulChannels := []string{}
+	failedChannels := []string{}
+
+	for i, channelID := range channelIDs {
+		// Update progress
+		progress := int64((i * 100) / len(channelIDs))
+		if setProgress != nil {
+			setProgress(progress)
+		}
+
+		// Process the channel - use a context with the user's session so that
+		// session-dependent code (e.g. auto-translation supplements) works correctly.
+		rctx := request.EmptyContext(logger).WithSession(&model.Session{UserId: userID})
+		result, err := appInstance.ProcessRecapChannelWithOptions(rctx, recapID, channelID, userID, agentID, options)
+		if err != nil {
+			logger.Warn("Failed to process channel",
+				mlog.String("channel_id", channelID),
+				mlog.Err(err))
+			failedChannels = append(failedChannels, channelID)
+			continue
+		}
+
+		if !result.Success {
+			logger.Warn("Channel processing unsuccessful", mlog.String("channel_id", channelID))
+			failedChannels = append(failedChannels, channelID)
+			continue
+		}
+
+		totalMessages += result.MessageCount
+		successfulChannels = append(successfulChannels, channelID)
+	}
+
+	// Update recap with final data (title is already set by user in CreateRecap)
+	recap, err := storeInstance.Recap().GetRecap(recapID)
+	if err != nil || recap == nil {
+		logger.Warn("Recap no longer available while finalizing job",
+			mlog.String("recap_id", recapID),
+			mlog.Err(err))
+		return nil
+	}
+	recap.TotalMessageCount = totalMessages
+	recap.UpdateAt = model.GetMillis()
+
+	if len(failedChannels) > 0 && len(successfulChannels) == 0 {
+		recap.Status = model.RecapStatusFailed
+		_, err := storeInstance.Recap().UpdateRecap(recap)
+		if err != nil {
+			logger.Error("Failed to update recap", mlog.Err(err))
+			return fmt.Errorf("failed to update recap: %w", err)
+		}
+		publishRecapUpdate(appInstance, recapID, userID)
+		return fmt.Errorf("all channels failed to process")
+	} else if len(failedChannels) > 0 {
+		recap.Status = model.RecapStatusCompleted
+		_, err := storeInstance.Recap().UpdateRecap(recap)
+		if err != nil {
+			logger.Error("Failed to update recap", mlog.Err(err))
+			return fmt.Errorf("failed to update recap: %w", err)
+		}
+		publishRecapUpdate(appInstance, recapID, userID)
+		logger.Warn("Some channels failed", mlog.Int("failed_count", len(failedChannels)))
+		// Job succeeds with warning
+	} else {
+		recap.Status = model.RecapStatusCompleted
+		_, err := storeInstance.Recap().UpdateRecap(recap)
+		if err != nil {
+			logger.Error("Failed to update recap", mlog.Err(err))
+			return fmt.Errorf("failed to update recap: %w", err)
+		}
+		publishRecapUpdate(appInstance, recapID, userID)
+	}
+
+	logger.Info("Recap job completed",
+		mlog.String("recap_id", recapID),
+		mlog.Int("successful_channels", len(successfulChannels)),
+		mlog.Int("failed_channels", len(failedChannels)))
+
+	return nil
+}
+
+func publishRecapUpdate(appInstance AppIface, recapID, userID string) {
+	message := model.NewWebSocketEvent(model.WebsocketEventRecapUpdated, "", "", userID, nil, "")
+	message.Add("recap_id", recapID)
+	appInstance.Publish(message)
+}

server/channels/jobs/recap/worker_test.go

@@ -0,0 +1,167 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package recap
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+type MockAppIface struct {
+	mock.Mock
+}
+
+func (m *MockAppIface) ProcessRecapChannelWithOptions(rctx request.CTX, recapID, channelID, userID, agentID string, options model.RecapProcessingOptions) (*model.RecapChannelResult, *model.AppError) {
+	args := m.Called(rctx, recapID, channelID, userID, agentID, options)
+	if args.Get(0) == nil {
+		return nil, args.Get(1).(*model.AppError)
+	}
+	return args.Get(0).(*model.RecapChannelResult), nil
+}
+
+func (m *MockAppIface) Publish(message *model.WebSocketEvent) {
+	m.Called(message)
+}
+
+func TestProcessRecapJob(t *testing.T) {
+	logger := mlog.CreateConsoleTestLogger(t)
+	job := &model.Job{
+		Data: map[string]string{
+			"recap_id":    "recap1",
+			"user_id":     "user1",
+			"channel_ids": "channel1,channel2",
+			"agent_id":    "agent1",
+		},
+	}
+
+	t.Run("successful processing", func(t *testing.T) {
+		mockStore := &mocks.Store{}
+		mockRecapStore := &mocks.RecapStore{}
+		mockStore.On("Recap").Return(mockRecapStore)
+
+		mockApp := &MockAppIface{}
+
+		// Setup expectations
+		mockRecapStore.On("UpdateRecapStatus", "recap1", model.RecapStatusProcessing).Return(nil)
+		mockApp.On("Publish", mock.Anything).Return()
+
+		mockApp.On("ProcessRecapChannelWithOptions", mock.Anything, "recap1", "channel1", "user1", "agent1", model.RecapProcessingOptions{}).Return(&model.RecapChannelResult{
+			ChannelID:    "channel1",
+			Success:      true,
+			MessageCount: 10,
+		}, nil)
+
+		mockApp.On("ProcessRecapChannelWithOptions", mock.Anything, "recap1", "channel2", "user1", "agent1", model.RecapProcessingOptions{}).Return(&model.RecapChannelResult{
+			ChannelID:    "channel2",
+			Success:      true,
+			MessageCount: 5,
+		}, nil)
+
+		recap := &model.Recap{Id: "recap1"}
+		mockRecapStore.On("GetRecap", "recap1").Return(recap, nil)
+		mockRecapStore.On("UpdateRecap", mock.MatchedBy(func(r *model.Recap) bool {
+			return r.TotalMessageCount == 15 && r.Status == model.RecapStatusCompleted
+		})).Return(recap, nil)
+
+		err := processRecapJob(logger, job, mockStore, mockApp, nil)
+		require.NoError(t, err)
+	})
+
+	t.Run("partial failure", func(t *testing.T) {
+		mockStore := &mocks.Store{}
+		mockRecapStore := &mocks.RecapStore{}
+		mockStore.On("Recap").Return(mockRecapStore)
+
+		mockApp := &MockAppIface{}
+
+		mockRecapStore.On("UpdateRecapStatus", "recap1", model.RecapStatusProcessing).Return(nil)
+		mockApp.On("Publish", mock.Anything).Return()
+
+		mockApp.On("ProcessRecapChannelWithOptions", mock.Anything, "recap1", "channel1", "user1", "agent1", model.RecapProcessingOptions{}).Return(&model.RecapChannelResult{
+			ChannelID:    "channel1",
+			Success:      true,
+			MessageCount: 10,
+		}, nil)
+
+		mockApp.On("ProcessRecapChannelWithOptions", mock.Anything, "recap1", "channel2", "user1", "agent1", model.RecapProcessingOptions{}).Return(nil, model.NewAppError("fail", "fail", nil, "", 500))
+
+		recap := &model.Recap{Id: "recap1"}
+		mockRecapStore.On("GetRecap", "recap1").Return(recap, nil)
+		mockRecapStore.On("UpdateRecap", mock.MatchedBy(func(r *model.Recap) bool {
+			return r.TotalMessageCount == 10 && r.Status == model.RecapStatusCompleted
+		})).Return(recap, nil)
+
+		err := processRecapJob(logger, job, mockStore, mockApp, nil)
+		require.NoError(t, err)
+	})
+
+	t.Run("complete failure", func(t *testing.T) {
+		mockStore := &mocks.Store{}
+		mockRecapStore := &mocks.RecapStore{}
+		mockStore.On("Recap").Return(mockRecapStore)
+
+		mockApp := &MockAppIface{}
+
+		mockRecapStore.On("UpdateRecapStatus", "recap1", model.RecapStatusProcessing).Return(nil)
+		mockApp.On("Publish", mock.Anything).Return()
+
+		mockApp.On("ProcessRecapChannelWithOptions", mock.Anything, "recap1", "channel1", "user1", "agent1", model.RecapProcessingOptions{}).Return(nil, model.NewAppError("fail", "fail", nil, "", 500))
+		mockApp.On("ProcessRecapChannelWithOptions", mock.Anything, "recap1", "channel2", "user1", "agent1", model.RecapProcessingOptions{}).Return(nil, model.NewAppError("fail", "fail", nil, "", 500))
+
+		recap := &model.Recap{Id: "recap1"}
+		mockRecapStore.On("GetRecap", "recap1").Return(recap, nil)
+		mockRecapStore.On("UpdateRecap", mock.MatchedBy(func(r *model.Recap) bool {
+			return r.TotalMessageCount == 0 && r.Status == model.RecapStatusFailed
+		})).Return(recap, nil)
+
+		err := processRecapJob(logger, job, mockStore, mockApp, nil)
+		require.Error(t, err)
+		require.Equal(t, "all channels failed to process", err.Error())
+	})
+
+	t.Run("passes scheduled options to channel processing", func(t *testing.T) {
+		jobWithOptions := &model.Job{
+			Data: map[string]string{
+				"recap_id":            "recap1",
+				"user_id":             "user1",
+				"channel_ids":         "channel1",
+				"agent_id":            "agent1",
+				"time_period":         model.TimePeriodLastWeek,
+				"custom_instructions": "Focus on blockers",
+			},
+		}
+
+		mockStore := &mocks.Store{}
+		mockRecapStore := &mocks.RecapStore{}
+		mockStore.On("Recap").Return(mockRecapStore)
+
+		mockApp := &MockAppIface{}
+		mockRecapStore.On("UpdateRecapStatus", "recap1", model.RecapStatusProcessing).Return(nil)
+		mockApp.On("Publish", mock.Anything).Return()
+		mockApp.On("ProcessRecapChannelWithOptions", mock.Anything, "recap1", "channel1", "user1", "agent1", model.RecapProcessingOptions{
+			TimePeriod:         model.TimePeriodLastWeek,
+			CustomInstructions: "Focus on blockers",
+		}).Return(&model.RecapChannelResult{
+			ChannelID:    "channel1",
+			Success:      true,
+			MessageCount: 3,
+		}, nil)
+
+		recap := &model.Recap{Id: "recap1"}
+		mockRecapStore.On("GetRecap", "recap1").Return(recap, nil)
+		mockRecapStore.On("UpdateRecap", mock.MatchedBy(func(r *model.Recap) bool {
+			return r.TotalMessageCount == 3 && r.Status == model.RecapStatusCompleted
+		})).Return(recap, nil)
+
+		err := processRecapJob(logger, jobWithOptions, mockStore, mockApp, nil)
+		require.NoError(t, err)
+		mockApp.AssertExpectations(t)
+	})
+}

server/channels/jobs/refresh_materialized_views/scheduler.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package refresh_materialized_views
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+func MakeScheduler(jobServer *jobs.JobServer, sqlDriverName string) *jobs.DailyScheduler {
+	startTime := func(cfg *model.Config) *time.Time {
+		parsedTime, err := time.Parse("15:04", *cfg.ServiceSettings.RefreshPostStatsRunTime)
+		if err == nil {
+			return &parsedTime
+		}
+		return nil
+	}
+	isEnabled := func(cfg *model.Config) bool {
+		return sqlDriverName == model.DatabaseDriverPostgres
+	}
+	return jobs.NewDailyScheduler(jobServer, model.JobTypeRefreshMaterializedViews, startTime, isEnabled)
+}

server/channels/jobs/refresh_materialized_views/worker.go

@@ -0,0 +1,34 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package refresh_materialized_views
+
+import (
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+)
+
+const jobName = "RefreshMaterializedViews"
+
+func MakeWorker(jobServer *jobs.JobServer, sqlDriverName string) *jobs.SimpleWorker {
+	isEnabled := func(cfg *model.Config) bool {
+		return sqlDriverName == model.DatabaseDriverPostgres
+	}
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+
+		if err := jobServer.Store.Post().RefreshPostStats(); err != nil {
+			return err
+		}
+
+		if err := jobServer.Store.FileInfo().RefreshFileStats(); err != nil {
+			return err
+		}
+
+		return jobServer.Store.User().RefreshPostStatsForUsers()
+	}
+
+	worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/resend_invitation_email/worker.go

@@ -0,0 +1,252 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+package resend_invitation_email
+
+import (
+	"encoding/json"
+	"net/http"
+	"os"
+	"strconv"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/configservice"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+const FourtyEightHoursInMillis int64 = 172800000
+
+type AppIface interface {
+	configservice.ConfigService
+	GetUserByEmail(email string) (*model.User, *model.AppError)
+	GetTeamMembersByIds(teamID string, userIDs []string, restrictions *model.ViewUsersRestrictions) ([]*model.TeamMember, *model.AppError)
+	InviteNewUsersToTeamGracefully(rctx request.CTX, memberInvite *model.MemberInvite, teamID, senderId string, reminderInterval string) ([]*model.EmailInviteWithError, *model.AppError)
+}
+
+type ResendInvitationEmailWorker struct {
+	name      string
+	stop      chan bool
+	stopped   chan bool
+	jobs      chan model.Job
+	jobServer *jobs.JobServer
+	logger    mlog.LoggerIFace
+	app       AppIface
+	store     store.Store
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface, store store.Store) *ResendInvitationEmailWorker {
+	const workerName = "ResendInvitationEmail"
+	worker := ResendInvitationEmailWorker{
+		name:      workerName,
+		stop:      make(chan bool, 1),
+		stopped:   make(chan bool, 1),
+		jobs:      make(chan model.Job),
+		jobServer: jobServer,
+		logger:    jobServer.Logger().With(mlog.String("worker_name", workerName)),
+		app:       app,
+		store:     store,
+	}
+	return &worker
+}
+
+func (rseworker *ResendInvitationEmailWorker) Run() {
+	rseworker.logger.Debug("Worker started")
+
+	defer func() {
+		rseworker.logger.Debug("Worker finished")
+		rseworker.stopped <- true
+	}()
+
+	for {
+		select {
+		case <-rseworker.stop:
+			rseworker.logger.Debug("Worker received stop signal")
+			return
+		case job := <-rseworker.jobs:
+			rseworker.DoJob(&job)
+		}
+	}
+}
+
+func (rseworker *ResendInvitationEmailWorker) IsEnabled(cfg *model.Config) bool {
+	return *cfg.ServiceSettings.EnableEmailInvitations
+}
+
+func (rseworker *ResendInvitationEmailWorker) Stop() {
+	rseworker.logger.Debug("Worker stopping")
+	rseworker.stop <- true
+	<-rseworker.stopped
+}
+
+func (rseworker *ResendInvitationEmailWorker) JobChannel() chan<- model.Job {
+	return rseworker.jobs
+}
+
+func (rseworker *ResendInvitationEmailWorker) DoJob(job *model.Job) {
+	logger := rseworker.logger.With(jobs.JobLoggerFields(job)...)
+	logger.Debug("Worker: Received a new candidate job.")
+	defer rseworker.jobServer.HandleJobPanic(logger, job)
+
+	elapsedTimeSinceSchedule, DurationInMillis := rseworker.GetDurations(job)
+	if elapsedTimeSinceSchedule > DurationInMillis {
+		if appErr := rseworker.ResendEmails(logger, job, "48"); appErr != nil {
+			rseworker.setJobError(logger, job, appErr)
+			return
+		}
+		rseworker.TearDown(logger, job)
+	}
+}
+
+func (rseworker *ResendInvitationEmailWorker) setJobSuccess(logger mlog.LoggerIFace, job *model.Job) {
+	if err := rseworker.jobServer.SetJobSuccess(job); err != nil {
+		logger.Error("Worker: Failed to set success for job", mlog.Err(err))
+		rseworker.setJobError(logger, job, err)
+	}
+}
+
+func (rseworker *ResendInvitationEmailWorker) setJobError(logger mlog.LoggerIFace, job *model.Job, appError *model.AppError) {
+	if err := rseworker.jobServer.SetJobError(job, appError); err != nil {
+		logger.Error("Worker: Failed to set job error", mlog.Err(err))
+	}
+}
+
+func (rseworker *ResendInvitationEmailWorker) cleanEmailData(emailStringData string) ([]string, error) {
+	// emailStringData looks like this ["user1@gmail.com","user2@gmail.com"]
+	emails := []string{}
+	err := json.Unmarshal([]byte(emailStringData), &emails)
+	if err != nil {
+		return nil, err
+	}
+
+	return emails, nil
+}
+
+func (rseworker *ResendInvitationEmailWorker) cleanChannelsData(channelStringData string) ([]string, error) {
+	// channelStringData looks like this ["uuuiiiiidddd","uuuiiiiidddd"]
+	channels := []string{}
+	err := json.Unmarshal([]byte(channelStringData), &channels)
+	if err != nil {
+		return nil, err
+	}
+
+	return channels, nil
+}
+
+func (rseworker *ResendInvitationEmailWorker) removeAlreadyJoined(teamID string, emailList []string) []string {
+	var notJoinedYet []string
+	for _, email := range emailList {
+		// check if the user with this email is on the system already
+		user, appErr := rseworker.app.GetUserByEmail(email)
+		if appErr != nil {
+			notJoinedYet = append(notJoinedYet, email)
+			continue
+		}
+		// now we check if they are part of the team already
+		userID := []string{user.Id}
+		members, appErr := rseworker.app.GetTeamMembersByIds(teamID, userID, nil)
+		if len(members) == 0 || appErr != nil {
+			notJoinedYet = append(notJoinedYet, email)
+		}
+	}
+
+	return notJoinedYet
+}
+
+func (rseworker *ResendInvitationEmailWorker) GetDurations(job *model.Job) (int64, int64) {
+	scheduledAt, _ := strconv.ParseInt(job.Data["scheduledAt"], 10, 64)
+	now := model.GetMillis()
+
+	elapsedTimeSinceSchedule := now - scheduledAt
+
+	duration := os.Getenv("MM_RESEND_INVITATION_EMAIL_JOB_DURATION")
+	DurationInMillis, parseError := strconv.ParseInt(duration, 10, 64)
+	if parseError != nil {
+		// default to 48 hours
+		DurationInMillis = FourtyEightHoursInMillis
+	}
+
+	return elapsedTimeSinceSchedule, DurationInMillis
+}
+
+func (rseworker *ResendInvitationEmailWorker) TearDown(logger mlog.LoggerIFace, job *model.Job) {
+	if _, err := rseworker.store.System().PermanentDeleteByName(job.Id); err != nil {
+		logger.Error("Worker: Failed to tear down data", mlog.Err(err))
+	}
+
+	rseworker.setJobSuccess(logger, job)
+}
+
+func (rseworker *ResendInvitationEmailWorker) ResendEmails(logger mlog.LoggerIFace, job *model.Job, interval string) *model.AppError {
+	rctx := request.EmptyContext(logger)
+
+	teamID := job.Data["teamID"]
+	emailListData := job.Data["emailList"]
+
+	emailList, err := rseworker.cleanEmailData(emailListData)
+	if err != nil {
+		appErr := model.NewAppError("worker: "+rseworker.name, "job_id: "+job.Id, nil, "", http.StatusInternalServerError).Wrap(err)
+		logger.Error("Worker: Failed to clean emails string data", mlog.Err(appErr))
+		return appErr
+	}
+
+	var channelList []string
+	if channelListData := job.Data["channelList"]; channelListData != "" {
+		channelList, err = rseworker.cleanChannelsData(channelListData)
+		if err != nil {
+			appErr := model.NewAppError("worker: "+rseworker.name, "job_id: "+job.Id, nil, "", http.StatusInternalServerError).Wrap(err)
+			logger.Error("Worker: Failed to clean channel string data", mlog.Err(appErr))
+			return appErr
+		}
+	}
+
+	emailList = rseworker.removeAlreadyJoined(teamID, emailList)
+	if len(emailList) == 0 {
+		return nil
+	}
+
+	memberInvite := model.MemberInvite{
+		Emails: emailList,
+	}
+
+	if len(channelList) > 0 {
+		memberInvite.ChannelIds = channelList
+	}
+
+	if profileListData := job.Data["profilesList"]; profileListData != "" {
+		var profiles []*model.MemberInviteProfile
+		if err := json.Unmarshal([]byte(profileListData), &profiles); err != nil {
+			appErr := model.NewAppError("worker: "+rseworker.name, "job_id: "+job.Id, nil, "", http.StatusInternalServerError).Wrap(err)
+			logger.Error("Worker: Failed to clean profiles string data", mlog.Err(appErr))
+			return appErr
+		}
+
+		remainingEmails := make(map[string]struct{}, len(emailList))
+		for _, email := range emailList {
+			remainingEmails[model.NormalizeEmail(email)] = struct{}{}
+		}
+		for _, profile := range profiles {
+			if profile == nil {
+				memberInvite.Profiles = append(memberInvite.Profiles, profile)
+				continue
+			}
+			if _, ok := remainingEmails[model.NormalizeEmail(profile.Email)]; ok {
+				memberInvite.Profiles = append(memberInvite.Profiles, profile)
+			}
+		}
+	}
+
+	invitesWithErrors, appErr := rseworker.app.InviteNewUsersToTeamGracefully(rctx, &memberInvite, teamID, job.Data["senderID"], interval)
+	if appErr != nil {
+		logger.Error("Worker: Failed to send emails", mlog.Err(appErr))
+		return appErr
+	}
+	for _, invite := range invitesWithErrors {
+		if invite != nil && invite.Error != nil {
+			logger.Error("Worker: Failed to resend invitation", mlog.String("email", invite.Email), mlog.Err(invite.Error))
+			return invite.Error
+		}
+	}
+	return nil
+}

server/channels/jobs/resend_invitation_email/worker_test.go

@@ -0,0 +1,95 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package resend_invitation_email
+
+import (
+	"encoding/json"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+)
+
+type fakeResendApp struct {
+	invitedWith *model.MemberInvite
+	joinedEmail string
+}
+
+func (a *fakeResendApp) Config() *model.Config                                 { return &model.Config{} }
+func (a *fakeResendApp) AddConfigListener(func(old, cur *model.Config)) string { return "" }
+func (a *fakeResendApp) RemoveConfigListener(string)                           {}
+func (a *fakeResendApp) GetUserByEmail(email string) (*model.User, *model.AppError) {
+	if email == a.joinedEmail {
+		return &model.User{Id: email}, nil
+	}
+	return nil, model.NewAppError("GetUserByEmail", "app.user.missing_account.const", nil, "", http.StatusNotFound)
+}
+func (a *fakeResendApp) GetTeamMembersByIds(teamID string, userIDs []string, restrictions *model.ViewUsersRestrictions) ([]*model.TeamMember, *model.AppError) {
+	if len(userIDs) == 1 && userIDs[0] == a.joinedEmail {
+		return []*model.TeamMember{{UserId: a.joinedEmail}}, nil
+	}
+	return nil, nil
+}
+func (a *fakeResendApp) InviteNewUsersToTeamGracefully(rctx request.CTX, memberInvite *model.MemberInvite, teamID, senderID, reminderInterval string) ([]*model.EmailInviteWithError, *model.AppError) {
+	a.invitedWith = memberInvite
+	return nil, nil
+}
+
+func newTestWorker(t *testing.T, app AppIface) *ResendInvitationEmailWorker {
+	t.Helper()
+	return &ResendInvitationEmailWorker{
+		name:   "ResendInvitationEmail",
+		logger: mlog.CreateConsoleTestLogger(t),
+		app:    app,
+	}
+}
+
+func TestResendEmailsCarriesProfilesForPendingUsers(t *testing.T) {
+	profiles := []*model.MemberInviteProfile{
+		{Email: "joined@example.com", Username: "joined.user"},
+		{Email: "waiting@example.com", Username: "waiting.user", FirstName: "Waiting", LastName: "User"},
+	}
+	profilesJSON, err := json.Marshal(profiles)
+	require.NoError(t, err)
+
+	app := &fakeResendApp{joinedEmail: "joined@example.com"}
+	worker := newTestWorker(t, app)
+	appErr := worker.ResendEmails(worker.logger, &model.Job{
+		Id: model.NewId(),
+		Data: map[string]string{
+			"emailList":    model.ArrayToJSON([]string{"joined@example.com", "waiting@example.com"}),
+			"teamID":       model.NewId(),
+			"senderID":     model.NewId(),
+			"profilesList": string(profilesJSON),
+		},
+	}, "48")
+
+	require.Nil(t, appErr)
+	require.Equal(t, []string{"waiting@example.com"}, app.invitedWith.Emails)
+	require.Equal(t, profiles[1:], app.invitedWith.Profiles)
+}
+
+func TestResendEmailsRejectsMalformedJobData(t *testing.T) {
+	for _, key := range []string{"emailList", "channelList", "profilesList"} {
+		t.Run(key, func(t *testing.T) {
+			app := &fakeResendApp{}
+			worker := newTestWorker(t, app)
+			data := map[string]string{
+				"emailList": model.ArrayToJSON([]string{"user@example.com"}),
+				"teamID":    model.NewId(),
+				"senderID":  model.NewId(),
+				key:         "not-json",
+			}
+
+			appErr := worker.ResendEmails(worker.logger, &model.Job{Id: model.NewId(), Data: data}, "48")
+
+			require.NotNil(t, appErr)
+			require.Nil(t, app.invitedWith)
+		})
+	}
+}

server/channels/jobs/s3_path_migration/s3_path_migration.go

@@ -0,0 +1,255 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package s3_path_migration
+
+import (
+	"errors"
+	"net/http"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+)
+
+const (
+	timeBetweenBatches = 1 * time.Second
+)
+
+type S3PathMigrationWorker struct {
+	name        string
+	jobServer   *jobs.JobServer
+	logger      mlog.LoggerIFace
+	store       store.Store
+	fileBackend *filestore.S3FileBackend
+
+	stop    chan struct{}
+	stopped chan bool
+	jobs    chan model.Job
+}
+
+func MakeWorker(jobServer *jobs.JobServer, store store.Store, fileBackend filestore.FileBackend) *S3PathMigrationWorker {
+	// If the type cast fails, it will be nil
+	// which is checked later.
+	s3Backend, _ := fileBackend.(*filestore.S3FileBackend)
+	const workerName = "S3PathMigration"
+	worker := &S3PathMigrationWorker{
+		name:        workerName,
+		jobServer:   jobServer,
+		logger:      jobServer.Logger().With(mlog.String("worker_name", workerName)),
+		store:       store,
+		fileBackend: s3Backend,
+		stop:        make(chan struct{}),
+		stopped:     make(chan bool, 1),
+		jobs:        make(chan model.Job),
+	}
+	return worker
+}
+
+func (worker *S3PathMigrationWorker) Run() {
+	worker.logger.Debug("Worker started")
+	// We have to re-assign the stop channel again, because
+	// it might happen that the job was restarted due to a config change.
+	worker.stop = make(chan struct{}, 1)
+
+	defer func() {
+		worker.logger.Debug("Worker finished")
+		worker.stopped <- true
+	}()
+
+	for {
+		select {
+		case <-worker.stop:
+			worker.logger.Debug("Worker received stop signal")
+			return
+		case job := <-worker.jobs:
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *S3PathMigrationWorker) Stop() {
+	worker.logger.Debug("Worker stopping")
+	close(worker.stop)
+	<-worker.stopped
+}
+
+func (worker *S3PathMigrationWorker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+func (worker *S3PathMigrationWorker) IsEnabled(_ *model.Config) bool {
+	return os.Getenv("MM_CLOUD_FILESTORE_BIFROST") != ""
+}
+
+func (worker *S3PathMigrationWorker) getJobMetadata(job *model.Job, key string) (int, *model.AppError) {
+	countStr := job.Data[key]
+	count := 0
+	var err error
+	if countStr != "" {
+		count, err = strconv.Atoi(countStr)
+		if err != nil {
+			return 0, model.NewAppError("getJobMetadata", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+	}
+	return count, nil
+}
+
+func (worker *S3PathMigrationWorker) DoJob(job *model.Job) {
+	logger := worker.logger.With(jobs.JobLoggerFields(job)...)
+	logger.Debug("Worker: Received a new candidate job.")
+	defer worker.jobServer.HandleJobPanic(logger, job)
+
+	var appErr *model.AppError
+	job, appErr = worker.jobServer.ClaimJob(job)
+	if appErr != nil {
+		logger.Warn("S3PathMigrationWorker experienced an error while trying to claim job", mlog.Err(appErr))
+		return
+	} else if job == nil {
+		return
+	}
+
+	if worker.fileBackend == nil {
+		err := errors.New("no S3 file backend found")
+		logger.Error("S3PathMigrationWorker: ", mlog.Err(err))
+		worker.setJobError(logger, job, model.NewAppError("DoJob", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err))
+		return
+	}
+
+	// Check if there is metadata for that job.
+	// If there isn't, it will be empty by default, which is the right value.
+	startFileID := job.Data["start_file_id"]
+
+	doneCount, appErr := worker.getJobMetadata(job, "done_file_count")
+	if appErr != nil {
+		logger.Error("S3PathMigrationWorker: failed to get done file count", mlog.Err(appErr))
+		worker.setJobError(logger, job, appErr)
+		return
+	}
+
+	startTime, appErr := worker.getJobMetadata(job, "start_create_at")
+	if appErr != nil {
+		logger.Error("S3PathMigrationWorker: failed to get start create_at", mlog.Err(appErr))
+		worker.setJobError(logger, job, appErr)
+		return
+	}
+	if startTime == 0 {
+		// Time of the commit because we know no files older than that are affected.
+		// Exact commit was done on 09:54AM June 27, IST.
+		// We take June 26 as an approximation.
+		startTime = int(time.Date(2023, time.June, 26, 0, 0, 0, 0, time.UTC).UnixMilli())
+	}
+
+	const pageSize = 100
+
+	for {
+		select {
+		case <-worker.stop:
+			logger.Info("Worker: S3 Migration has been canceled via Worker Stop. Setting the job back to pending.")
+			if err := worker.jobServer.SetJobPending(job); err != nil {
+				worker.logger.Error("Worker: Failed to mark job as pending", mlog.Err(err))
+			}
+			return
+		case <-time.After(timeBetweenBatches):
+			var files []*model.FileForIndexing
+			tries := 0
+			for files == nil {
+				var err error
+				// Take batches of `pageSize`
+				files, err = worker.store.FileInfo().GetFilesBatchForIndexing(int64(startTime), startFileID, true, pageSize)
+				if err != nil {
+					if tries > 3 {
+						logger.Error("Worker: Failed to get files after multiple retries. Exiting")
+						worker.setJobError(logger, job, model.NewAppError("DoJob", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err))
+						return
+					}
+					logger.Warn("Failed to get file info for s3 migration. Retrying .. ", mlog.Err(err))
+
+					// Wait a bit before trying again.
+					time.Sleep(15 * time.Second)
+				}
+
+				tries++
+			}
+
+			if len(files) == 0 {
+				logger.Info("S3PathMigrationWorker: Job is complete")
+				worker.setJobSuccess(logger, job)
+				worker.markAsComplete(logger, job)
+				return
+			}
+
+			// Iterate through the rows in each page.
+			for _, f := range files {
+				logger.Debug("Processing file ID", mlog.String("id", f.Id))
+				// We do not fail the job if a single image failed to encode.
+				if f.Path != "" {
+					if err := worker.fileBackend.DecodeFilePathIfNeeded(f.Path); err != nil {
+						logger.Warn("Failed to encode S3 file path", mlog.String("path", f.Path), mlog.String("id", f.Id), mlog.Err(err))
+					}
+				}
+				if f.PreviewPath != "" {
+					if err := worker.fileBackend.DecodeFilePathIfNeeded(f.PreviewPath); err != nil {
+						logger.Warn("Failed to encode S3 file path", mlog.String("path", f.PreviewPath), mlog.String("id", f.Id), mlog.Err(err))
+					}
+				}
+				if f.ThumbnailPath != "" {
+					if err := worker.fileBackend.DecodeFilePathIfNeeded(f.ThumbnailPath); err != nil {
+						logger.Warn("Failed to encode S3 file path", mlog.String("path", f.ThumbnailPath), mlog.String("id", f.Id), mlog.Err(err))
+					}
+				}
+			}
+
+			// Work on each batch and save the batch starting ID in metadata
+			lastFile := files[len(files)-1]
+			startFileID = lastFile.Id
+			startTime = int(lastFile.CreateAt)
+
+			if job.Data == nil {
+				job.Data = make(model.StringMap)
+			}
+			job.Data["start_file_id"] = startFileID
+			job.Data["start_create_at"] = strconv.Itoa(startTime)
+			doneCount += len(files)
+			job.Data["done_file_count"] = strconv.Itoa(doneCount)
+		}
+	}
+}
+
+func (worker *S3PathMigrationWorker) markAsComplete(logger mlog.LoggerIFace, job *model.Job) {
+	system := model.System{
+		Name:  model.MigrationKeyS3Path,
+		Value: "true",
+	}
+
+	// Note that if this fails, then the job would have still succeeded.
+	// So it will try to run the same job again next time, but then
+	// it will just fall through everything because all files would have
+	// converted. The actual job is idempotent, so there won't be a problem.
+	if err := worker.jobServer.Store.System().Save(&system); err != nil {
+		logger.Error("Worker: Failed to mark s3 path migration as completed in the systems table.", mlog.Err(err))
+	}
+}
+
+func (worker *S3PathMigrationWorker) setJobSuccess(logger mlog.LoggerIFace, job *model.Job) {
+	if err := worker.jobServer.SetJobProgress(job, 100); err != nil {
+		logger.Error("Worker: Failed to update progress for job", mlog.Err(err))
+		worker.setJobError(logger, job, err)
+	}
+
+	if err := worker.jobServer.SetJobSuccess(job); err != nil {
+		logger.Error("S3PathMigrationWorker: Failed to set success for job", mlog.Err(err))
+		worker.setJobError(logger, job, err)
+	}
+}
+
+func (worker *S3PathMigrationWorker) setJobError(logger mlog.LoggerIFace, job *model.Job, appError *model.AppError) {
+	if err := worker.jobServer.SetJobError(job, appError); err != nil {
+		logger.Error("S3PathMigrationWorker: Failed to set job error", mlog.Err(err))
+	}
+}

server/channels/jobs/scheduled_recap/scheduler.go

@@ -0,0 +1,83 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package scheduled_recap
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+// SchedulerPollingInterval defines how often the scheduler polls for due scheduled recaps.
+const SchedulerPollingInterval = 1 * time.Minute
+
+// Scheduler polls for due scheduled recaps and creates jobs for them.
+type Scheduler struct {
+	*jobs.PeriodicScheduler
+	store     store.Store
+	jobServer *jobs.JobServer
+}
+
+// MakeScheduler creates a new scheduler for scheduled recaps.
+func MakeScheduler(jobServer *jobs.JobServer, storeInstance store.Store) *Scheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return cfg.AIRecapsEnabled()
+	}
+	return &Scheduler{
+		PeriodicScheduler: jobs.NewPeriodicScheduler(
+			jobServer,
+			model.JobTypeScheduledRecap,
+			SchedulerPollingInterval,
+			isEnabled,
+		),
+		store:     storeInstance,
+		jobServer: jobServer,
+	}
+}
+
+// NextScheduleTime overrides to use tight polling interval.
+func (s *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastJob *model.Job) *time.Time {
+	next := now.Add(SchedulerPollingInterval)
+	return &next
+}
+
+// ScheduleJob polls for due scheduled recaps and creates jobs for each.
+func (s *Scheduler) ScheduleJob(rctx request.CTX, cfg *model.Config, pendingJobs bool, lastJob *model.Job) (*model.Job, *model.AppError) {
+	now := model.GetMillis()
+	dueRecaps, err := s.store.ScheduledRecap().GetDueBefore(now, 100)
+	if err != nil {
+		mlog.Error("Failed to get due scheduled recaps", mlog.Err(err))
+		return nil, nil
+	}
+
+	for _, sr := range dueRecaps {
+		// The worker re-fetches the full row by ID, so the job only needs the ID.
+		jobData := model.StringMap{
+			"scheduled_recap_id": sr.Id,
+		}
+
+		job, jobErr := s.jobServer.CreateJobOnceByTypeAndData(
+			rctx,
+			model.JobTypeScheduledRecap,
+			jobData,
+			map[string]string{"scheduled_recap_id": sr.Id},
+		)
+		if jobErr != nil {
+			mlog.Warn("Scheduled recap job creation failed",
+				mlog.String("scheduled_recap_id", sr.Id),
+				mlog.Err(jobErr))
+			continue
+		}
+		if job == nil {
+			mlog.Debug("Scheduled recap job already queued",
+				mlog.String("scheduled_recap_id", sr.Id))
+		}
+	}
+
+	return nil, nil
+}

server/channels/jobs/scheduled_recap/scheduler_test.go

@@ -0,0 +1,69 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package scheduled_recap
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
+	"github.com/mattermost/mattermost/server/v8/channels/utils/testutils"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestScheduleJobEnqueuesEachDueRecapAndSkipsDuplicateAtomically(t *testing.T) {
+	cfg := &model.Config{}
+	cfg.SetDefaults()
+	cfg.FeatureFlags.EnableAIRecaps = true
+
+	mockStore := &storetest.Store{}
+	t.Cleanup(func() {
+		mockStore.AssertExpectations(t)
+	})
+
+	jobServer := jobs.NewJobServer(&testutils.StaticConfigService{Cfg: cfg}, mockStore, nil, mlog.CreateConsoleTestLogger(t), nil)
+	jobServer.RegisterJobType(model.JobTypeScheduledRecap, jobs.NewSimpleWorker(
+		model.JobTypeScheduledRecap,
+		jobServer,
+		func(logger mlog.LoggerIFace, job *model.Job) error { return nil },
+		func(cfg *model.Config) bool { return true },
+	), nil)
+
+	dueRecap1 := testScheduledRecap(true)
+	dueRecap2 := testScheduledRecap(true)
+	duplicateRecap := *dueRecap1
+	dueRecaps := []*model.ScheduledRecap{dueRecap1, dueRecap2, &duplicateRecap}
+
+	mockStore.ScheduledRecapStore.
+		On("GetDueBefore", mock.AnythingOfType("int64"), 100).
+		Return(dueRecaps, nil)
+
+	for _, sr := range []*model.ScheduledRecap{dueRecap1, dueRecap2} {
+		mockStore.JobStore.
+			On("SaveOnceByTypeAndData", mock.MatchedBy(func(job *model.Job) bool {
+				return job.Type == model.JobTypeScheduledRecap &&
+					len(job.Data) == 1 &&
+					job.Data["scheduled_recap_id"] == sr.Id
+			}), map[string]string{"scheduled_recap_id": sr.Id}).
+			Return(func(job *model.Job, data map[string]string) *model.Job { return job }, nil).
+			Once()
+	}
+
+	mockStore.JobStore.
+		On("SaveOnceByTypeAndData", mock.MatchedBy(func(job *model.Job) bool {
+			return job.Type == model.JobTypeScheduledRecap &&
+				job.Data["scheduled_recap_id"] == dueRecap1.Id
+		}), map[string]string{"scheduled_recap_id": dueRecap1.Id}).
+		Return(nil, nil).
+		Once()
+
+	scheduler := MakeScheduler(jobServer, mockStore)
+	job, appErr := scheduler.ScheduleJob(request.EmptyContext(mlog.CreateConsoleTestLogger(t)), cfg, false, nil)
+	require.Nil(t, appErr)
+	require.Nil(t, job)
+}

server/channels/jobs/scheduled_recap/worker.go

@@ -0,0 +1,138 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package scheduled_recap
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+// AppIface defines the app methods required by the scheduled recap worker.
+// This interface will be implemented by the App layer in 03-02.
+type AppIface interface {
+	CreateRecapFromSchedule(rctx request.CTX, scheduledRecap *model.ScheduledRecap) (*model.Recap, *model.AppError)
+}
+
+// MakeWorker creates a new worker for processing scheduled recap jobs.
+func MakeWorker(jobServer *jobs.JobServer, storeInstance store.Store, app AppIface) *jobs.SimpleWorker {
+	const workerName = "ScheduledRecap"
+
+	isEnabled := func(cfg *model.Config) bool {
+		return cfg.AIRecapsEnabled()
+	}
+
+	execute := func(logger mlog.LoggerIFace, job *model.Job) error {
+		defer jobServer.HandleJobPanic(logger, job)
+		return processScheduledRecapJob(logger, job, storeInstance, app)
+	}
+
+	return jobs.NewSimpleWorker(workerName, jobServer, execute, isEnabled)
+}
+
+func processScheduledRecapJob(logger mlog.LoggerIFace, job *model.Job, storeInstance store.Store, app AppIface) error {
+	scheduledRecapID := job.Data["scheduled_recap_id"]
+
+	// Get the scheduled recap
+	sr, err := storeInstance.ScheduledRecap().Get(scheduledRecapID)
+	if err != nil {
+		return fmt.Errorf("scheduled recap not found: %w", err)
+	}
+
+	// Verify still enabled
+	if !sr.Enabled {
+		logger.Info("Scheduled recap is disabled, skipping",
+			mlog.String("scheduled_recap_id", scheduledRecapID))
+		return nil
+	}
+
+	// CreateRecapFromSchedule performs the atomic daily-limit check; an over-limit
+	// run surfaces as the max_recaps_reached error, which we treat as a skip.
+	rctx := request.EmptyContext(logger)
+	_, appErr := app.CreateRecapFromSchedule(rctx, sr)
+	if appErr != nil {
+		if appErr.Id == "app.recap.max_recaps_reached.app_error" {
+			if saveErr := saveSkippedRecap(storeInstance, sr); saveErr != nil {
+				logger.Error("Failed to save skipped recap", mlog.Err(saveErr))
+				return fmt.Errorf("failed to save skipped recap: %w", saveErr)
+			}
+			logger.Info("Scheduled recap skipped due to daily limit",
+				mlog.String("scheduled_recap_id", scheduledRecapID),
+				mlog.String("user_id", sr.UserId))
+			return finalizeSchedule(logger, storeInstance, sr)
+		}
+		return fmt.Errorf("failed to create recap from schedule: %w", appErr)
+	}
+
+	logger.Info("Scheduled recap executed successfully",
+		mlog.String("scheduled_recap_id", scheduledRecapID))
+
+	return finalizeSchedule(logger, storeInstance, sr)
+}
+
+// finalizeSchedule advances the schedule to its next run and, for non-recurring
+// recaps, disables it last so a one-shot recap is never left enabled.
+func finalizeSchedule(logger mlog.LoggerIFace, storeInstance store.Store, sr *model.ScheduledRecap) error {
+	if err := advanceSchedule(logger, storeInstance, sr); err != nil {
+		return err
+	}
+
+	if !sr.IsRecurring {
+		logger.Info("Disabling non-recurring scheduled recap",
+			mlog.String("scheduled_recap_id", sr.Id))
+		if setErr := storeInstance.ScheduledRecap().SetEnabled(sr.Id, false); setErr != nil {
+			return fmt.Errorf("failed to disable non-recurring scheduled recap: %w", setErr)
+		}
+	}
+
+	return nil
+}
+
+// advanceSchedule computes the next run time and marks the scheduled recap as executed.
+// If the next run time can't be computed, the recap is disabled.
+func advanceSchedule(logger mlog.LoggerIFace, storeInstance store.Store, sr *model.ScheduledRecap) error {
+	nextRunAt, computeErr := sr.ComputeNextRunAt(time.Now())
+	if computeErr != nil {
+		logger.Error("Failed to compute next run time",
+			mlog.String("scheduled_recap_id", sr.Id),
+			mlog.Err(computeErr))
+		if setErr := storeInstance.ScheduledRecap().SetEnabled(sr.Id, false); setErr != nil {
+			return fmt.Errorf("failed to disable scheduled recap after next-run computation failure: %w", setErr)
+		}
+		return fmt.Errorf("failed to compute next run time: %w", computeErr)
+	}
+
+	if markErr := storeInstance.ScheduledRecap().MarkExecuted(sr.Id, model.GetMillis(), nextRunAt); markErr != nil {
+		logger.Error("Failed to mark as executed",
+			mlog.String("scheduled_recap_id", sr.Id),
+			mlog.Err(markErr))
+		return fmt.Errorf("failed to mark scheduled recap as executed: %w", markErr)
+	}
+
+	return nil
+}
+
+func saveSkippedRecap(storeInstance store.Store, sr *model.ScheduledRecap) error {
+	skippedRecap := &model.Recap{
+		Id:               model.NewId(),
+		UserId:           sr.UserId,
+		Title:            sr.Title,
+		Status:           model.RecapStatusSkipped,
+		SkipReason:       model.SkipReasonDailyLimit,
+		ScheduledRecapId: sr.Id,
+		CreateAt:         model.GetMillis(),
+		UpdateAt:         model.GetMillis(),
+	}
+
+	if _, saveErr := storeInstance.Recap().SaveRecap(skippedRecap); saveErr != nil {
+		return saveErr
+	}
+
+	return nil
+}

server/channels/jobs/scheduled_recap/worker_test.go

@@ -0,0 +1,148 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package scheduled_recap
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+type mockScheduledRecapApp struct {
+	mock.Mock
+}
+
+func (m *mockScheduledRecapApp) CreateRecapFromSchedule(rctx request.CTX, scheduledRecap *model.ScheduledRecap) (*model.Recap, *model.AppError) {
+	args := m.Called(rctx, scheduledRecap)
+	if args.Get(0) == nil {
+		return nil, args.Get(1).(*model.AppError)
+	}
+	return args.Get(0).(*model.Recap), nil
+}
+
+func TestProcessScheduledRecapJobReturnsPersistenceErrors(t *testing.T) {
+	logger := mlog.CreateConsoleTestLogger(t)
+
+	t.Run("mark executed failure", func(t *testing.T) {
+		scheduledRecap := testScheduledRecap(true)
+		job := &model.Job{Data: map[string]string{"scheduled_recap_id": scheduledRecap.Id}}
+
+		mockStore := &mocks.Store{}
+		mockScheduledStore := &mocks.ScheduledRecapStore{}
+		mockStore.On("ScheduledRecap").Return(mockScheduledStore)
+		mockScheduledStore.On("Get", scheduledRecap.Id).Return(scheduledRecap, nil)
+		mockScheduledStore.On("MarkExecuted", scheduledRecap.Id, mock.AnythingOfType("int64"), mock.AnythingOfType("int64")).Return(errors.New("mark failed"))
+
+		mockApp := &mockScheduledRecapApp{}
+		mockApp.On("CreateRecapFromSchedule", mock.Anything, scheduledRecap).Return(&model.Recap{Id: model.NewId()}, nil)
+
+		err := processScheduledRecapJob(logger, job, mockStore, mockApp)
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "failed to mark scheduled recap as executed")
+		mockScheduledStore.AssertExpectations(t)
+		mockApp.AssertExpectations(t)
+	})
+
+	t.Run("non recurring disable failure", func(t *testing.T) {
+		scheduledRecap := testScheduledRecap(false)
+		job := &model.Job{Data: map[string]string{"scheduled_recap_id": scheduledRecap.Id}}
+
+		mockStore := &mocks.Store{}
+		mockScheduledStore := &mocks.ScheduledRecapStore{}
+		mockStore.On("ScheduledRecap").Return(mockScheduledStore)
+		mockScheduledStore.On("Get", scheduledRecap.Id).Return(scheduledRecap, nil)
+		mockScheduledStore.On("MarkExecuted", scheduledRecap.Id, mock.AnythingOfType("int64"), mock.AnythingOfType("int64")).Return(nil)
+		mockScheduledStore.On("SetEnabled", scheduledRecap.Id, false).Return(errors.New("disable failed"))
+
+		mockApp := &mockScheduledRecapApp{}
+		mockApp.On("CreateRecapFromSchedule", mock.Anything, scheduledRecap).Return(&model.Recap{Id: model.NewId()}, nil)
+
+		err := processScheduledRecapJob(logger, job, mockStore, mockApp)
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "failed to disable non-recurring scheduled recap")
+		mockScheduledStore.AssertExpectations(t)
+		mockApp.AssertExpectations(t)
+	})
+}
+
+func TestProcessScheduledRecapJobDailyLimitSkip(t *testing.T) {
+	logger := mlog.CreateConsoleTestLogger(t)
+	limitErr := model.NewAppError("CreateRecapFromSchedule", "app.recap.max_recaps_reached.app_error", nil, "", 429)
+
+	t.Run("recurring schedule advances but stays enabled", func(t *testing.T) {
+		scheduledRecap := testScheduledRecap(true)
+		job := &model.Job{Data: map[string]string{"scheduled_recap_id": scheduledRecap.Id}}
+
+		mockStore := &mocks.Store{}
+		mockScheduledStore := &mocks.ScheduledRecapStore{}
+		mockRecapStore := &mocks.RecapStore{}
+		mockStore.On("ScheduledRecap").Return(mockScheduledStore)
+		mockStore.On("Recap").Return(mockRecapStore)
+		mockScheduledStore.On("Get", scheduledRecap.Id).Return(scheduledRecap, nil)
+		mockScheduledStore.On("MarkExecuted", scheduledRecap.Id, mock.AnythingOfType("int64"), mock.AnythingOfType("int64")).Return(nil)
+		mockRecapStore.On("SaveRecap", mock.MatchedBy(func(r *model.Recap) bool {
+			return r.Status == model.RecapStatusSkipped && r.ScheduledRecapId == scheduledRecap.Id
+		})).Return(&model.Recap{}, nil)
+
+		mockApp := &mockScheduledRecapApp{}
+		mockApp.On("CreateRecapFromSchedule", mock.Anything, scheduledRecap).Return(nil, limitErr)
+
+		err := processScheduledRecapJob(logger, job, mockStore, mockApp)
+		require.NoError(t, err)
+		mockScheduledStore.AssertExpectations(t)
+		mockRecapStore.AssertExpectations(t)
+		mockApp.AssertExpectations(t)
+		// A recurring schedule must not be disabled on the skip path.
+		mockScheduledStore.AssertNotCalled(t, "SetEnabled", mock.Anything, mock.Anything)
+	})
+
+	t.Run("non recurring schedule is disabled on skip", func(t *testing.T) {
+		scheduledRecap := testScheduledRecap(false)
+		job := &model.Job{Data: map[string]string{"scheduled_recap_id": scheduledRecap.Id}}
+
+		mockStore := &mocks.Store{}
+		mockScheduledStore := &mocks.ScheduledRecapStore{}
+		mockRecapStore := &mocks.RecapStore{}
+		mockStore.On("ScheduledRecap").Return(mockScheduledStore)
+		mockStore.On("Recap").Return(mockRecapStore)
+		mockScheduledStore.On("Get", scheduledRecap.Id).Return(scheduledRecap, nil)
+		mockScheduledStore.On("MarkExecuted", scheduledRecap.Id, mock.AnythingOfType("int64"), mock.AnythingOfType("int64")).Return(nil)
+		mockScheduledStore.On("SetEnabled", scheduledRecap.Id, false).Return(nil)
+		mockRecapStore.On("SaveRecap", mock.MatchedBy(func(r *model.Recap) bool {
+			return r.Status == model.RecapStatusSkipped && r.ScheduledRecapId == scheduledRecap.Id
+		})).Return(&model.Recap{}, nil)
+
+		mockApp := &mockScheduledRecapApp{}
+		mockApp.On("CreateRecapFromSchedule", mock.Anything, scheduledRecap).Return(nil, limitErr)
+
+		err := processScheduledRecapJob(logger, job, mockStore, mockApp)
+		require.NoError(t, err)
+		mockScheduledStore.AssertExpectations(t)
+		mockRecapStore.AssertExpectations(t)
+		mockApp.AssertExpectations(t)
+	})
+}
+
+func testScheduledRecap(isRecurring bool) *model.ScheduledRecap {
+	return &model.ScheduledRecap{
+		Id:          model.NewId(),
+		UserId:      model.NewId(),
+		Title:       "Scheduled Recap",
+		DaysOfWeek:  model.EveryDay,
+		TimeOfDay:   "09:00",
+		Timezone:    "America/New_York",
+		TimePeriod:  model.TimePeriodLast24h,
+		ChannelMode: model.ChannelModeSpecific,
+		ChannelIds:  []string{model.NewId()},
+		AgentId:     "test-agent",
+		IsRecurring: isRecurring,
+		Enabled:     true,
+	}
+}

server/channels/jobs/schedulers.go

@@ -0,0 +1,196 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+)
+
+type Scheduler interface {
+	Enabled(cfg *model.Config) bool
+	NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time
+	ScheduleJob(rctx request.CTX, cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError)
+}
+
+type Schedulers struct {
+	stop                 chan bool
+	stopped              chan bool
+	configChanged        chan *model.Config
+	clusterLeaderChanged chan bool
+	listenerId           string
+	jobs                 *JobServer
+	isLeader             bool
+	running              bool
+
+	schedulers   map[string]Scheduler
+	nextRunTimes map[string]*time.Time
+}
+
+var (
+	ErrSchedulersNotRunning    = errors.New("job schedulers are not running")
+	ErrSchedulersRunning       = errors.New("job schedulers are running")
+	ErrSchedulersUninitialized = errors.New("job schedulers are not initialized")
+)
+
+func (schedulers *Schedulers) AddScheduler(name string, scheduler Scheduler) {
+	schedulers.schedulers[name] = scheduler
+}
+
+// Start starts the schedulers. This call is not safe for concurrent use.
+// Synchronization should be implemented by the caller.
+func (schedulers *Schedulers) Start() {
+	schedulers.stop = make(chan bool)
+	schedulers.stopped = make(chan bool)
+	schedulers.listenerId = schedulers.jobs.ConfigService.AddConfigListener(schedulers.handleConfigChange)
+
+	go func() {
+		mlog.Info("Starting schedulers.")
+
+		defer func() {
+			mlog.Info("Schedulers stopped.")
+			close(schedulers.stopped)
+		}()
+
+		now := time.Now()
+		for name, scheduler := range schedulers.schedulers {
+			if !scheduler.Enabled(schedulers.jobs.Config()) {
+				schedulers.nextRunTimes[name] = nil
+			} else {
+				schedulers.setNextRunTime(schedulers.jobs.Config(), name, now, false)
+			}
+		}
+
+		for {
+			timer := time.NewTimer(1 * time.Minute)
+			select {
+			case <-schedulers.stop:
+				mlog.Debug("Schedulers received stop signal.")
+				timer.Stop()
+				return
+			case now = <-timer.C:
+				cfg := schedulers.jobs.Config()
+
+				for name, nextTime := range schedulers.nextRunTimes {
+					if nextTime == nil {
+						continue
+					}
+
+					if time.Now().After(*nextTime) {
+						scheduler := schedulers.schedulers[name]
+						if scheduler == nil || !schedulers.isLeader || !scheduler.Enabled(cfg) {
+							continue
+						}
+						rctx := request.EmptyContext(schedulers.jobs.Logger())
+						if _, err := schedulers.scheduleJob(rctx, cfg, name, scheduler); err != nil {
+							mlog.Error("Failed to schedule job", mlog.String("scheduler", name), mlog.Err(err))
+							continue
+						}
+						schedulers.setNextRunTime(cfg, name, now, true)
+					}
+				}
+			case newCfg := <-schedulers.configChanged:
+				for name, scheduler := range schedulers.schedulers {
+					if !schedulers.isLeader || !scheduler.Enabled(newCfg) {
+						schedulers.nextRunTimes[name] = nil
+					} else {
+						schedulers.setNextRunTime(newCfg, name, now, false)
+					}
+				}
+			case isLeader := <-schedulers.clusterLeaderChanged:
+				for name := range schedulers.schedulers {
+					schedulers.isLeader = isLeader
+					if !isLeader {
+						schedulers.nextRunTimes[name] = nil
+					} else {
+						schedulers.setNextRunTime(schedulers.jobs.Config(), name, now, false)
+					}
+				}
+			}
+			timer.Stop()
+		}
+	}()
+
+	schedulers.running = true
+}
+
+// Stop stops the schedulers. This call is not safe for concurrent use.
+// Synchronization should be implemented by the caller.
+func (schedulers *Schedulers) Stop() {
+	mlog.Info("Stopping schedulers.")
+	close(schedulers.stop)
+	<-schedulers.stopped
+	schedulers.jobs.ConfigService.RemoveConfigListener(schedulers.listenerId)
+	schedulers.listenerId = ""
+	schedulers.running = false
+}
+
+func (schedulers *Schedulers) setNextRunTime(cfg *model.Config, name string, now time.Time, pendingJobs bool) {
+	scheduler := schedulers.schedulers[name]
+
+	if !pendingJobs {
+		pj, err := schedulers.jobs.CheckForPendingJobsByType(name)
+		if err != nil {
+			mlog.Error("Failed to set next job run time", mlog.Err(err))
+			schedulers.nextRunTimes[name] = nil
+			return
+		}
+		pendingJobs = pj
+	}
+
+	lastSuccessfulJob, err := schedulers.jobs.GetLastSuccessfulJobByType(name)
+	if err != nil {
+		mlog.Error("Failed to set next job run time", mlog.Err(err))
+		schedulers.nextRunTimes[name] = nil
+		return
+	}
+
+	schedulers.nextRunTimes[name] = scheduler.NextScheduleTime(cfg, now, pendingJobs, lastSuccessfulJob)
+	mlog.Debug("Next run time for scheduler", mlog.String("scheduler_name", name), mlog.String("next_runtime", fmt.Sprintf("%v", schedulers.nextRunTimes[name])))
+}
+
+func (schedulers *Schedulers) scheduleJob(rctx request.CTX, cfg *model.Config, name string, scheduler Scheduler) (*model.Job, *model.AppError) {
+	pendingJobs, err := schedulers.jobs.CheckForPendingJobsByType(name)
+	if err != nil {
+		return nil, err
+	}
+
+	lastSuccessfulJob, err := schedulers.jobs.GetLastSuccessfulJobByType(name)
+	if err != nil {
+		return nil, err
+	}
+
+	return scheduler.ScheduleJob(rctx, cfg, pendingJobs, lastSuccessfulJob)
+}
+
+func (schedulers *Schedulers) handleConfigChange(_, newConfig *model.Config) {
+	mlog.Debug("Schedulers received config change.")
+	select {
+	case schedulers.configChanged <- newConfig:
+	case <-schedulers.stop:
+	}
+}
+
+func (schedulers *Schedulers) handleClusterLeaderChange(isLeader bool) {
+	select {
+	case schedulers.clusterLeaderChanged <- isLeader:
+	default:
+		mlog.Debug("Sending cluster leader change message to schedulers failed.")
+
+		// Drain the buffered channel to make room for the latest change.
+		select {
+		case <-schedulers.clusterLeaderChanged:
+		default:
+		}
+
+		// Enqueue the latest change. This operation is safe due to this method
+		// being called under lock.
+		schedulers.clusterLeaderChanged <- isLeader
+	}
+}

server/channels/jobs/schedulers_test.go

@@ -0,0 +1,188 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+package jobs
+
+import (
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
+	"github.com/mattermost/mattermost/server/v8/channels/utils/testutils"
+)
+
+type MockScheduler struct {
+	mock.Mock
+}
+
+func (scheduler *MockScheduler) Enabled(cfg *model.Config) bool {
+	return true
+}
+
+func (scheduler *MockScheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
+	nextTime := time.Now().Add(60 * time.Second)
+	return &nextTime
+}
+
+func (scheduler *MockScheduler) ScheduleJob(rctx request.CTX, cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
+	return nil, nil
+}
+
+func TestScheduler(t *testing.T) {
+	if os.Getenv("ENABLE_FULLY_PARALLEL_TESTS") == "true" {
+		t.Parallel()
+	}
+
+	mockStore := &storetest.Store{}
+	defer mockStore.AssertExpectations(t)
+
+	job := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: model.GetMillis(),
+		Status:   model.JobStatusPending,
+		Type:     model.JobTypeMessageExport,
+	}
+	// mock job store doesn't return a previously successful job, forcing fallback to config
+	mockStore.JobStore.On("GetNewestJobByStatusesAndType", mock.AnythingOfType("[]string"), mock.AnythingOfType("string")).Return(job, nil)
+	mockStore.JobStore.On("GetCountByStatusAndType", mock.AnythingOfType("string"), mock.AnythingOfType("string")).Return(int64(1), nil)
+
+	jobServer := &JobServer{
+		Store: mockStore,
+		ConfigService: &testutils.StaticConfigService{
+			Cfg: &model.Config{
+				// mock config
+				DataRetentionSettings: model.DataRetentionSettings{
+					EnableMessageDeletion: new(true),
+				},
+				MessageExportSettings: model.MessageExportSettings{
+					EnableExport: new(true),
+				},
+			},
+		},
+	}
+
+	jobServer.initSchedulers()
+	jobServer.RegisterJobType(model.JobTypeDataRetention, nil, new(MockScheduler))
+	jobServer.RegisterJobType(model.JobTypeMessageExport, nil, new(MockScheduler))
+
+	t.Run("Base", func(t *testing.T) {
+		err := jobServer.StartSchedulers()
+		require.NoError(t, err)
+
+		time.Sleep(2 * time.Second)
+
+		err = jobServer.StopSchedulers()
+		require.NoError(t, err)
+
+		// They should be all on here
+		for _, element := range jobServer.schedulers.nextRunTimes {
+			assert.NotNil(t, element)
+		}
+	})
+
+	t.Run("ClusterLeaderChanged", func(t *testing.T) {
+		jobServer.initSchedulers()
+		err := jobServer.StartSchedulers()
+		require.NoError(t, err)
+
+		time.Sleep(2 * time.Second)
+		jobServer.HandleClusterLeaderChange(false)
+
+		err = jobServer.StopSchedulers()
+		require.NoError(t, err)
+
+		// They should be turned off
+		for _, element := range jobServer.schedulers.nextRunTimes {
+			assert.Nil(t, element)
+		}
+	})
+
+	t.Run("ClusterLeaderChangedBeforeStart", func(t *testing.T) {
+		jobServer.initSchedulers()
+		jobServer.HandleClusterLeaderChange(false)
+		err := jobServer.StartSchedulers()
+		require.NoError(t, err)
+
+		time.Sleep(2 * time.Second)
+		err = jobServer.StopSchedulers()
+		require.NoError(t, err)
+
+		for _, element := range jobServer.schedulers.nextRunTimes {
+			assert.Nil(t, element)
+		}
+	})
+
+	t.Run("DoubleClusterLeaderChangedBeforeStart", func(t *testing.T) {
+		jobServer.initSchedulers()
+		jobServer.HandleClusterLeaderChange(false)
+		jobServer.HandleClusterLeaderChange(true)
+		err := jobServer.StartSchedulers()
+		require.NoError(t, err)
+
+		time.Sleep(2 * time.Second)
+		err = jobServer.StopSchedulers()
+		require.NoError(t, err)
+
+		for _, element := range jobServer.schedulers.nextRunTimes {
+			assert.NotNil(t, element)
+		}
+	})
+
+	t.Run("ConfigChanged", func(t *testing.T) {
+		jobServer.initSchedulers()
+		err := jobServer.StartSchedulers()
+		require.NoError(t, err)
+
+		time.Sleep(2 * time.Second)
+		jobServer.HandleClusterLeaderChange(false)
+		// After running a config change, they should stay off
+		jobServer.schedulers.handleConfigChange(nil, nil)
+		err = jobServer.StopSchedulers()
+		require.NoError(t, err)
+
+		for _, element := range jobServer.schedulers.nextRunTimes {
+			assert.Nil(t, element)
+		}
+	})
+
+	t.Run("ConfigChangedDeadlock", func(t *testing.T) {
+		jobServer.initSchedulers()
+		err := jobServer.StartSchedulers()
+		require.NoError(t, err)
+
+		time.Sleep(2 * time.Second)
+
+		var wg sync.WaitGroup
+		wg.Add(2)
+		go func() {
+			defer wg.Done()
+			err := jobServer.StopSchedulers()
+			require.NoError(t, err)
+		}()
+		go func() {
+			defer wg.Done()
+			jobServer.schedulers.handleConfigChange(nil, nil)
+		}()
+
+		wg.Wait()
+	})
+}
+
+func TestRandomDelay(t *testing.T) {
+	if os.Getenv("ENABLE_FULLY_PARALLEL_TESTS") == "true" {
+		t.Parallel()
+	}
+
+	cases := []int64{5, 10, 100}
+	for _, c := range cases {
+		out := getRandomDelay(c)
+		require.Less(t, out.Milliseconds(), c)
+	}
+}

server/channels/jobs/server.go

@@ -0,0 +1,135 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"sync"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/configservice"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/mattermost/mattermost/server/v8/einterfaces"
+)
+
+type JobServer struct {
+	ConfigService configservice.ConfigService
+	Store         store.Store
+	metrics       einterfaces.MetricsInterface
+	logger        mlog.LoggerIFace
+	publish       func(*model.WebSocketEvent)
+
+	// mut is used to protect the following fields from concurrent access.
+	mut        sync.Mutex
+	workers    *Workers
+	schedulers *Schedulers
+}
+
+func NewJobServer(configService configservice.ConfigService, store store.Store, metrics einterfaces.MetricsInterface, logger mlog.LoggerIFace, publish func(*model.WebSocketEvent)) *JobServer {
+	srv := &JobServer{
+		ConfigService: configService,
+		Store:         store,
+		metrics:       metrics,
+		logger:        logger,
+		publish:       publish,
+	}
+	srv.initWorkers()
+	srv.initSchedulers()
+	return srv
+}
+
+func (srv *JobServer) initWorkers() {
+	workers := NewWorkers(srv.ConfigService)
+	workers.Watcher = srv.MakeWatcher(workers, DefaultWatcherPollingInterval)
+	srv.workers = workers
+}
+
+func (srv *JobServer) initSchedulers() {
+	schedulers := &Schedulers{
+		configChanged:        make(chan *model.Config),
+		clusterLeaderChanged: make(chan bool, 1),
+		jobs:                 srv,
+		isLeader:             true,
+		schedulers:           make(map[string]Scheduler),
+		nextRunTimes:         make(map[string]*time.Time),
+	}
+
+	srv.schedulers = schedulers
+}
+
+func (srv *JobServer) Config() *model.Config {
+	return srv.ConfigService.Config()
+}
+
+func (srv *JobServer) Logger() mlog.LoggerIFace {
+	return srv.logger
+}
+
+func (srv *JobServer) RegisterJobType(name string, worker model.Worker, scheduler Scheduler) {
+	srv.mut.Lock()
+	defer srv.mut.Unlock()
+	if worker != nil {
+		srv.workers.AddWorker(name, worker)
+	}
+	if scheduler != nil {
+		srv.schedulers.AddScheduler(name, scheduler)
+	}
+}
+
+func (srv *JobServer) StartWorkers() error {
+	srv.mut.Lock()
+	defer srv.mut.Unlock()
+	if srv.workers == nil {
+		return ErrWorkersUninitialized
+	} else if srv.workers.running {
+		return ErrWorkersRunning
+	}
+	srv.workers.Start()
+	return nil
+}
+
+func (srv *JobServer) StartSchedulers() error {
+	srv.mut.Lock()
+	defer srv.mut.Unlock()
+	if srv.schedulers == nil {
+		return ErrSchedulersUninitialized
+	} else if srv.schedulers.running {
+		return ErrSchedulersRunning
+	}
+	srv.schedulers.Start()
+	return nil
+}
+
+func (srv *JobServer) StopWorkers() error {
+	srv.mut.Lock()
+	defer srv.mut.Unlock()
+	if srv.workers == nil {
+		return ErrWorkersUninitialized
+	} else if !srv.workers.running {
+		return ErrWorkersNotRunning
+	}
+	srv.workers.Stop()
+	return nil
+}
+
+func (srv *JobServer) StopSchedulers() error {
+	srv.mut.Lock()
+	defer srv.mut.Unlock()
+	if srv.schedulers == nil {
+		return ErrSchedulersUninitialized
+	} else if !srv.schedulers.running {
+		return ErrSchedulersNotRunning
+	}
+	srv.schedulers.Stop()
+	return nil
+}
+
+func (srv *JobServer) HandleClusterLeaderChange(isLeader bool) {
+	srv.mut.Lock()
+	defer srv.mut.Unlock()
+	if srv.schedulers != nil {
+		srv.schedulers.handleClusterLeaderChange(isLeader)
+	}
+}

server/channels/jobs/server_test.go

@@ -0,0 +1,140 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestStartWorkers(t *testing.T) {
+	if os.Getenv("ENABLE_FULLY_PARALLEL_TESTS") == "true" {
+		t.Parallel()
+	}
+
+	t.Run("uninitialized", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		err := jobServer.StartWorkers()
+		require.Equal(t, ErrWorkersUninitialized, err)
+	})
+
+	t.Run("already running", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		jobServer.initWorkers()
+		err := jobServer.StartWorkers()
+		require.NoError(t, err)
+		err = jobServer.StartWorkers()
+		require.Equal(t, ErrWorkersRunning, err)
+		// Parking the go routing to let the worker watcher start
+		time.Sleep(1 * time.Millisecond)
+		err = jobServer.StopWorkers()
+		require.NoError(t, err)
+	})
+
+	t.Run("not running", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		jobServer.initWorkers()
+		err := jobServer.StartWorkers()
+		require.NoError(t, err)
+		// Parking the go routing to let the worker watcher start
+		time.Sleep(1 * time.Millisecond)
+		err = jobServer.StopWorkers()
+		require.NoError(t, err)
+	})
+}
+
+func TestStopWorkers(t *testing.T) {
+	if os.Getenv("ENABLE_FULLY_PARALLEL_TESTS") == "true" {
+		t.Parallel()
+	}
+
+	t.Run("uninitialized", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		err := jobServer.StopWorkers()
+		require.Equal(t, ErrWorkersUninitialized, err)
+	})
+
+	t.Run("not running", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		jobServer.initWorkers()
+		err := jobServer.StopWorkers()
+		require.Equal(t, ErrWorkersNotRunning, err)
+	})
+
+	t.Run("running", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		jobServer.initWorkers()
+		err := jobServer.StartWorkers()
+		require.NoError(t, err)
+		// Parking the go routing to let the worker watcher start
+		time.Sleep(1 * time.Millisecond)
+		err = jobServer.StopWorkers()
+		require.NoError(t, err)
+	})
+}
+
+func TestStartSchedulers(t *testing.T) {
+	if os.Getenv("ENABLE_FULLY_PARALLEL_TESTS") == "true" {
+		t.Parallel()
+	}
+
+	t.Run("uninitialized", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		err := jobServer.StartSchedulers()
+		require.Equal(t, ErrSchedulersUninitialized, err)
+	})
+
+	t.Run("initialized", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		jobServer.initSchedulers()
+		err := jobServer.StartSchedulers()
+		require.NoError(t, err)
+
+		err = jobServer.StopSchedulers()
+		require.NoError(t, err)
+	})
+
+	t.Run("already running", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		jobServer.initSchedulers()
+		err := jobServer.StartSchedulers()
+		require.NoError(t, err)
+		err = jobServer.StartSchedulers()
+		require.Equal(t, ErrSchedulersRunning, err)
+
+		err = jobServer.StopSchedulers()
+		require.NoError(t, err)
+	})
+}
+
+func TestStopSchedulers(t *testing.T) {
+	if os.Getenv("ENABLE_FULLY_PARALLEL_TESTS") == "true" {
+		t.Parallel()
+	}
+
+	t.Run("uninitialized", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		err := jobServer.StopSchedulers()
+		require.Equal(t, ErrSchedulersUninitialized, err)
+	})
+
+	t.Run("not running", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		jobServer.initSchedulers()
+		err := jobServer.StopSchedulers()
+		require.Equal(t, ErrSchedulersNotRunning, err)
+	})
+
+	t.Run("running", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		jobServer.initSchedulers()
+		err := jobServer.StartSchedulers()
+		require.NoError(t, err)
+		err = jobServer.StopSchedulers()
+		require.NoError(t, err)
+	})
+}

server/channels/jobs/workers.go

@@ -0,0 +1,91 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"errors"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/configservice"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+)
+
+type Workers struct {
+	ConfigService configservice.ConfigService
+	Watcher       *Watcher
+
+	workers map[string]model.Worker
+
+	listenerId string
+	running    bool
+}
+
+var (
+	ErrWorkersNotRunning    = errors.New("job workers are not running")
+	ErrWorkersRunning       = errors.New("job workers are running")
+	ErrWorkersUninitialized = errors.New("job workers are not initialized")
+)
+
+func NewWorkers(configService configservice.ConfigService) *Workers {
+	return &Workers{
+		ConfigService: configService,
+		workers:       make(map[string]model.Worker),
+	}
+}
+
+func (workers *Workers) AddWorker(name string, worker model.Worker) {
+	workers.workers[name] = worker
+}
+
+func (workers *Workers) Get(name string) model.Worker {
+	return workers.workers[name]
+}
+
+// Start starts the workers. This call is not safe for concurrent use.
+// Synchronization should be implemented by the caller.
+func (workers *Workers) Start() {
+	mlog.Info("Starting workers")
+
+	for _, w := range workers.workers {
+		if w.IsEnabled(workers.ConfigService.Config()) {
+			go w.Run()
+		}
+	}
+
+	go workers.Watcher.Start()
+
+	workers.listenerId = workers.ConfigService.AddConfigListener(workers.handleConfigChange)
+	workers.running = true
+}
+
+func (workers *Workers) handleConfigChange(oldConfig *model.Config, newConfig *model.Config) {
+	mlog.Debug("Workers received config change.")
+
+	for _, w := range workers.workers {
+		if w.IsEnabled(oldConfig) && !w.IsEnabled(newConfig) {
+			w.Stop()
+		}
+		if !w.IsEnabled(oldConfig) && w.IsEnabled(newConfig) {
+			go w.Run()
+		}
+	}
+}
+
+// Stop stops the workers. This call is not safe for concurrent use.
+// Synchronization should be implemented by the caller.
+func (workers *Workers) Stop() {
+	workers.ConfigService.RemoveConfigListener(workers.listenerId)
+
+	workers.Watcher.Stop()
+
+	for _, w := range workers.workers {
+		if w.IsEnabled(workers.ConfigService.Config()) {
+			w.Stop()
+		}
+	}
+
+	workers.running = false
+
+	mlog.Info("Stopped workers")
+}