snapshot of https://github.com/mattermost/mattermost.git (master, 2026-09-16; blobs > 131072B omitted)

server/channels/jobs/migrations/advanced_permissions_phase_2.go

@@ -0,0 +1,112 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+)
+
+type AdvancedPermissionsPhase2Progress struct {
+	CurrentTable  string `json:"current_table"`
+	LastTeamID    string `json:"last_team_id"`
+	LastChannelID string `json:"last_channel_id"`
+	LastUserID    string `json:"last_user"`
+}
+
+func (p *AdvancedPermissionsPhase2Progress) ToJSON() string {
+	b, _ := json.Marshal(p)
+	return string(b)
+}
+
+func AdvancedPermissionsPhase2ProgressFromJSON(data io.Reader) *AdvancedPermissionsPhase2Progress {
+	var o *AdvancedPermissionsPhase2Progress
+	err := json.NewDecoder(data).Decode(&o)
+	if err != nil {
+		mlog.Warn("Error decoding advanced permissions phase 2 progress", mlog.Err(err))
+	}
+	return o
+}
+
+func (p *AdvancedPermissionsPhase2Progress) IsValid() bool {
+	if !model.IsValidId(p.LastChannelID) {
+		return false
+	}
+
+	if !model.IsValidId(p.LastTeamID) {
+		return false
+	}
+
+	if !model.IsValidId(p.LastUserID) {
+		return false
+	}
+
+	switch p.CurrentTable {
+	case "TeamMembers":
+	case "ChannelMembers":
+	default:
+		return false
+	}
+
+	return true
+}
+
+func (worker *Worker) runAdvancedPermissionsPhase2Migration(lastDone string) (bool, string, *model.AppError) {
+	var progress *AdvancedPermissionsPhase2Progress
+	if lastDone == "" {
+		// Haven't started the migration yet.
+		progress = &AdvancedPermissionsPhase2Progress{
+			CurrentTable:  "TeamMembers",
+			LastChannelID: strings.Repeat("0", 26),
+			LastTeamID:    strings.Repeat("0", 26),
+			LastUserID:    strings.Repeat("0", 26),
+		}
+	} else {
+		err := json.NewDecoder(strings.NewReader(lastDone)).Decode(&progress)
+		if err != nil {
+			return false, "", model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "migrations.worker.run_advanced_permissions_phase_2_migration.invalid_progress", map[string]any{"lastDone": lastDone}, "", http.StatusInternalServerError).Wrap(err)
+		}
+		if !progress.IsValid() {
+			return false, "", model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "migrations.worker.run_advanced_permissions_phase_2_migration.invalid_progress", map[string]any{"progress": progress.ToJSON()}, "", http.StatusInternalServerError)
+		}
+	}
+
+	if progress.CurrentTable == "TeamMembers" {
+		// Run a TeamMembers migration batch.
+		result, err := worker.store.Team().MigrateTeamMembers(progress.LastTeamID, progress.LastUserID)
+		if err != nil {
+			return false, progress.ToJSON(), model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "app.team.migrate_team_members.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+		if result == nil {
+			// We haven't progressed. That means that we've reached the end of this stage of the migration, and should now advance to the next stage.
+			progress.LastUserID = strings.Repeat("0", 26)
+			progress.CurrentTable = "ChannelMembers"
+			return false, progress.ToJSON(), nil
+		}
+
+		progress.LastTeamID = result["TeamId"]
+		progress.LastUserID = result["UserId"]
+	} else if progress.CurrentTable == "ChannelMembers" {
+		// Run a ChannelMembers migration batch.
+		data, err := worker.store.Channel().MigrateChannelMembers(progress.LastChannelID, progress.LastUserID)
+		if err != nil {
+			return false, progress.ToJSON(), model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "app.channel.migrate_channel_members.select.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+		if data == nil {
+			// We haven't progressed. That means we've reached the end of this final stage of the migration.
+
+			return true, progress.ToJSON(), nil
+		}
+
+		progress.LastChannelID = data["ChannelId"]
+		progress.LastUserID = data["UserId"]
+	}
+
+	return false, progress.ToJSON(), nil
+}

server/channels/jobs/migrations/helper_test.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/stretchr/testify/require"
+)
+
+func Setup(tb testing.TB) store.Store {
+	store := mainHelper.GetStore()
+	store.DropAllTables()
+	return store
+}
+
+func deleteAllJobsByTypeAndMigrationKey(t *testing.T, store store.Store, jobType string, migrationKey string) {
+	ctx := request.TestContext(t)
+	jobs, err := store.Job().GetAllByType(ctx, model.JobTypeMigrations)
+	require.NoError(t, err)
+
+	for _, job := range jobs {
+		if key, ok := job.Data[JobDataKeyMigration]; ok && key == migrationKey {
+			_, err = store.Job().Delete(job.Id)
+			require.NoError(t, err)
+		}
+	}
+}

server/channels/jobs/migrations/main_test.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost/server/v8/channels/testlib"
+)
+
+var mainHelper *testlib.MainHelper
+
+func TestMain(m *testing.M) {
+	var options = testlib.HelperOptions{
+		EnableStore:     true,
+		EnableResources: true,
+	}
+
+	mainHelper = testlib.NewMainHelperWithOptions(&options)
+	defer mainHelper.Close()
+
+	mainHelper.Main(m)
+}

server/channels/jobs/migrations/migrations.go

@@ -0,0 +1,55 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"net/http"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+const (
+	MigrationStateUnscheduled = "unscheduled"
+	MigrationStateInProgress  = "in_progress"
+	MigrationStateCompleted   = "completed"
+
+	JobDataKeyMigration         = "migration_key"
+	JobDataKeyMigrationLastDone = "last_done"
+)
+
+func MakeMigrationsList() []string {
+	return []string{
+		model.MigrationKeyAdvancedPermissionsPhase2,
+	}
+}
+
+func GetMigrationState(rctx request.CTX, migration string, store store.Store) (string, *model.Job, *model.AppError) {
+	if _, err := store.System().GetByName(migration); err == nil {
+		return MigrationStateCompleted, nil, nil
+	}
+
+	jobs, err := store.Job().GetAllByType(rctx, model.JobTypeMigrations)
+	if err != nil {
+		return "", nil, model.NewAppError("GetMigrationState", "app.job.get_all.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	for _, job := range jobs {
+		if key, ok := job.Data[JobDataKeyMigration]; ok {
+			if key != migration {
+				continue
+			}
+
+			switch job.Status {
+			case model.JobStatusInProgress, model.JobStatusPending:
+				return MigrationStateInProgress, job, nil
+			default:
+				return MigrationStateUnscheduled, job, nil
+			}
+		}
+	}
+
+	return MigrationStateUnscheduled, nil, nil
+}

server/channels/jobs/migrations/migrations_test.go

@@ -0,0 +1,105 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+)
+
+func TestGetMigrationState(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+	}
+	store := Setup(t)
+	ctx := request.TestContext(t)
+
+	migrationKey := model.NewId()
+
+	deleteAllJobsByTypeAndMigrationKey(t, store, model.JobTypeMigrations, migrationKey)
+
+	// Test with no job yet.
+	state, job, err := GetMigrationState(ctx, migrationKey, store)
+	assert.Nil(t, err)
+	assert.Nil(t, job)
+	assert.Equal(t, "unscheduled", state)
+
+	// Test with the system table showing the migration as done.
+	system := model.System{
+		Name:  migrationKey,
+		Value: "true",
+	}
+	nErr := store.System().Save(&system)
+	assert.NoError(t, nErr)
+
+	state, job, err = GetMigrationState(ctx, migrationKey, store)
+	assert.Nil(t, err)
+	assert.Nil(t, job)
+	assert.Equal(t, "completed", state)
+
+	_, nErr = store.System().PermanentDeleteByName(migrationKey)
+	assert.NoError(t, nErr)
+
+	// Test with a job scheduled in "pending" state.
+	j1 := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: model.GetMillis(),
+		Data: map[string]string{
+			JobDataKeyMigration: migrationKey,
+		},
+		Status: model.JobStatusPending,
+		Type:   model.JobTypeMigrations,
+	}
+
+	j1, nErr = store.Job().Save(j1)
+	require.NoError(t, nErr)
+
+	state, job, err = GetMigrationState(ctx, migrationKey, store)
+	assert.Nil(t, err)
+	assert.Equal(t, j1.Id, job.Id)
+	assert.Equal(t, "in_progress", state)
+
+	// Test with a job scheduled in "in progress" state.
+	j2 := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: j1.CreateAt + 1,
+		Data: map[string]string{
+			JobDataKeyMigration: migrationKey,
+		},
+		Status: model.JobStatusInProgress,
+		Type:   model.JobTypeMigrations,
+	}
+
+	j2, nErr = store.Job().Save(j2)
+	require.NoError(t, nErr)
+
+	state, job, err = GetMigrationState(ctx, migrationKey, store)
+	assert.Nil(t, err)
+	assert.Equal(t, j2.Id, job.Id)
+	assert.Equal(t, "in_progress", state)
+
+	// Test with a job scheduled in "error" state.
+	j3 := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: j2.CreateAt + 1,
+		Data: map[string]string{
+			JobDataKeyMigration: migrationKey,
+		},
+		Status: model.JobStatusError,
+		Type:   model.JobTypeMigrations,
+	}
+
+	j3, nErr = store.Job().Save(j3)
+	require.NoError(t, nErr)
+
+	state, job, err = GetMigrationState(ctx, migrationKey, store)
+	assert.Nil(t, err)
+	assert.Equal(t, j3.Id, job.Id)
+	assert.Equal(t, "unscheduled", state)
+}

server/channels/jobs/migrations/scheduler.go

@@ -0,0 +1,110 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+const (
+	MigrationJobWedgedTimeoutMilliseconds = 3600000 // 1 hour
+)
+
+type Scheduler struct {
+	jobServer              *jobs.JobServer
+	store                  store.Store
+	allMigrationsCompleted bool
+}
+
+var _ jobs.Scheduler = (*Scheduler)(nil)
+
+func MakeScheduler(jobServer *jobs.JobServer, store store.Store) *Scheduler {
+	return &Scheduler{jobServer, store, false}
+}
+
+func (scheduler *Scheduler) Enabled(_ *model.Config) bool {
+	return true
+}
+
+//nolint:unparam
+func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
+	if scheduler.allMigrationsCompleted {
+		return nil
+	}
+
+	nextTime := time.Now().Add(60 * time.Second)
+	return &nextTime
+}
+
+//nolint:unparam
+func (scheduler *Scheduler) ScheduleJob(rctx request.CTX, cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
+	rctx.Logger().Debug("Scheduling Job", mlog.String("scheduler", model.JobTypeMigrations))
+
+	// Work through the list of migrations in order. Schedule the first one that isn't done (assuming it isn't in progress already).
+	for _, key := range MakeMigrationsList() {
+		state, job, err := GetMigrationState(rctx, key, scheduler.store)
+		if err != nil {
+			rctx.Logger().Error("Failed to determine status of migration: ", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key), mlog.Err(err))
+			return nil, nil
+		}
+
+		logger := rctx.Logger().With(jobs.JobLoggerFields(job)...)
+
+		if state == MigrationStateCompleted {
+			// This migration is done. Continue to check the next.
+			continue
+		}
+
+		if state == MigrationStateInProgress {
+			// Check the migration job isn't wedged.
+			if job != nil && job.LastActivityAt < model.GetMillis()-MigrationJobWedgedTimeoutMilliseconds && job.CreateAt < model.GetMillis()-MigrationJobWedgedTimeoutMilliseconds {
+				logger.Warn("Job appears to be wedged. Rescheduling another instance.", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key))
+				if err := scheduler.jobServer.SetJobError(job, nil); err != nil {
+					logger.Error("Worker: Failed to set job error", mlog.String("scheduler", model.JobTypeMigrations), mlog.Err(err))
+				}
+				return scheduler.createJob(rctx, key, job)
+			}
+
+			return nil, nil
+		}
+
+		if state == MigrationStateUnscheduled {
+			logger.Debug("Scheduling a new job for migration.", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key))
+			return scheduler.createJob(rctx, key, job)
+		}
+
+		logger.Error("Unknown migration state. Not doing anything.", mlog.String("migration_state", state))
+		return nil, nil
+	}
+
+	// If we reached here, then there aren't any migrations left to run.
+	scheduler.allMigrationsCompleted = true
+	rctx.Logger().Debug("All migrations are complete.", mlog.String("scheduler", model.JobTypeMigrations))
+
+	return nil, nil
+}
+
+func (scheduler *Scheduler) createJob(rctx request.CTX, migrationKey string, lastJob *model.Job) (*model.Job, *model.AppError) {
+	var lastDone string
+	if lastJob != nil {
+		lastDone = lastJob.Data[JobDataKeyMigrationLastDone]
+	}
+
+	data := map[string]string{
+		JobDataKeyMigration:         migrationKey,
+		JobDataKeyMigrationLastDone: lastDone,
+	}
+
+	job, err := scheduler.jobServer.CreateJob(rctx, model.JobTypeMigrations, data)
+	if err != nil {
+		return nil, err
+	}
+	return job, nil
+}

server/channels/jobs/migrations/worker.go

@@ -0,0 +1,188 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"context"
+	"net/http"
+	"sync/atomic"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+const (
+	TimeBetweenBatches = 100
+)
+
+type Worker struct {
+	name      string
+	stop      chan struct{}
+	stopped   chan bool
+	jobs      chan model.Job
+	jobServer *jobs.JobServer
+	logger    mlog.LoggerIFace
+	store     store.Store
+	closed    atomic.Int32
+}
+
+func MakeWorker(jobServer *jobs.JobServer, store store.Store) *Worker {
+	const workerName = "Migrations"
+	worker := Worker{
+		name:      workerName,
+		stop:      make(chan struct{}),
+		stopped:   make(chan bool, 1),
+		jobs:      make(chan model.Job),
+		jobServer: jobServer,
+		logger:    jobServer.Logger().With(mlog.String("worker_name", workerName)),
+		store:     store,
+	}
+
+	return &worker
+}
+
+func (worker *Worker) Run() {
+	// Set to open if closed before. We are not bothered about multiple opens.
+	if worker.closed.CompareAndSwap(1, 0) {
+		worker.stop = make(chan struct{})
+	}
+	worker.logger.Debug("Worker started")
+
+	defer func() {
+		worker.logger.Debug("Worker finished")
+		worker.stopped <- true
+	}()
+
+	for {
+		select {
+		case <-worker.stop:
+			worker.logger.Debug("Worker received stop signal")
+			return
+		case job := <-worker.jobs:
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *Worker) Stop() {
+	// Set to close, and if already closed before, then return.
+	if !worker.closed.CompareAndSwap(0, 1) {
+		return
+	}
+	worker.logger.Debug("Worker stopping")
+	close(worker.stop)
+	<-worker.stopped
+}
+
+func (worker *Worker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+func (worker *Worker) IsEnabled(_ *model.Config) bool {
+	return true
+}
+
+func (worker *Worker) DoJob(job *model.Job) {
+	logger := worker.logger.With(jobs.JobLoggerFields(job)...)
+	logger.Debug("Worker: Received a new candidate job.")
+
+	defer worker.jobServer.HandleJobPanic(logger, job)
+
+	var appErr *model.AppError
+	job, appErr = worker.jobServer.ClaimJob(job)
+	if appErr != nil {
+		logger.Warn("Worker experienced an error while trying to claim job", mlog.Err(appErr))
+		return
+	} else if job == nil {
+		return
+	}
+
+	var cancelContext request.CTX = request.EmptyContext(worker.logger)
+	cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
+	cancelWatcherChan := make(chan struct{}, 1)
+	cancelContext = cancelContext.WithContext(cancelCtx)
+	go worker.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan)
+	defer cancelCancelWatcher()
+
+	for {
+		select {
+		case <-cancelWatcherChan:
+			logger.Debug("Worker: Job has been canceled via CancellationWatcher")
+			worker.setJobCanceled(logger, job)
+			return
+
+		case <-worker.stop:
+			logger.Debug("Worker: Job has been canceled via Worker Stop")
+			worker.setJobCanceled(logger, job)
+			return
+
+		case <-time.After(TimeBetweenBatches * time.Millisecond):
+			done, progress, err := worker.runMigration(job.Data[JobDataKeyMigration], job.Data[JobDataKeyMigrationLastDone])
+			if err != nil {
+				logger.Error("Worker: Failed to run migration", mlog.Err(err))
+				worker.setJobError(logger, job, err)
+				return
+			} else if done {
+				logger.Info("Worker: Job is complete")
+				worker.setJobSuccess(logger, job)
+				return
+			} else {
+				job.Data[JobDataKeyMigrationLastDone] = progress
+				if err := worker.jobServer.UpdateInProgressJobData(job); err != nil {
+					logger.Error("Worker: Failed to update migration status data for job", mlog.Err(err))
+					worker.setJobError(logger, job, err)
+					return
+				}
+			}
+		}
+	}
+}
+
+func (worker *Worker) setJobSuccess(logger mlog.LoggerIFace, job *model.Job) {
+	if err := worker.jobServer.SetJobSuccess(job); err != nil {
+		logger.Error("Worker: Failed to set success for job", mlog.Err(err))
+		worker.setJobError(logger, job, err)
+	}
+}
+
+func (worker *Worker) setJobError(logger mlog.LoggerIFace, job *model.Job, appError *model.AppError) {
+	if err := worker.jobServer.SetJobError(job, appError); err != nil {
+		logger.Error("Worker: Failed to set job error", mlog.Err(err))
+	}
+}
+
+func (worker *Worker) setJobCanceled(logger mlog.LoggerIFace, job *model.Job) {
+	if err := worker.jobServer.SetJobCanceled(job); err != nil {
+		logger.Error("Worker: Failed to mark job as canceled", mlog.Err(err))
+	}
+}
+
+// Return parameters:
+// - whether the migration is completed on this run (true) or still incomplete (false).
+// - the updated lastDone string for the migration.
+// - any error which may have occurred while running the migration.
+func (worker *Worker) runMigration(key string, lastDone string) (bool, string, *model.AppError) {
+	var done bool
+	var progress string
+	var err *model.AppError
+
+	switch key {
+	case model.MigrationKeyAdvancedPermissionsPhase2:
+		done, progress, err = worker.runAdvancedPermissionsPhase2Migration(lastDone)
+	default:
+		return false, "", model.NewAppError("MigrationsWorker.runMigration", "migrations.worker.run_migration.unknown_key", map[string]any{"key": key}, "", http.StatusInternalServerError)
+	}
+
+	if done {
+		if nErr := worker.store.System().Save(&model.System{Name: key, Value: "true"}); nErr != nil {
+			return false, "", model.NewAppError("runMigration", "migrations.system.save.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
+		}
+	}
+
+	return done, progress, err
+}