snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/automation/service/session.go

@@ -0,0 +1,588 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/automation/types"
+	"github.com/cortezaproject/corteza/server/pkg/actionlog"
+	"github.com/cortezaproject/corteza/server/pkg/auth"
+	"github.com/cortezaproject/corteza/server/pkg/errors"
+	"github.com/cortezaproject/corteza/server/pkg/expr"
+	"github.com/cortezaproject/corteza/server/pkg/logger"
+	"github.com/cortezaproject/corteza/server/pkg/options"
+	"github.com/cortezaproject/corteza/server/pkg/sentry"
+	"github.com/cortezaproject/corteza/server/pkg/wfexec"
+	"github.com/cortezaproject/corteza/server/store"
+	"github.com/modern-go/reflect2"
+	"go.uber.org/zap"
+)
+
+type (
+	promptSender interface {
+		Send(kind string, payload interface{}, userIDs ...uint64) error
+	}
+
+	session struct {
+		store        store.Storer
+		actionlog    actionlog.Recorder
+		ac           sessionAccessController
+		opt          options.WorkflowOpt
+		log          *zap.Logger
+		mux          sync.RWMutex
+		pool         map[uint64]*types.Session
+		spawnQueue   chan *spawn
+		promptSender promptSender
+	}
+
+	spawn struct {
+		workflowID uint64
+		session    chan *wfexec.Session
+		graph      *wfexec.Graph
+		invoker    auth.Identifiable
+		runner     auth.Identifiable
+		trace      bool
+		callStack  []uint64
+	}
+
+	sessionAccessController interface {
+		CanSearchSessions(context.Context) bool
+		CanManageSessionsOnWorkflow(context.Context, *types.Workflow) bool
+	}
+
+	WaitFn func(ctx context.Context) (*expr.Vars, uint64, wfexec.SessionStatus, types.Stacktrace, error)
+)
+
+const (
+	// when the state changes, state-change-handler is called and for non-fatal,
+	// non-interactive or non-delay steps (that are much more frequent) we need
+	// to limit how often the store is updated with the updated session info
+	//
+	// We use the size of the stacktrace and for every F (see the value of the constant)
+	// we flush the session info to the store.
+	sessionStateFlushFrequency = 1000
+)
+
+func Session(log *zap.Logger, opt options.WorkflowOpt, ps promptSender) *session {
+	return &session{
+		log:          log,
+		opt:          opt,
+		actionlog:    DefaultActionlog,
+		store:        DefaultStore,
+		ac:           DefaultAccessControl,
+		pool:         make(map[uint64]*types.Session),
+		spawnQueue:   make(chan *spawn),
+		promptSender: ps,
+	}
+}
+
+func (svc *session) Search(ctx context.Context, filter types.SessionFilter) (rr types.SessionSet, f types.SessionFilter, err error) {
+	var (
+		sap = &sessionActionProps{filter: &filter}
+	)
+
+	err = func() (err error) {
+		if !svc.ac.CanSearchSessions(ctx) {
+			return SessionErrNotAllowedToSearch()
+		}
+
+		if rr, f, err = store.SearchAutomationSessions(ctx, svc.store, filter); err != nil {
+			return err
+		}
+
+		return nil
+	}()
+
+	return rr, f, svc.recordAction(ctx, sap, SessionActionSearch, err)
+}
+
+func (svc *session) LookupByID(ctx context.Context, sessionID uint64) (res *types.Session, err error) {
+	var (
+		sap = &sessionActionProps{session: &types.Session{ID: sessionID}}
+		wf  *types.Workflow
+	)
+
+	err = store.Tx(ctx, svc.store, func(ctx context.Context, s store.Storer) error {
+		if !svc.ac.CanSearchSessions(ctx) {
+			return SessionErrNotAllowedToRead()
+		}
+
+		if res, err = loadSession(ctx, s, sessionID); err != nil {
+			return err
+		}
+
+		if wf, err = loadWorkflow(ctx, s, res.WorkflowID); err != nil {
+			return err
+		}
+
+		if !svc.ac.CanManageSessionsOnWorkflow(ctx, wf) {
+			return SessionErrNotAllowedToManage()
+		}
+
+		return nil
+	})
+
+	return res, svc.recordAction(ctx, sap, SessionActionLookup, err)
+}
+
+func (svc *session) resumeAll(ctx context.Context) error {
+	// In theory we could resume active/pending/prompt sessions from persistent store
+	// so that they can survive server termination
+
+	// @todo resume active sessions from storage
+	//       load all active sessions from store and load them into the pool
+	//
+	return nil
+}
+
+func (svc *session) suspendAll(ctx context.Context) error {
+	// In theory we could suspend active/pending/prompt sessions to persistent store
+	// so that they can survive server termination
+
+	// @todo suspend active sessions to storage:
+	//       stop watcher queue
+	//       run gc
+	//       stop worker on each session
+	//       flush session to store (like we're doing in the status handler
+	return nil
+}
+
+// PendingPrompts returns all prompts on all sessions owned by current user
+func (svc *session) PendingPrompts(ctx context.Context) (pp []*wfexec.PendingPrompt) {
+	var (
+		i = auth.GetIdentityFromContext(ctx)
+	)
+
+	if i == nil {
+		return
+	}
+
+	svc.mux.RLock()
+	defer svc.mux.RUnlock()
+
+	pp = make([]*wfexec.PendingPrompt, 0, len(svc.pool))
+	for _, s := range svc.pool {
+		pp = append(pp, s.PendingPrompts(i.Identity())...)
+	}
+
+	return
+}
+
+// Start new workflow session on a specific step with a given identity and scope
+//
+// # Start is an asynchronous operation
+//
+// Please note that context passed to the function is NOT the the one that is
+// used for the execution of the workflow. See watch function!
+//
+// It does not check user's permissions to execute workflow(s) so it should be used only when !
+func (svc *session) Start(ctx context.Context, g *wfexec.Graph, ssp types.SessionStartParams) (wait WaitFn, _ uint64, err error) {
+	var (
+		start wfexec.Step
+	)
+
+	if g == nil {
+		return nil, 0, errors.InvalidData("cannot start workflow, uninitialized graph")
+	}
+
+	if len(ssp.CallStack) > svc.opt.CallStackSize {
+		return nil, 0, WorkflowErrMaximumCallStackSizeExceeded()
+	}
+
+	ssp.CallStack = append(ssp.CallStack, ssp.WorkflowID)
+
+	if ssp.Invoker == nil {
+		return nil, 0, errors.InvalidData("cannot start workflow without user")
+	}
+
+	if ssp.Runner == nil {
+		ssp.Runner = ssp.Invoker
+	}
+
+	if ssp.StepID == 0 {
+		// starting step is not explicitly set
+		// find orphan step
+		switch oo := g.Orphans(); len(oo) {
+		case 1:
+			start = oo[0]
+		case 0:
+			return nil, 0, errors.InvalidData("could not find starting step")
+		default:
+			return nil, 0, errors.InvalidData("cannot start workflow session multiple starting steps found")
+		}
+	} else if start = g.StepByID(ssp.StepID); start == nil {
+		return nil, 0, errors.InvalidData("trigger staring step references non-existing step")
+	} else if len(g.Parents(g.StepByID(ssp.StepID))) > 0 {
+		return nil, 0, errors.InvalidData("cannot start workflow on a step with parents")
+	}
+
+	var (
+		ses = svc.spawn(g, ssp.WorkflowID, ssp.Trace, ssp.CallStack, ssp.Runner, ssp.Invoker)
+	)
+
+	ses.CreatedAt = *now()
+	ses.CreatedBy = ssp.Invoker.Identity()
+	ses.Status = types.SessionStarted
+	ses.Apply(ssp)
+
+	_ = ssp.Input.AssignFieldValue("eventType", expr.Must(expr.NewString(ssp.EventType)))
+	_ = ssp.Input.AssignFieldValue("resourceType", expr.Must(expr.NewString(ssp.ResourceType)))
+	_ = ssp.Input.AssignFieldValue("invoker", expr.Must(expr.NewAny(ssp.Invoker)))
+	_ = ssp.Input.AssignFieldValue("runner", expr.Must(expr.NewAny(ssp.Runner)))
+
+	if err = ses.Exec(ctx, start, ssp.Input); err != nil {
+		return
+	}
+
+	return func(ctx context.Context) (scope *expr.Vars, sessionID uint64, status wfexec.SessionStatus, stacktrace types.Stacktrace, err error) {
+		sessionID = ses.ID
+		scope, status, stacktrace, err = ses.WaitResults(ctx)
+		return
+	}, ses.ID, nil
+}
+
+// Resume resumes suspended session/state
+//
+// Session can only be resumed by knowing session and state ID. Resume is an asynchronous operation
+//
+// There is minimum access-control deep inside wfexec.Session.Resume function
+// that compares identity with state owner
+func (svc *session) Resume(sessionID, stateID uint64, i auth.Identifiable, input *expr.Vars) error {
+	var (
+		ctx = auth.SetIdentityToContext(context.Background(), i)
+	)
+
+	svc.mux.RLock()
+	defer svc.mux.RUnlock()
+	ses := svc.pool[sessionID]
+	if ses == nil {
+		return errors.NotFound("session not found")
+	}
+
+	resPrompt, err := ses.Resume(ctx, stateID, input)
+	if err != nil {
+		return err
+	}
+
+	if err = svc.promptSender.Send("workflowSessionResumed", resPrompt, resPrompt.OwnerId); err != nil {
+		svc.log.Error("failed to send prompt resume status to user", zap.Error(err))
+	}
+
+	return nil
+}
+
+// Terminates session ID
+func (svc *session) Cancel(ctx context.Context, sessionID uint64) (err error) {
+	svc.mux.RLock()
+
+	var (
+		wf  *types.Workflow
+		ses = svc.pool[sessionID]
+	)
+
+	// unlock right away!
+	// when session is canceled, handler pick it up and
+	// locks it again
+	svc.mux.RUnlock()
+
+	if ses == nil {
+		return errors.NotFound("session not found or already canceled")
+	}
+
+	if wf, err = loadWorkflow(ctx, svc.store, ses.WorkflowID); err != nil {
+		return
+	}
+
+	if !svc.ac.CanManageSessionsOnWorkflow(ctx, wf) {
+		return SessionErrNotAllowedToManage()
+	}
+
+	ses.Cancel()
+	return nil
+}
+
+// spawns a new session
+//
+// We need initial context for the session because we want to catch all cancellations or timeouts from there
+// and not from any potential HTTP requests or similar temporary context that can prematurely destroy a workflow session
+func (svc *session) spawn(g *wfexec.Graph, workflowID uint64, trace bool, callStack []uint64, runner, invoker auth.Identifiable) (ses *types.Session) {
+	s := &spawn{
+		workflowID: workflowID,
+		session:    make(chan *wfexec.Session, 1),
+		graph:      g,
+		trace:      trace,
+		callStack:  callStack,
+		invoker:    invoker,
+		runner:     runner,
+	}
+
+	// Send new-session request
+	svc.spawnQueue <- s
+
+	// blocks until session is set
+	ses = types.NewSession(<-s.session)
+	if !svc.opt.StackTraceEnabled {
+		ses.DisableStacktrace()
+	}
+
+	if svc.opt.StackTraceFull {
+		ses.FullStacktrace()
+	}
+
+	if trace {
+		ses.Traced()
+	}
+
+	svc.mux.Lock()
+	svc.pool[ses.ID] = ses
+	svc.mux.Unlock()
+	return ses
+}
+
+// Watch looks over session's spawn queue
+func (svc *session) Watch(ctx context.Context) {
+	gcTicker := time.NewTicker(time.Second)
+	lpTicker := time.NewTicker(time.Second * 30)
+
+	go func() {
+		defer sentry.Recover()
+		defer gcTicker.Stop()
+		defer svc.log.Info("stopped")
+
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case s := <-svc.spawnQueue:
+				var execCtx = context.Background()
+
+				opts := []wfexec.SessionOpt{
+					wfexec.SetWorkflowID(s.workflowID),
+					wfexec.SetCallStack(s.callStack...),
+					wfexec.SetHandler(svc.stateChangeHandler(ctx)),
+				}
+
+				if svc.opt.ExecDebug {
+					log := svc.log.
+						Named("exec").
+						With(logger.Uint64("workflowID", s.workflowID)).
+						With(logger.Uint64("runnerID", s.runner.Identity())).
+						With(logger.Uint64s("runnerRoles", s.runner.Roles()))
+
+					opts = append(
+						opts,
+						wfexec.SetLogger(log),
+						wfexec.SetDumpStacktraceOnPanic(true),
+					)
+				}
+
+				// Encode runner into execution context
+				// runner is used as identity and for access control
+				execCtx = auth.SetIdentityToContext(execCtx, s.runner)
+
+				// Encode invoker into execution context
+				// invoker is used
+				execCtx = context.WithValue(execCtx, workflowInvokerCtxKey{}, s.invoker)
+
+				s.session <- wfexec.NewSession(execCtx, s.graph, opts...)
+				// case time for a pool cleanup
+				// @todo cleanup pool when sessions are complete
+
+			case <-gcTicker.C:
+				svc.gc()
+
+			case <-lpTicker.C:
+				svc.logPending()
+			}
+		}
+
+		// @todo serialize sessions & suspended states
+		//svc.suspendAll(ctx)
+	}()
+
+	svc.log.Debug("watcher initialized")
+}
+
+// garbage collection for stale sessions
+func (svc *session) gc() {
+	svc.mux.Lock()
+	defer svc.mux.Unlock()
+
+	for _, s := range svc.pool {
+		if s.GC() {
+			delete(svc.pool, s.ID)
+		}
+	}
+}
+
+// garbage collection for stale sessions
+func (svc *session) logPending() {
+	svc.mux.RLock()
+	defer svc.mux.RUnlock()
+
+	var (
+		total                                    = len(svc.pool)
+		pending, pending1m, pending1h, pending1d int
+	)
+
+	for _, s := range svc.pool {
+		switch age := now().Sub(s.CreatedAt); {
+		case age > time.Hour*24:
+			pending1d++
+		case age > time.Hour:
+			pending1h++
+		case age > time.Minute:
+			pending1m++
+		default:
+			pending++
+		}
+	}
+
+	if total > 0 {
+		svc.log.Info(
+			"workflow session garbage collector stats",
+			zap.Int("total", total),
+			zap.Int("pending", pending),
+			zap.Int("pending1m", pending1m),
+			zap.Int("pending1h", pending1h),
+			zap.Int("pending1d", pending1d),
+		)
+	}
+}
+
+// stateChangeHandler keeps track of session status changes and frequently stores session into db
+func (svc *session) stateChangeHandler(ctx context.Context) wfexec.StateChangeHandler {
+	return func(status wfexec.SessionStatus, state *wfexec.State, s *wfexec.Session) (err error) {
+		svc.mux.Lock()
+		defer svc.mux.Unlock()
+
+		// Handle potential state change handler panics and gracefully error out
+		defer func() {
+			reason := recover()
+			if reflect2.IsNil(reason) {
+				return
+			}
+
+			err = fmt.Errorf("automation/service/session.go#stateChangeHandler panic: %v", reason)
+		}()
+
+		ses := svc.pool[s.ID()]
+		if ses == nil {
+			log := svc.log.With(
+				logger.Uint64("sessionID", s.ID()),
+				zap.Stringer("status", status),
+			)
+
+			log.Warn("could not find session to update")
+			return
+		}
+
+		ses.FlushCounter++
+
+		var (
+			// By default, we want to update session when new status is prompted, delayed, completed, canceled or failed
+			// But if status is active, we'll flush it every X frames (sessionStateFlushFrequency)
+			update = true
+
+			frame *wfexec.Frame
+		)
+
+		// Building a frame deep-copies the whole scope, so only do it when the
+		// variables are going to be read back: when the session is traced, or
+		// when it is settling on a status someone will want explained.
+		//
+		// ElapsedTime is calculated on append; the oldest frames can be
+		// dropped, so the session keeps track of when tracing started.
+		if state != nil && svc.opt.StackTraceEnabled {
+			if ses.KeepsFrameScope() || status != wfexec.SessionActive {
+				frame = state.MakeFrame()
+			} else {
+				frame = state.MakeLightFrame()
+			}
+
+			ses.AppendRuntimeStacktrace(frame)
+		}
+
+		switch status {
+		case wfexec.SessionPrompted:
+			ses.SuspendedAt = now()
+			ses.Status = types.SessionPrompted
+
+			// Send the pending prompts to user
+			if svc.promptSender != nil {
+				for _, pp := range s.UnsentPendingPrompts() {
+					// Prevent prompts from being re-sent
+					//
+					// This here collects all of the pending prompts and (re-)sends them.
+					// The problem occurs if you display multiple prompts in parallel --
+					// they get duplicated.
+					//
+					// Patching back-end with this simple fix, but we should probably
+					// refactor the logic that handles prompting a bit.
+					pp.Original.MarkSent()
+
+					if err := svc.promptSender.Send("workflowSessionPrompt", pp, pp.OwnerId); err != nil {
+						svc.log.Error("failed to send prompt to user", zap.Error(err))
+					}
+				}
+			}
+
+		case wfexec.SessionDelayed:
+			ses.SuspendedAt = now()
+			ses.Status = types.SessionSuspended
+
+		case wfexec.SessionCompleted:
+			ses.SuspendedAt = nil
+			ses.CompletedAt = now()
+			ses.Status = types.SessionCompleted
+
+		case wfexec.SessionFailed:
+			ses.SuspendedAt = nil
+			ses.CompletedAt = now()
+			if state != nil {
+				ses.Error = state.Error()
+			}
+			ses.Status = types.SessionFailed
+
+		case wfexec.SessionCanceled:
+			ses.SuspendedAt = nil
+			ses.CompletedAt = now()
+			ses.Status = types.SessionCanceled
+
+		default:
+			// force update every X iterations
+			update = ses.FlushCounter >= sessionStateFlushFrequency
+			if !update {
+				return
+			}
+		}
+
+		// Reset the counter whenever we flush due to whatever scenario
+		ses.FlushCounter = 0
+		ses.CopyRuntimeStacktrace()
+		if err := store.UpsertAutomationSession(ctx, svc.store, ses); err != nil {
+			log := svc.log.With(
+				logger.Uint64("sessionID", s.ID()),
+				zap.Stringer("status", status),
+			)
+
+			log.Error("failed to update session", zap.Error(err))
+		}
+
+		return
+	}
+}
+
+func loadSession(ctx context.Context, s store.Storer, sessionID uint64) (res *types.Session, err error) {
+	if sessionID == 0 {
+		return nil, SessionErrInvalidID()
+	}
+
+	if res, err = store.LookupAutomationSessionByID(ctx, s, sessionID); errors.IsNotFound(err) {
+		return nil, SessionErrNotFound()
+	}
+
+	return
+}