snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/pkg/wfexec/frame_test.go

@@ -0,0 +1,83 @@
+package wfexec
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/pkg/expr"
+	"github.com/stretchr/testify/require"
+)
+
+// stateWithScope builds a state whose scope holds a record cache, the shape of
+// a workflow that keeps looked-up records in a variable
+func stateWithScope(records int) State {
+	cache := make(map[string]interface{}, records)
+	for i := 0; i < records; i++ {
+		cache[fmt.Sprintf("rec%d", i)] = fmt.Sprintf("some reasonably sized record value %d", i)
+	}
+
+	vv, err := expr.NewVars(map[string]interface{}{"cache": cache})
+	if err != nil {
+		panic(err)
+	}
+
+	return State{
+		created:   time.Now(),
+		sessionId: 42,
+		stateId:   43,
+		action:    "step executed",
+		scope:     vv,
+		input:     expr.EmptyVars(),
+		results:   expr.EmptyVars(),
+	}
+}
+
+func TestState_MakeLightFrameSkipsVariables(t *testing.T) {
+	var (
+		req = require.New(t)
+		s   = stateWithScope(50)
+
+		light = s.MakeLightFrame()
+		full  = s.MakeFrame()
+	)
+
+	// the light frame still describes the step
+	req.Equal(s.sessionId, light.SessionID)
+	req.Equal(s.stateId, light.StateID)
+	req.Equal(s.action, light.Action)
+	req.Equal(full.SessionID, light.SessionID)
+	req.Equal(full.StateID, light.StateID)
+	req.Equal(full.Action, light.Action)
+
+	// ...but carries none of the variables
+	req.Nil(light.Scope)
+	req.Nil(light.Input)
+	req.Nil(light.Results)
+
+	// while the full frame snapshots them
+	req.NotNil(full.Scope)
+	req.Len(full.Scope.GetValue(), 1)
+}
+
+// BenchmarkState_MakeFrame contrasts the cost of snapshotting a step against
+// only describing it
+func BenchmarkState_MakeFrame(b *testing.B) {
+	for _, records := range []int{100, 1000} {
+		s := stateWithScope(records)
+
+		b.Run(fmt.Sprintf("full/%d-records", records), func(b *testing.B) {
+			b.ReportAllocs()
+			for n := 0; n < b.N; n++ {
+				_ = s.MakeFrame()
+			}
+		})
+
+		b.Run(fmt.Sprintf("light/%d-records", records), func(b *testing.B) {
+			b.ReportAllocs()
+			for n := 0; n < b.N; n++ {
+				_ = s.MakeLightFrame()
+			}
+		})
+	}
+}

server/pkg/wfexec/gateways.go

@@ -0,0 +1,256 @@
+package wfexec
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/cortezaproject/corteza/server/pkg/expr"
+)
+
+// GatewayPath structure is used by subset of gateway nodes
+//
+// It allows to evaluate test Expression to help determine the
+// gateway if a certain path should be used or not
+type (
+	GatewayPaths []*GatewayPath
+	GatewayPath  struct {
+		test pathTester
+		to   Step
+	}
+
+	pathTester func(context.Context, *expr.Vars) (bool, error)
+)
+
+// NewGatewayPath validates Expression and returns initialized GatewayPath
+func NewGatewayPath(s Step, t pathTester) (gwp *GatewayPath, err error) {
+	return &GatewayPath{to: s, test: t}, nil
+}
+
+// joinGateway handles merging/joining of multiple paths into
+// a single path forward
+type (
+	joinGateway struct {
+		StepIdentifier
+		paths  Steps
+		scopes map[uint64]map[Step]*expr.Vars
+		dirty  map[uint64]map[Step]map[string]bool
+		l      sync.Mutex
+	}
+)
+
+// AddPath registers an additional parent path
+//
+// Used while building the graph, when a parent could not be resolved at the
+// time the gateway was created.
+func (gw *joinGateway) AddPath(s Step) {
+	gw.l.Lock()
+	defer gw.l.Unlock()
+
+	if !gw.paths.Contains(s) {
+		gw.paths = append(gw.paths, s)
+	}
+}
+
+// JoinGateway fn initializes join gateway with all paths that are expected to be partial
+func JoinGateway(ss ...Step) *joinGateway {
+	return &joinGateway{
+		paths: ss,
+
+		// group scopes by session and step
+		// this prevents scope corruption when same workflow
+		// is executed multiple times
+		//
+		// might not be the best way where to keep the state of the join-gateway
+		// but it beats hidden variables in the scope or dedicated prop in the
+		// ExecRequest
+		scopes: make(map[uint64]map[Step]*expr.Vars),
+		dirty:  make(map[uint64]map[Step]map[string]bool),
+	}
+}
+
+// Exec fn on join gateway can be called multiple times, even multiple times parent the same parent
+//
+// Func will collect results from each parent path.
+//
+// Join gateway is ready to continue when all configured paths have been collected
+// Results are merged to preserve changes from all parallel paths
+func (gw *joinGateway) Exec(_ context.Context, r *ExecRequest) (ExecResponse, error) {
+	gw.l.Lock()
+	defer gw.l.Unlock()
+
+	if !gw.paths.Contains(r.Parent) {
+		return nil, fmt.Errorf("unknown parent for join gateway")
+	}
+
+	if len(gw.scopes[r.SessionID]) == 0 {
+		gw.scopes[r.SessionID] = make(map[Step]*expr.Vars)
+		gw.dirty[r.SessionID] = make(map[Step]map[string]bool)
+	}
+
+	gw.scopes[r.SessionID][r.Parent] = r.Scope
+	gw.dirty[r.SessionID][r.Parent] = r.dirty
+
+	if len(gw.scopes[r.SessionID]) < len(gw.paths) {
+		return &partial{}, nil
+	}
+
+	// All collected; start from the first path's scope and apply what each
+	// path changed on top of it
+	//
+	// Merging whole scopes instead would let a path that never touched a
+	// variable overwrite another path's change with its own stale copy.
+	var merged *expr.Vars
+	if len(gw.paths) > 0 && gw.scopes[r.SessionID][gw.paths[0]] != nil {
+		merged = gw.scopes[r.SessionID][gw.paths[0]].MustMerge()
+	}
+
+	if merged == nil {
+		merged = &expr.Vars{}
+	}
+
+	// when more than one path changed the same variable, the last path wins
+	var changedNames []string
+	for _, p := range gw.paths {
+		changed := gw.changesOf(r.SessionID, p)
+
+		if !changed.IsEmpty() {
+			merged = merged.MustMerge(changed)
+		}
+
+		_ = changed.Each(func(k string, _ expr.TypedValue) error {
+			changedNames = append(changedNames, k)
+			return nil
+		})
+	}
+
+	// all inbound paths visited, cleanup collected state for the session
+	delete(gw.scopes, r.SessionID)
+	delete(gw.dirty, r.SessionID)
+
+	// only what the paths changed counts as this path's changes; reporting the
+	// whole merged scope would make an enclosing join treat every variable as
+	// changed here
+	return &joined{scope: merged, changed: changedNames}, nil
+}
+
+// changesOf returns variables the given path wrote after it branched off
+func (gw *joinGateway) changesOf(sessionID uint64, p Step) *expr.Vars {
+	var (
+		out   = &expr.Vars{}
+		scope = gw.scopes[sessionID][p]
+	)
+
+	if scope == nil {
+		return out
+	}
+
+	for name := range gw.dirty[sessionID][p] {
+		if v, err := scope.Select(name); err == nil {
+			_ = out.Set(name, v)
+		}
+	}
+
+	return out
+}
+
+// forkGateway handles forking to multiple paths
+type forkGateway struct {
+	StepIdentifier
+}
+
+// ForkGateway fn initializes fork gateway
+// No arguments are required; Graph Graph config is used to
+// determine all possible fork paths on the fly
+func ForkGateway() *forkGateway {
+	return &forkGateway{}
+}
+
+// Exec fn on fork gateway always returns empty Steps slice
+// This signals Graph executor to collect child nodes directly parent Graph
+func (gw forkGateway) Exec(context.Context, *ExecRequest) (ExecResponse, error) {
+	return Steps{}, nil
+}
+
+// inclGateway is an inclusive gateway that can return one or more paths
+type inclGateway struct {
+	StepIdentifier
+	paths []*GatewayPath
+}
+
+// InclGateway fn initializes inclusive gateway
+func InclGateway(pp ...*GatewayPath) (*inclGateway, error) {
+	if len(pp) < 2 {
+		return nil, fmt.Errorf("expecting at least two paths for incusive gateway")
+	}
+
+	for _, p := range pp {
+		if p.test == nil {
+			return nil, fmt.Errorf("all inclusve gateway paths must have valid test Expression")
+		}
+	}
+
+	return &inclGateway{paths: pp}, nil
+}
+
+// Exec fn on inclGateway uses current scope to test all configured paths
+//
+// One or more matched paths can be returned!
+func (gw inclGateway) Exec(ctx context.Context, r *ExecRequest) (ExecResponse, error) {
+	var paths Steps
+	for _, p := range gw.paths {
+		if result, err := p.test(ctx, r.Scope); err != nil {
+			return nil, err
+		} else if result {
+			paths = append(paths, p.to)
+		}
+	}
+
+	if len(paths) == 0 {
+		return nil, fmt.Errorf("inclusive gateway must match at least one condition")
+	}
+
+	return paths, nil
+}
+
+// exclGateway is an exclusive gateway that can return exactly one path
+type exclGateway struct {
+	StepIdentifier
+	paths []*GatewayPath
+}
+
+// ExclGateway fn initializes exclusive gateway
+func ExclGateway(pp ...*GatewayPath) (*exclGateway, error) {
+	t := len(pp)
+	if t < 2 {
+		return nil, fmt.Errorf("expecting at least two paths for exclusive gateway")
+	}
+
+	for i, p := range pp {
+		if p.test == nil && i != t-1 {
+			return nil, fmt.Errorf("all exclusive gateway paths must have valid test Expression")
+		}
+	}
+
+	return &exclGateway{paths: pp}, nil
+}
+
+// Exec fn on exclGateway uses current scope to test all configured paths
+//
+// Exactly one matched path can be returned.
+func (gw exclGateway) Exec(ctx context.Context, r *ExecRequest) (ExecResponse, error) {
+	for _, p := range gw.paths {
+		if p.test == nil {
+			// empty & last; treat it as else part of the if condition
+			return p.to, nil
+		}
+
+		if result, err := p.test(ctx, r.Scope); err != nil {
+			return nil, err
+		} else if result {
+			return p.to, nil
+		}
+	}
+
+	return nil, fmt.Errorf("exclusive gateway must match one condition")
+}

server/pkg/wfexec/gateways_test.go

@@ -0,0 +1,114 @@
+package wfexec
+
+import (
+	"context"
+	"github.com/cortezaproject/corteza/server/pkg/expr"
+	"github.com/spf13/cast"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func gt(key string, val int64) pathTester {
+	return func(ctx context.Context, variables *expr.Vars) (bool, error) {
+		return cast.ToInt64(expr.Must(expr.Select(variables, key)).Get()) > val, nil
+	}
+}
+
+func makeIntVars(k string, num int) *expr.Vars {
+	o := &expr.Vars{}
+	_ = o.Set(k, num)
+	return o
+}
+
+func TestJoinGateway(t *testing.T) {
+	var (
+		req        = require.New(t)
+		p1, p2, p3 = &wfTestStep{name: "p1"}, &wfTestStep{name: "p2"}, &wfTestStep{name: "p3"}
+		gw         = JoinGateway(p1, p2, p3)
+
+		r   ExecResponse
+		err error
+	)
+
+	r, err = gw.Exec(context.TODO(), &ExecRequest{Parent: p1, SessionID: 1})
+	req.NoError(err)
+	req.Equal(&partial{}, r)
+
+	r, err = gw.Exec(context.TODO(), &ExecRequest{Parent: p2, SessionID: 1})
+	req.NoError(err)
+	req.Equal(&partial{}, r)
+
+	r, err = gw.Exec(context.TODO(), &ExecRequest{Parent: p3, SessionID: 1})
+	req.NoError(err)
+	req.IsType(&joined{}, r)
+
+	r, err = gw.Exec(context.TODO(), &ExecRequest{Parent: p1, SessionID: 2})
+	req.NoError(err)
+	req.Equal(&partial{}, r)
+}
+
+func TestForkGateway(t *testing.T) {
+	var (
+		req = require.New(t)
+		gw  = ForkGateway()
+	)
+
+	r, err := gw.Exec(context.TODO(), nil)
+	req.NoError(err)
+	req.Equal(Steps{}, r)
+	req.Empty(r)
+}
+
+func TestInclGateway(t *testing.T) {
+	var (
+		req = require.New(t)
+
+		s1, s2, s3 = &wfTestStep{name: "s1"}, &wfTestStep{name: "s2"}, &wfTestStep{name: "s3"}
+		gwp1, _    = NewGatewayPath(s1, gt("a", 10))
+		gwp2, _    = NewGatewayPath(s2, gt("a", 5))
+		gwp3, _    = NewGatewayPath(s3, gt("a", 0))
+
+		gw, err = InclGateway(gwp1, gwp2, gwp3)
+	)
+
+	r, err := gw.Exec(context.Background(), &ExecRequest{Scope: makeIntVars("a", 11)})
+	req.NoError(err)
+	req.Equal(Steps{s1, s2, s3}, r)
+
+	r, err = gw.Exec(context.Background(), &ExecRequest{Scope: makeIntVars("a", 6)})
+	req.NoError(err)
+	req.Equal(Steps{s2, s3}, r)
+
+	r, err = gw.Exec(context.Background(), &ExecRequest{Scope: makeIntVars("a", 1)})
+	req.NoError(err)
+	req.Equal(Steps{s3}, r)
+
+	r, err = gw.Exec(context.Background(), &ExecRequest{Scope: makeIntVars("a", 0)})
+	req.Error(err)
+	req.Nil(r)
+}
+
+func TestExclGateway(t *testing.T) {
+	var (
+		req = require.New(t)
+
+		s1, s2, s3 = &wfTestStep{name: "s1"}, &wfTestStep{name: "s2"}, &wfTestStep{name: "s3"}
+		gwp1, _    = NewGatewayPath(s1, gt("a", 10))
+		gwp2, _    = NewGatewayPath(s2, gt("a", 5))
+		gwp3, _    = NewGatewayPath(s3, nil)
+
+		gw, err = ExclGateway(gwp1, gwp2, gwp3)
+	)
+
+	r, err := gw.Exec(context.Background(), &ExecRequest{Scope: makeIntVars("a", 11)})
+	req.NoError(err)
+	req.Equal(s1, r)
+
+	r, err = gw.Exec(context.Background(), &ExecRequest{Scope: makeIntVars("a", 6)})
+	req.NoError(err)
+	req.Equal(s2, r)
+
+	r, err = gw.Exec(context.Background(), &ExecRequest{Scope: makeIntVars("a", 1)})
+	req.NoError(err)
+	req.Equal(s3, r)
+}

server/pkg/wfexec/graph.go

@@ -0,0 +1,78 @@
+package wfexec
+
+import (
+	"context"
+)
+
+type (
+	// list of Graph steps with relations
+	Graph struct {
+		steps    []Step
+		children map[Step][]Step
+		parents  map[Step][]Step
+		index    map[uint64]Step
+	}
+)
+
+func NewGraph() *Graph {
+	wf := &Graph{
+		steps:    make([]Step, 0, 1024),
+		children: make(map[Step][]Step),
+		parents:  make(map[Step][]Step),
+		index:    make(map[uint64]Step),
+	}
+
+	return wf
+}
+
+func (g *Graph) AddStep(s Step, cc ...Step) {
+	g.steps = append(g.steps, s)
+
+	if id := s.ID(); id != 0 {
+		g.index[id] = s
+	}
+
+	if len(cc) > 0 {
+		for _, c := range cc {
+			g.AddParent(c, s)
+		}
+	}
+}
+
+func (g *Graph) Len() int {
+	return len(g.steps)
+}
+
+func (g *Graph) StepByID(ID uint64) Step {
+	return g.index[ID]
+}
+
+func (g *Graph) AddParent(c, p Step) {
+	g.parents[c] = append(g.parents[c], p)
+	g.children[p] = append(g.children[p], c)
+}
+
+func (g *Graph) Children(s Step) Steps {
+	return g.children[s]
+}
+
+func (g *Graph) Parents(s Step) Steps {
+	return g.parents[s]
+}
+
+func (g *Graph) Exec(context.Context, *ExecRequest) (ExecResponse, error) {
+	// @todo
+	return nil, nil
+}
+
+func (g *Graph) Orphans() (oo Steps) {
+	for _, step := range g.steps {
+		if len(g.Parents(step)) > 0 {
+			continue
+		}
+
+		oo = append(oo, step)
+	}
+
+	return
+}

server/pkg/wfexec/graph_test.go

@@ -0,0 +1,34 @@
+package wfexec
+
+import (
+	"context"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+type (
+	wfTestStep struct {
+		StepIdentifier
+		name string
+	}
+)
+
+func (wfTestStep) Exec(context.Context, *ExecRequest) (ExecResponse, error) {
+	return nil, nil
+}
+
+func TestWorkflow(t *testing.T) {
+	var (
+		req = require.New(t)
+		wf  = NewGraph()
+
+		s          = &wfTestStep{name: "s1"}
+		c1, c2, c3 = &wfTestStep{name: "c1"}, &wfTestStep{name: "c2"}, &wfTestStep{name: "c3"}
+	)
+
+	wf.AddStep(s, c1, c2, c3)
+	req.Equal(wf.Children(s), Steps{c1, c2, c3})
+	req.Equal(wf.Parents(c1), Steps{s})
+	req.Equal(wf.Parents(c2), Steps{s})
+	req.Equal(wf.Parents(c3), Steps{s})
+}

server/pkg/wfexec/iterator.go

@@ -0,0 +1,116 @@
+package wfexec
+
+import (
+	"context"
+
+	"github.com/cortezaproject/corteza/server/pkg/expr"
+)
+
+type (
+	// Iterator can be returned from Exec fn as ExecResponse
+	//
+	// It helps session's exec fn() to properly navigate through graph
+	// by calling is/break/iterator/next function
+	Iterator interface {
+		// Is the given step this iterator step
+		Is(Step) bool
+
+		// Initialize iterator
+		Start(context.Context, *expr.Vars) error
+
+		// Break fn is called when loop is forcefully broken
+		Break() Step
+
+		Iterator() Step
+
+		// Next is called before each iteration and returns
+		// 1st step of the iteration branch and variables that are added to the scope
+		Next(context.Context, *expr.Vars) (Step, *expr.Vars, error)
+	}
+
+	ResultEvaluator interface {
+		EvalResults(ctx context.Context, results *expr.Vars) (out *expr.Vars, err error)
+	}
+
+	IteratorHandler interface {
+		Start(context.Context, *expr.Vars) error
+		More(context.Context, *expr.Vars) (bool, error)
+		Next(context.Context, *expr.Vars) (*expr.Vars, error)
+	}
+
+	// Handles communication between Session's exec() fn and iterator handler
+	genericIterator struct {
+		iter, next, exit Step
+
+		h IteratorHandler
+	}
+)
+
+const (
+	DefaultMaxIteratorBufferSize uint = 1000
+)
+
+var (
+	MaxIteratorBufferSize uint = DefaultMaxIteratorBufferSize
+)
+
+// GenericIterator creates a wrapper around IteratorHandler and
+// returns genericIterator that implements Iterator interface
+func GenericIterator(iter, next, exit Step, h IteratorHandler) Iterator {
+	return &genericIterator{
+		iter: iter,
+		next: next,
+		exit: exit,
+		h:    h,
+	}
+}
+
+func (i *genericIterator) Is(s Step) bool                                { return i.iter == s }
+func (i *genericIterator) Start(ctx context.Context, s *expr.Vars) error { return i.h.Start(ctx, s) }
+func (i *genericIterator) Break() Step                                   { return i.exit }
+func (i *genericIterator) Iterator() Step                                { return i.iter }
+
+// Next calls More and Next functions on iterator handler.
+//
+// If iterator step (iter field) implements ResultEvaluator it calls
+// EvalResults on it before returning it. If iterator step does not implement it,
+// results are omitted.
+func (i *genericIterator) Next(ctx context.Context, scope *expr.Vars) (next Step, out *expr.Vars, err error) {
+	var (
+		more    bool
+		results *expr.Vars
+	)
+	if more, err = i.h.More(ctx, scope); err != nil || !more {
+		return
+	}
+
+	if results, err = i.h.Next(ctx, scope); err != nil {
+		return
+	}
+
+	if re, is := i.iter.(ResultEvaluator); is {
+		if out, err = re.EvalResults(ctx, results); err != nil {
+			return
+		}
+	}
+
+	next = i.next
+
+	return
+}
+
+func GenericResourceNextCheck(useLimit bool, ptr, buffSize, total, limit uint, hasMore bool) bool {
+	// if we can go more (inverted)...
+	// Treat limit = 0 the same as no limit (consistent with other bits of the system)
+	if (useLimit && limit != 0) && ptr+total >= limit {
+		return false
+	}
+
+	// if we have some buffer left...
+	if ptr < buffSize {
+		return true
+	}
+
+	// if we can get more...
+	return hasMore
+}

server/pkg/wfexec/main_test.go

@@ -0,0 +1,10 @@
+package wfexec
+
+import (
+	"github.com/cortezaproject/corteza/server/pkg/cli"
+	"github.com/cortezaproject/corteza/server/pkg/id"
+)
+
+func init() {
+	id.Init(cli.Context())
+}

server/pkg/wfexec/prompt.go

@@ -0,0 +1,68 @@
+package wfexec
+
+import (
+	"time"
+
+	"github.com/cortezaproject/corteza/server/pkg/expr"
+)
+
+type (
+	prompted struct {
+		// when !nul, assuming waiting for input we're waiting for input
+		payload *expr.Vars
+
+		// user to be prompted
+		ownerId uint64
+
+		// state to be resumed
+		state *State
+
+		sent bool
+
+		// prompt reference; something client can use
+		// for orientation, what kind of prompt is expected
+		ref string
+	}
+
+	PendingPrompt struct {
+		Ref       string     `json:"ref"`
+		SessionID uint64     `json:"sessionID,string"`
+		CreatedAt time.Time  `json:"createdAt"`
+		StateID   uint64     `json:"stateID,string"`
+		Payload   *expr.Vars `json:"payload"`
+		OwnerId   uint64     `json:"-"`
+
+		Original *prompted `json:"-"`
+	}
+
+	ResumedPrompt struct {
+		StateID uint64 `json:"stateID,string"`
+		OwnerId uint64 `json:"-"`
+	}
+)
+
+func Prompt(ownerId uint64, ref string, payload *expr.Vars) *prompted {
+	return &prompted{payload: payload, ref: ref, ownerId: ownerId}
+}
+
+func (p *prompted) toPending() *PendingPrompt {
+	return &PendingPrompt{
+		Ref:       p.ref,
+		CreatedAt: p.state.created,
+		StateID:   p.state.stateId,
+		Payload:   p.payload,
+		OwnerId:   p.ownerId,
+		Original:  p,
+	}
+}
+
+func (p *prompted) toResumed() *ResumedPrompt {
+	return &ResumedPrompt{
+		StateID: p.state.stateId,
+		OwnerId: p.ownerId,
+	}
+}
+
+func (p *prompted) MarkSent() {
+	p.sent = true
+}

server/pkg/wfexec/response.go

@@ -0,0 +1,61 @@
+package wfexec
+
+import (
+	"github.com/cortezaproject/corteza/server/pkg/expr"
+	"time"
+)
+
+type (
+	ExecResponse interface{}
+	partial      struct{}
+
+	errHandler struct {
+		handler Step
+		results *expr.Vars
+	}
+
+	termination struct{}
+
+	delayed struct {
+		// when not nil, assuming delayed
+		resumeAt time.Time
+
+		// state to be resumed
+		state *State
+	}
+
+	// when session is resumed from a delay we'll replace
+	// delay step on state with the a generic step that will return resumed{}
+	resumed struct{}
+)
+
+func Delay(until time.Time) *delayed {
+	return &delayed{resumeAt: until}
+}
+
+func Resume() *resumed {
+	return &resumed{}
+}
+
+func ErrorHandler(h Step, results *expr.Vars) *errHandler {
+	return &errHandler{handler: h, results: results}
+}
+
+func Termination() *termination {
+	return &termination{}
+}
+
+type (
+	loopBreak    struct{}
+	loopContinue struct{}
+)
+
+// joined is returned by the join gateway; it carries the merged scope and the
+// names of variables the joined paths changed
+type joined struct {
+	scope   *expr.Vars
+	changed []string
+}
+
+func LoopBreak() *loopBreak       { return &loopBreak{} }
+func LoopContinue() *loopContinue { return &loopContinue{} }

server/pkg/wfexec/session_test.go

@@ -0,0 +1,312 @@
+package wfexec
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/pkg/expr"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/atomic"
+)
+
+type (
+	sesTestStep struct {
+		StepIdentifier
+		name string
+		exec func(context.Context, *ExecRequest) (ExecResponse, error)
+	}
+
+	sesTestTemporal struct {
+		StepIdentifier
+		delay time.Duration
+		until time.Time
+	}
+)
+
+var (
+	// used for testing to produce lower numbers that are easier to inspect and compare
+	testID = atomic.NewUint64(0)
+)
+
+func (s *sesTestStep) Exec(ctx context.Context, r *ExecRequest) (ExecResponse, error) {
+	if s.exec != nil {
+		return s.exec(ctx, r)
+	}
+
+	var (
+		args = &struct {
+			Path    string
+			Counter int64
+		}{}
+	)
+
+	if err := r.Scope.Decode(args); err != nil {
+		return nil, err
+	}
+
+	return expr.NewVars(map[string]interface{}{
+		"counter": args.Counter + 1,
+		"path":    args.Path + "/" + s.name,
+		s.name:    "executed",
+	})
+}
+
+func (s *sesTestTemporal) Exec(ctx context.Context, r *ExecRequest) (ExecResponse, error) {
+	if s.until.IsZero() {
+		s.until = now().Add(s.delay)
+	}
+
+	if now().Before(s.until) {
+		return Delay(s.until), nil
+	}
+
+	return expr.NewVars(map[string]interface{}{
+		"waitForMoment": "executed",
+	})
+}
+
+func TestSession_TwoStepWorkflow(t *testing.T) {
+	var (
+		ctx = context.Background()
+		req = require.New(t)
+		wf  = NewGraph()
+		ses = NewSession(ctx, wf)
+
+		s1 = &sesTestStep{name: "s1"}
+		s2 = &sesTestStep{name: "s2"}
+
+		scope = &expr.Vars{}
+	)
+
+	scope.Set("two", 1)
+	scope.Set("three", 1)
+
+	wf.AddStep(s1, s2) // 1st execute s1 then s2
+	req.NoError(ses.Exec(ctx, s1, scope))
+	req.NoError(ses.Wait(ctx))
+	req.NoError(ses.Error())
+	req.NotNil(ses.Result())
+	req.Equal("/s1/s2", expr.Must(expr.Select(ses.Result(), "path")).Get())
+}
+
+func TestSession_SplitAndMerge(t *testing.T) {
+	var (
+		ctx = context.Background()
+		req = require.New(t)
+		wf  = NewGraph()
+		ses = NewSession(ctx, wf, SetDumpStacktraceOnPanic(true))
+
+		start  = &sesTestStep{name: "start"}
+		split1 = &sesTestStep{name: "split1"}
+		split2 = &sesTestStep{name: "split2"}
+		split3 = &sesTestStep{name: "split3"}
+
+		end = JoinGateway(split1, split2, split3)
+	)
+
+	wf.AddStep(start, split1, split2, split3)
+	wf.AddStep(split1, end)
+	wf.AddStep(split2, end)
+	wf.AddStep(split3, end)
+	ses.Exec(ctx, start, nil)
+	ses.Wait(ctx)
+	req.True(ses.Idle())
+	req.NoError(ses.Error())
+	req.NotNil(ses.Result())
+	// split3 only!
+	req.Equal("/start/split3", expr.Must(expr.Select(ses.Result(), "path")).Get())
+	req.Contains(ses.Result().Dict(), "split1")
+	req.Contains(ses.Result().Dict(), "split2")
+	req.Contains(ses.Result().Dict(), "split3")
+}
+
+func TestSession_Delays(t *testing.T) {
+	t.SkipNow()
+	var (
+		// how fast we want to go (lower = faster)
+		//
+		unit  = time.Millisecond
+		delay = unit * 3
+
+		ctx = context.Background()
+		req = require.New(t)
+		wf  = NewGraph()
+		ses = NewSession(ctx, wf,
+			// for testing we need much shorter worker intervals
+			SetWorkerIntervalSuspended(unit),
+		)
+
+		start = &sesTestStep{name: "start"}
+
+		waitForMoment = &sesTestTemporal{delay: delay}
+
+		waitForInputStateId atomic.Uint64
+		waitForInput        = &sesTestStep{name: "waitForInput", exec: func(ctx context.Context, r *ExecRequest) (ExecResponse, error) {
+			if !r.Input.Has("input") {
+				waitForInputStateId.Store(r.StateID)
+				return Prompt(0, "", nil), nil
+			}
+
+			out := &expr.Vars{}
+			_ = out.Set("waitForInput", "executed")
+			r.Input.Copy(out, "input")
+
+			return out, nil
+		}}
+	)
+
+	ctx, cancelFn := context.WithTimeout(ctx, time.Second*5)
+	defer cancelFn()
+
+	wf.AddStep(start, waitForMoment)
+	wf.AddStep(waitForMoment, waitForInput)
+
+	req.NoError(ses.Exec(ctx, start, nil))
+
+	// wait-for-moment step needs to be executed before we can resume wait-for-input
+	req.NoError(ses.Wait(ctx))
+	time.Sleep(delay + unit)
+	req.NotZero(waitForInputStateId.Load())
+
+	// should not be completed yet...
+	req.True(ses.Idle())
+	req.True(ses.Delayed())
+
+	// push in the input
+	input := &expr.Vars{}
+	input.Set("inout", "foo")
+	_, err := ses.Resume(ctx, waitForInputStateId.Load(), input)
+	req.NoError(err)
+
+	req.False(ses.Delayed())
+	req.NoError(ses.Wait(ctx))
+	time.Sleep(2 * unit)
+
+	// should not be completed yet...
+	req.True(ses.Idle())
+	req.NoError(ses.Error())
+	req.NotNil(ses.Result())
+	req.Contains(ses.Result().Dict(), "waitForMoment")
+	req.Contains(ses.Result().Dict(), "waitForInput")
+	req.Equal("foo", expr.Must(expr.Select(ses.Result(), "input")).Get())
+}
+
+func TestSession_ErrHandler(t *testing.T) {
+	var (
+		ctx = context.Background()
+		req = require.New(t)
+		wf  = NewGraph()
+		ses = NewSession(
+			ctx,
+			wf,
+
+			// enable if you need to see what is going on
+			// SetLogger(logger.MakeDebugLogger()),
+
+			// enable if you need to see what is going on
+			// SetHandler(func(status SessionStatus, state *State, session *Session) {
+			//	if state.step != nil {
+			//		println(state.step.(*sesTestStep).name)
+			//	}
+			// }),
+		)
+
+		cb_1_1 = &sesTestStep{name: "catch-branch-1-1"}
+		cb_1_2 = &sesTestStep{name: "catch-branch-1-2"}
+		tb_1_1 = &sesTestStep{name: "try-branch-1-1"}
+
+		eh_1 = &sesTestStep{name: "err-handler", exec: func(ctx context.Context, request *ExecRequest) (ExecResponse, error) {
+			return ErrorHandler(cb_1_1, nil), nil
+		}}
+		er_1 = &sesTestStep{name: "err-raiser", exec: func(ctx context.Context, request *ExecRequest) (ExecResponse, error) {
+			return nil, fmt.Errorf("would-be-handled-error")
+		}}
+
+		cb_2_1 = &sesTestStep{name: "catch-branch-2-1"}
+		cb_2_2 = &sesTestStep{name: "catch-branch-2-2"}
+		tb_2_1 = &sesTestStep{name: "try-branch-2-1"}
+
+		eh_2 = &sesTestStep{name: "err-handler", exec: func(ctx context.Context, request *ExecRequest) (ExecResponse, error) {
+			return ErrorHandler(cb_2_1, nil), nil
+		}}
+		er_2 = &sesTestStep{name: "err-raiser", exec: func(ctx context.Context, request *ExecRequest) (ExecResponse, error) {
+			return nil, fmt.Errorf("would-be-handled-error")
+		}}
+	)
+
+	wf.AddStep(eh_1, tb_1_1)   // error handling step (entrypoint!)
+	wf.AddStep(tb_1_1)         // add try step
+	wf.AddStep(tb_1_1, er_1)   // add  error raising step right after 1st step in try branch
+	wf.AddStep(cb_1_1, cb_1_2) // catch branch step 1 & 2
+
+	wf.AddStep(cb_1_2, eh_2)   // 2nd error handling step right after 1st catch branch
+	wf.AddStep(eh_2, tb_2_1)   // step in try branch
+	wf.AddStep(tb_2_1, er_2)   // 2nd error raising step on 2nd try branch
+	wf.AddStep(cb_2_1, cb_2_2) // 2nd catch branch step 1 & 2
+
+	req.NoError(ses.Exec(ctx, eh_1, nil))
+
+	req.NoError(ses.Wait(ctx))
+
+	req.Equal(
+		"/try-branch-1-1/catch-branch-1-1/catch-branch-1-2/try-branch-2-1/catch-branch-2-1/catch-branch-2-2",
+		ses.Result().Dict()["path"],
+	)
+}
+
+func TestSession_ExecStepWithParents(t *testing.T) {
+	var (
+		ctx = context.Background()
+		req = require.New(t)
+		wf  = NewGraph()
+		ses = NewSession(ctx, wf)
+
+		p = &sesTestStep{name: "p"}
+		c = &sesTestStep{name: "c"}
+	)
+
+	wf.AddStep(p, c)
+
+	req.Equal(SessionActive, ses.Status())
+	req.Error(ses.Exec(ctx, c, nil))
+	req.Error(ses.Wait(ctx))
+	req.Equal(SessionFailed, ses.Status())
+}
+
+func bmSessionSimpleStepSequence(c uint64, b *testing.B) {
+	var (
+		ctx = context.Background()
+		g   = NewGraph()
+		err error
+	)
+
+	for i := uint64(1); i <= c; i++ {
+		s := &sesTestStep{name: "start"}
+		s.SetID(i)
+		g.AddStep(s)
+		if i > 1 {
+			g.AddParent(s, g.StepByID(i-1))
+		}
+	}
+
+	b.StartTimer()
+	for n := 0; n < b.N; n++ {
+		ses := NewSession(ctx, g)
+		if err = ses.Exec(ctx, g.StepByID(1), nil); err != nil {
+			b.Fatal(err.Error())
+		}
+
+		ses.Wait(ctx)
+	}
+	b.StopTimer()
+}
+
+func BenchmarkSessionSimple1StepSequence(b *testing.B)       { bmSessionSimpleStepSequence(1, b) }
+func BenchmarkSessionSimple10StepSequence(b *testing.B)      { bmSessionSimpleStepSequence(10, b) }
+func BenchmarkSessionSimple100StepSequence(b *testing.B)     { bmSessionSimpleStepSequence(100, b) }
+func BenchmarkSessionSimple1000StepSequence(b *testing.B)    { bmSessionSimpleStepSequence(1000, b) }
+func BenchmarkSessionSimple10000StepSequence(b *testing.B)   { bmSessionSimpleStepSequence(10000, b) }
+func BenchmarkSessionSimple100000StepSequence(b *testing.B)  { bmSessionSimpleStepSequence(100000, b) }
+func BenchmarkSessionSimple1000000StepSequence(b *testing.B) { bmSessionSimpleStepSequence(1000000, b) }

server/pkg/wfexec/state.go

@@ -0,0 +1,284 @@
+package wfexec
+
+import (
+	"sync"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/pkg/auth"
+	"github.com/cortezaproject/corteza/server/pkg/expr"
+)
+
+type (
+
+	// state holds information about Session ID
+	State struct {
+		created   time.Time
+		completed *time.Time
+
+		// state identifier
+		stateId uint64
+
+		// who's running this?
+		owner auth.Identifiable
+
+		// Session identifier
+		sessionId uint64
+
+		// parent, parent step
+		parent Step
+
+		// current step
+		step Step
+
+		// next steps
+		next Steps
+
+		// step error (if any)
+		err error
+
+		// input variables that were sent to resume the session
+		input *expr.Vars
+
+		// scope
+		scope *expr.Vars
+
+		// step execution results
+		results *expr.Vars
+
+		// error handling step
+		errHandler Step
+
+		// error handler result variable names
+		//
+		// Holds the error/errorMessage/errorStepID to variable-name mapping
+		// declared on the error handling step. Kept apart from results
+		// (previous step outputs) so that a normal step running between the
+		// handler and the actual error does not overwrite it.
+		errHandlerResults *expr.Vars
+
+		// error handled flag, this gets restarted on every new state!
+		errHandled bool
+
+		loops []Iterator
+
+		// names of variables written on this path since it branched off
+		//
+		// Used by the join gateway to merge only what each parallel branch
+		// actually changed instead of whole scopes; reset for every path
+		// when execution branches.
+		dirty map[string]bool
+
+		action string
+	}
+)
+
+func NewState(ses *Session, owner auth.Identifiable, caller, current Step, scope *expr.Vars) *State {
+	return &State{
+		stateId:   nextID(),
+		owner:     owner,
+		sessionId: ses.id,
+		created:   *now(),
+		parent:    caller,
+		step:      current,
+		scope:     scope,
+
+		loops: make([]Iterator, 0, 4),
+	}
+}
+
+func FinalState(ses *Session, scope *expr.Vars) *State {
+	return &State{
+		stateId:   nextID(),
+		sessionId: ses.id,
+		created:   *now(),
+		completed: now(),
+		scope:     scope,
+	}
+}
+
+func (s State) Next(current Step, scope *expr.Vars) *State {
+	return &State{
+		stateId:           nextID(),
+		owner:             s.owner,
+		sessionId:         s.sessionId,
+		parent:            s.step,
+		errHandler:        s.errHandler,
+		errHandlerResults: s.errHandlerResults,
+		results:           s.results,
+		loops:             s.loops,
+		dirty:             s.dirty,
+
+		step:  current,
+		scope: scope,
+	}
+}
+
+// NextBranch returns next state for one of the parallel paths
+//
+// Variables written before the branching point are not part of the branch's
+// changes, so the set of written variables starts empty.
+func (s State) NextBranch(current Step, scope *expr.Vars) *State {
+	st := s.Next(current, scope)
+	st.dirty = nil
+	return st
+}
+
+// markDirty records variables written by the current step
+func (s *State) markDirty(vv *expr.Vars) {
+	if vv.IsEmpty() {
+		return
+	}
+
+	var nn []string
+	_ = vv.Each(func(k string, _ expr.TypedValue) error {
+		nn = append(nn, k)
+		return nil
+	})
+
+	s.markDirtyNames(nn...)
+}
+
+// markDirtyNames records variables written directly to the scope
+func (s *State) markDirtyNames(nn ...string) {
+	if len(nn) == 0 {
+		return
+	}
+
+	if s.dirty == nil {
+		s.dirty = make(map[string]bool)
+	} else {
+		// state's set is shared with the states it spawned; copy before write
+		dirty := make(map[string]bool, len(s.dirty)+1)
+		for k := range s.dirty {
+			dirty[k] = true
+		}
+		s.dirty = dirty
+	}
+
+	for _, n := range nn {
+		s.dirty[n] = true
+	}
+}
+
+func (s State) MakeRequest() *ExecRequest {
+	return &ExecRequest{
+		SessionID: s.sessionId,
+		StateID:   s.stateId,
+		Scope:     s.scope,
+		Input:     s.input,
+		Results:   s.results,
+		Parent:    s.parent,
+		dirty:     s.dirty,
+	}
+}
+
+func (s *State) newLoop(i Iterator) {
+	s.loops = append(s.loops, i)
+}
+
+// ends loop and returns step that leads out of the loop
+func (s *State) loopEnd() (out Steps) {
+	l := len(s.loops) - 1
+	if l < 0 {
+		panic("not inside a loop")
+	}
+
+	out = Steps{s.loops[l].Break()}
+	s.loops = s.loops[:l]
+	return
+}
+
+func (s State) loopCurr() Iterator {
+	l := len(s.loops)
+	if l > 0 {
+		return s.loops[l-1]
+	}
+
+	return nil
+}
+
+// MakeLightFrame builds a frame that describes the step without snapshotting
+// any of the state's variables
+//
+// Cloning the scope is what makes stacktrace collection expensive, and a frame
+// only needs its variables when someone is going to read them back.
+func (s State) MakeLightFrame() *Frame {
+	f := &Frame{
+		CreatedAt: s.created,
+		SessionID: s.sessionId,
+		StateID:   s.stateId,
+		NextSteps: s.next.IDs(),
+		Action:    s.action,
+	}
+
+	s.describe(f)
+	return f
+}
+
+// describe fills in the parts of a frame that do not require cloning
+func (s State) describe(f *Frame) {
+	if s.err != nil {
+		f.Error = s.err.Error()
+	}
+
+	if s.step != nil {
+		f.StepID = s.step.ID()
+	}
+
+	if s.parent != nil {
+		f.ParentID = s.parent.ID()
+	}
+
+	if s.completed != nil {
+		f.StepTime = uint(s.completed.Sub(s.created) / time.Millisecond)
+	}
+}
+
+// MakeFrame builds a frame with a full snapshot of the state's variables
+func (s State) MakeFrame() *Frame {
+	var (
+		// might not be the most optimal way but we need to
+		// un-reference scope, input, result variables
+		unref = func(vars *expr.Vars) *expr.Vars {
+			aux, err := vars.Clone()
+			if err != nil {
+				return expr.EmptyVars()
+			}
+
+			// Since we're cloning vars, this will always hold
+			return aux.(*expr.Vars)
+		}
+	)
+
+	f := s.MakeLightFrame()
+
+	var wg sync.WaitGroup
+	wg.Add(3)
+
+	go func() {
+		f.Input = unref(s.input)
+		wg.Done()
+	}()
+
+	go func() {
+		f.Scope = unref(s.scope)
+		wg.Done()
+	}()
+
+	go func() {
+		f.Results = unref(s.results)
+		wg.Done()
+	}()
+
+	wg.Wait()
+
+	return f
+}
+
+func (s *State) Error() string {
+	if s.err == nil {
+		return ""
+	}
+
+	return s.err.Error()
+}

server/pkg/wfexec/step.go

@@ -0,0 +1,34 @@
+package wfexec
+
+import (
+	"context"
+)
+
+type (
+	Steps []Step
+	Step  interface {
+		ID() uint64
+		SetID(uint64)
+		Exec(context.Context, *ExecRequest) (ExecResponse, error)
+	}
+
+	StepIdentifier struct{ id uint64 }
+
+	execFn func(context.Context, *ExecRequest) (ExecResponse, error)
+
+	genericStep struct {
+		StepIdentifier
+		fn execFn
+	}
+)
+
+func (i *StepIdentifier) ID() uint64      { return i.id }
+func (i *StepIdentifier) SetID(id uint64) { i.id = id }
+
+func NewGenericStep(fn execFn) *genericStep {
+	return &genericStep{fn: fn}
+}
+
+func (g *genericStep) Exec(ctx context.Context, r *ExecRequest) (ExecResponse, error) {
+	return g.fn(ctx, r)
+}