snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/automation/envoy/store_decode.gen.go

@@ -0,0 +1,394 @@
+package envoy
+
+// This file is auto-generated.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+//
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/automation/types"
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/pkg/id"
+	"github.com/cortezaproject/corteza/server/store"
+	"github.com/pkg/errors"
+)
+
+type (
+	// StoreDecoder is responsible for generating Envoy nodes from already stored
+	// resources which can then be managed by Envoy and imported via an encoder.
+	StoreDecoder struct{}
+
+	filterWrap struct {
+		rt string
+		f  envoyx.ResourceFilter
+	}
+)
+
+const (
+	paramsKeyStorer = "storer"
+	paramsKeyDAL    = "dal"
+)
+
+var (
+	// @todo temporary fix to make unused pkg/id not throw errors
+	_ = id.Next
+)
+
+// Decode returns a set of envoy nodes based on the provided params
+//
+// StoreDecoder expects the DecodeParam of `storer` and `dal` which conform
+// to the store.Storer and dal.FullService interfaces.
+func (d StoreDecoder) Decode(ctx context.Context, p envoyx.DecodeParams) (out envoyx.NodeSet, err error) {
+	// @todo we can optionally not require them based on what we're doing
+	s, err := d.getStorer(p)
+	if err != nil {
+		return
+	}
+	dl, err := d.getDal(p)
+	if err != nil {
+		return
+	}
+
+	return d.decode(ctx, s, dl, p)
+}
+
+func (d StoreDecoder) decode(ctx context.Context, s store.Storer, dl dal.FullService, p envoyx.DecodeParams) (out envoyx.NodeSet, err error) {
+	// Preprocessing and basic filtering (to omit what this decoder can't handle)
+	wrappedFilters := d.prepFilters(p.Filter)
+
+	// Get all scoped nodes
+	scopedNodes, err := d.getScopeNodes(ctx, s, dl, wrappedFilters)
+	if err != nil {
+		return
+	}
+
+	// Get all reference nodes
+	refNodes, refRefs, err := d.getReferenceNodes(ctx, s, dl, wrappedFilters)
+	if err != nil {
+		return
+	}
+
+	// Process filters to get the envoy nodes
+	err = func() (err error) {
+		var aux envoyx.NodeSet
+		for i, wf := range wrappedFilters {
+			switch wf.rt {
+			case types.WorkflowResourceType:
+				aux, err = d.decodeWorkflow(ctx, s, dl, d.makeWorkflowFilter(scopedNodes[i], refNodes[i], wf.f))
+				if err != nil {
+					return
+				}
+				for _, a := range aux {
+					a.Identifiers = a.Identifiers.Merge(wf.f.Identifiers)
+					a.References = envoyx.MergeRefs(a.References, refRefs[i])
+				}
+				out = append(out, aux...)
+
+			case types.TriggerResourceType:
+				aux, err = d.decodeTrigger(ctx, s, dl, d.makeTriggerFilter(scopedNodes[i], refNodes[i], wf.f))
+				if err != nil {
+					return
+				}
+				for _, a := range aux {
+					a.Identifiers = a.Identifiers.Merge(wf.f.Identifiers)
+					a.References = envoyx.MergeRefs(a.References, refRefs[i])
+				}
+				out = append(out, aux...)
+
+			default:
+				aux, err = d.extendDecoder(ctx, s, dl, p, wf.rt, refNodes[i], wf.f)
+				if err != nil {
+					return
+				}
+				for _, a := range aux {
+					a.Identifiers = a.Identifiers.Merge(wf.f.Identifiers)
+					a.References = envoyx.MergeRefs(a.References, refRefs[i])
+				}
+				out = append(out, aux...)
+			}
+		}
+		return
+	}()
+	if err != nil {
+		err = errors.Wrap(err, "failed to decode filters")
+		return
+	}
+
+	return
+} // // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource workflow
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (d StoreDecoder) decodeWorkflow(ctx context.Context, s store.Storer, dl dal.FullService, f types.WorkflowFilter) (out envoyx.NodeSet, err error) {
+	// @todo this might need to be improved.
+	//       Currently, no resource is vast enough to pose a problem.
+	rr, _, err := store.SearchAutomationWorkflows(ctx, s, f)
+	if err != nil {
+		return
+	}
+
+	for _, r := range rr {
+		var n *envoyx.Node
+		n, err = WorkflowToEnvoyNode(r)
+		if err != nil {
+			return
+		}
+		out = append(out, n)
+	}
+
+	aux, err := d.extendedWorkflowDecoder(ctx, s, dl, f, out)
+	if err != nil {
+		return
+	}
+	out = append(out, aux...)
+
+	return
+}
+
+func WorkflowToEnvoyNode(r *types.Workflow) (node *envoyx.Node, err error) {
+	// Identifiers
+	ii := envoyx.MakeIdentifiers(
+		r.Handle,
+		r.ID,
+	)
+
+	// Handle references
+	// Omit any non-defined values
+	refs := map[string]envoyx.Ref{}
+	if r.CreatedBy > 0 {
+		refs["CreatedBy"] = envoyx.Ref{
+			ResourceType: "corteza::system:user",
+			Identifiers:  envoyx.MakeIdentifiers(r.CreatedBy),
+		}
+	}
+	if r.DeletedBy > 0 {
+		refs["DeletedBy"] = envoyx.Ref{
+			ResourceType: "corteza::system:user",
+			Identifiers:  envoyx.MakeIdentifiers(r.DeletedBy),
+		}
+	}
+	if r.OwnedBy > 0 {
+		refs["OwnedBy"] = envoyx.Ref{
+			ResourceType: "corteza::system:user",
+			Identifiers:  envoyx.MakeIdentifiers(r.OwnedBy),
+		}
+	}
+	if r.RunAs > 0 {
+		refs["RunAs"] = envoyx.Ref{
+			ResourceType: "corteza::system:user",
+			Identifiers:  envoyx.MakeIdentifiers(r.RunAs),
+		}
+	}
+	if r.UpdatedBy > 0 {
+		refs["UpdatedBy"] = envoyx.Ref{
+			ResourceType: "corteza::system:user",
+			Identifiers:  envoyx.MakeIdentifiers(r.UpdatedBy),
+		}
+	}
+
+	var scope envoyx.Scope
+
+	node = &envoyx.Node{
+		Resource: r,
+
+		ResourceType: types.WorkflowResourceType,
+		Identifiers:  ii,
+		References:   refs,
+		Scope:        scope,
+	}
+	return
+}
+
+// Resource should define a custom filter builder
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource trigger
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (d StoreDecoder) decodeTrigger(ctx context.Context, s store.Storer, dl dal.FullService, f types.TriggerFilter) (out envoyx.NodeSet, err error) {
+	// @todo this might need to be improved.
+	//       Currently, no resource is vast enough to pose a problem.
+	rr, _, err := store.SearchAutomationTriggers(ctx, s, f)
+	if err != nil {
+		return
+	}
+
+	for _, r := range rr {
+		var n *envoyx.Node
+		n, err = TriggerToEnvoyNode(r)
+		if err != nil {
+			return
+		}
+		out = append(out, n)
+	}
+
+	return
+}
+
+func TriggerToEnvoyNode(r *types.Trigger) (node *envoyx.Node, err error) {
+	// Identifiers
+	ii := envoyx.MakeIdentifiers(
+		r.ID,
+	)
+
+	// Handle references
+	// Omit any non-defined values
+	refs := map[string]envoyx.Ref{}
+	if r.CreatedBy > 0 {
+		refs["CreatedBy"] = envoyx.Ref{
+			ResourceType: "corteza::system:user",
+			Identifiers:  envoyx.MakeIdentifiers(r.CreatedBy),
+		}
+	}
+	if r.DeletedBy > 0 {
+		refs["DeletedBy"] = envoyx.Ref{
+			ResourceType: "corteza::system:user",
+			Identifiers:  envoyx.MakeIdentifiers(r.DeletedBy),
+		}
+	}
+	if r.OwnedBy > 0 {
+		refs["OwnedBy"] = envoyx.Ref{
+			ResourceType: "corteza::system:user",
+			Identifiers:  envoyx.MakeIdentifiers(r.OwnedBy),
+		}
+	}
+	if r.UpdatedBy > 0 {
+		refs["UpdatedBy"] = envoyx.Ref{
+			ResourceType: "corteza::system:user",
+			Identifiers:  envoyx.MakeIdentifiers(r.UpdatedBy),
+		}
+	}
+	if r.WorkflowID > 0 {
+		refs["WorkflowID"] = envoyx.Ref{
+			ResourceType: "corteza::automation:workflow",
+			Identifiers:  envoyx.MakeIdentifiers(r.WorkflowID),
+		}
+	}
+
+	var scope envoyx.Scope
+
+	node = &envoyx.Node{
+		Resource: r,
+
+		ResourceType: types.TriggerResourceType,
+		Identifiers:  ii,
+		References:   refs,
+		Scope:        scope,
+	}
+	return
+}
+
+// Resource should define a custom filter builder
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Utilities
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (d StoreDecoder) getStorer(p envoyx.DecodeParams) (s store.Storer, err error) {
+	aux, ok := p.Params[paramsKeyStorer]
+	if ok {
+		s, ok = aux.(store.Storer)
+		if ok {
+			return
+		}
+	}
+
+	err = errors.Errorf("store decoder expects a storer conforming to store.Storer interface")
+	return
+}
+
+func (d StoreDecoder) getDal(p envoyx.DecodeParams) (dl dal.FullService, err error) {
+	aux, ok := p.Params[paramsKeyDAL]
+	if ok {
+		dl, ok = aux.(dal.FullService)
+		if ok {
+			return
+		}
+	}
+
+	err = errors.Errorf("store decoder expects a DAL conforming to dal.FullService interface")
+	return
+}
+
+func (d StoreDecoder) prepFilters(ff map[string]envoyx.ResourceFilter) (out []filterWrap) {
+	out = make([]filterWrap, 0, len(ff))
+	for rt, f := range ff {
+		// Handle resources that don't belong to this decoder
+		if !strings.HasPrefix(rt, "corteza::automation") {
+			continue
+		}
+
+		out = append(out, filterWrap{rt: rt, f: f})
+	}
+
+	return
+}
+
+func (d StoreDecoder) getScopeNodes(ctx context.Context, s store.Storer, dl dal.FullService, ff []filterWrap) (scopes envoyx.NodeSet, err error) {
+	// Get all requested scopes
+	scopes = make(envoyx.NodeSet, len(ff))
+
+	// @note skipping scope logic since it's currently only supported within
+	//       Compose resources.
+
+	return
+}
+
+// getReferenceNodes returns all of the nodes referenced by the nodes defined by the filters
+//
+// The nodes are provided as a slice (the same order as the filters) and as a map for easier lookups.
+func (d StoreDecoder) getReferenceNodes(ctx context.Context, s store.Storer, dl dal.FullService, ff []filterWrap) (nodes []map[string]*envoyx.Node, refs []map[string]envoyx.Ref, err error) {
+	nodes = make([]map[string]*envoyx.Node, len(ff))
+	refs = make([]map[string]envoyx.Ref, len(ff))
+	err = func() (err error) {
+		for i, a := range ff {
+			if len(a.f.Refs) == 0 {
+				continue
+			}
+
+			auxr := make(map[string]*envoyx.Node, len(a.f.Refs))
+			auxa := make(map[string]envoyx.Ref)
+			for field, ref := range a.f.Refs {
+				f := ref.ResourceFilter()
+				aux, err := d.decode(ctx, s, dl, envoyx.DecodeParams{
+					Type:   envoyx.DecodeTypeStore,
+					Filter: f,
+				})
+				if err != nil {
+					return err
+				}
+
+				// @todo consider changing this.
+				//       Currently it's required because the .decode may return some
+				//       nested nodes as well.
+				//       Consider a flag or a new function.
+				aux = envoyx.NodesForResourceType(ref.ResourceType, aux...)
+				if len(aux) == 0 {
+					return fmt.Errorf("invalid reference %v", ref)
+				}
+				if len(aux) > 1 {
+					return fmt.Errorf("ambiguous reference: too many resources returned %v", a.f)
+				}
+
+				auxr[field] = aux[0]
+				auxa[field] = aux[0].ToRef()
+			}
+
+			nodes[i] = auxr
+			refs[i] = auxa
+		}
+		return
+	}()
+	if err != nil {
+		err = errors.Wrap(err, "failed to decode node references")
+		return
+	}
+
+	return
+}

server/automation/envoy/store_decode.go

@@ -0,0 +1,64 @@
+package envoy
+
+import (
+	"context"
+
+	"github.com/cortezaproject/corteza/server/automation/types"
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/pkg/id"
+	"github.com/cortezaproject/corteza/server/store"
+)
+
+func (e StoreEncoder) prepare(ctx context.Context, p envoyx.EncodeParams, s store.Storer, rt string, nn envoyx.NodeSet) (err error) {
+	return
+}
+
+func (d StoreDecoder) extendDecoder(ctx context.Context, s store.Storer, dl dal.FullService, _ envoyx.DecodeParams, rt string, nodes map[string]*envoyx.Node, f envoyx.ResourceFilter) (out envoyx.NodeSet, err error) {
+	return
+}
+
+func (d StoreDecoder) makeWorkflowFilter(scope *envoyx.Node, refs map[string]*envoyx.Node, auxf envoyx.ResourceFilter) (out types.WorkflowFilter) {
+	out.Limit = auxf.Limit
+
+	ids, hh := auxf.Identifiers.IdentsAsStrings()
+	_ = ids
+	_ = hh
+
+	out.WorkflowID = ids
+
+	if len(hh) > 0 {
+		out.Handle = hh[0]
+	}
+
+	return
+}
+
+func (d StoreDecoder) makeTriggerFilter(scope *envoyx.Node, refs map[string]*envoyx.Node, auxf envoyx.ResourceFilter) (out types.TriggerFilter) {
+	out.Limit = auxf.Limit
+
+	ids, hh := auxf.Identifiers.Idents()
+	_ = ids
+	_ = hh
+
+	out.TriggerID = id.Strings(ids...)
+
+	return
+}
+
+func (d StoreDecoder) extendedWorkflowDecoder(ctx context.Context, s store.Storer, dl dal.FullService, f types.WorkflowFilter, base envoyx.NodeSet) (out envoyx.NodeSet, err error) {
+	for _, b := range base {
+		wf := b.Resource.(*types.Workflow)
+
+		filters, err := d.decodeTrigger(ctx, s, dl, types.TriggerFilter{
+			WorkflowID: id.Strings(wf.ID),
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		out = append(out, filters...)
+	}
+
+	return
+}

server/automation/envoy/store_encode.gen.go

@@ -0,0 +1,557 @@
+package envoy
+
+// This file is auto-generated.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+//
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	"github.com/cortezaproject/corteza/server/automation/types"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/pkg/expr"
+	"github.com/cortezaproject/corteza/server/pkg/id"
+	"github.com/cortezaproject/corteza/server/store"
+	"github.com/pkg/errors"
+)
+
+type (
+	// StoreEncoder is responsible for encoding Corteza resources into the
+	// database via the Storer or the DAL interface
+	//
+	// @todo consider having a different encoder for the DAL resources
+	StoreEncoder struct{}
+)
+
+// Prepare performs some initial processing on the resource before it can be encoded
+//
+// Preparation runs validation, default value initialization, matching with
+// already existing instances, ...
+//
+// The prepare function receives a set of nodes grouped by the resource type.
+// This enables some batching optimization and simplifications when it comes to
+// matching with existing resources.
+//
+// Prepare does not receive any placeholder nodes which are used solely
+// for dependency resolution.
+func (e StoreEncoder) Prepare(ctx context.Context, p envoyx.EncodeParams, rt string, nn envoyx.NodeSet) (err error) {
+	s, err := e.grabStorer(p)
+	if err != nil {
+		return
+	}
+
+	switch rt {
+	case types.WorkflowResourceType:
+		return e.prepareWorkflow(ctx, p, s, nn)
+
+	case types.TriggerResourceType:
+		return e.prepareTrigger(ctx, p, s, nn)
+	default:
+		return e.prepare(ctx, p, s, rt, nn)
+	}
+
+	return
+}
+
+// Encode encodes the given Corteza resources into the primary store
+//
+// Encoding should not do any additional processing apart from matching with
+// dependencies and runtime validation
+//
+// The Encode function is called for every resource type where the resource
+// appears at the root of the dependency tree.
+// All of the root-level resources for that resource type are passed into the function.
+// The encoding function must traverse the branches to encode all of the dependencies.
+//
+// This flow is used to simplify the flow of how resources are encoded into YAML
+// (and other documents) as well as to simplify batching.
+//
+// Encode does not receive any placeholder nodes which are used solely
+// for dependency resolution.
+func (e StoreEncoder) Encode(ctx context.Context, p envoyx.EncodeParams, rt string, nodes envoyx.NodeSet, tree envoyx.Traverser) (err error) {
+	s, err := e.grabStorer(p)
+	if err != nil {
+		return
+	}
+
+	switch rt {
+	case types.WorkflowResourceType:
+		return e.encodeWorkflows(ctx, p, s, nodes, tree)
+
+	case types.TriggerResourceType:
+		return e.encodeTriggers(ctx, p, s, nodes, tree)
+	default:
+		return e.encode(ctx, p, s, rt, nodes, tree)
+	}
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource workflow
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+// prepareWorkflow prepares the resources of the given type for encoding
+func (e StoreEncoder) prepareWorkflow(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
+	// Grab an index of already existing resources of this type
+	// @note since these resources should be fairly low-volume and existing for
+	//       a short time (and because we batch by resource type); fetching them all
+	//       into memory shouldn't hurt too much.
+	// @todo do some benchmarks and potentially implement some smarter check such as
+	//       a bloom filter or something similar.
+
+	// Get node scopes
+	scopedNodes, err := e.getScopeNodes(ctx, s, nn)
+	if err != nil {
+		err = errors.Wrap(err, "failed to get scope nodes")
+		return
+	}
+
+	// Initializing the index here (and using a hashmap) so it's not escaped to the heap
+	existing := make(map[int]types.Workflow, len(nn))
+	err = e.matchupWorkflows(ctx, s, existing, scopedNodes, nn)
+	if err != nil {
+		err = errors.Wrap(err, "failed to matchup existing Workflows")
+		return
+	}
+
+	for i, n := range nn {
+		if n.Resource == nil {
+			panic("unexpected state: cannot call prepareWorkflow with nodes without a defined Resource")
+		}
+
+		res, ok := n.Resource.(*types.Workflow)
+		if !ok {
+			panic("unexpected resource type: node expecting type of workflow")
+		}
+
+		existing, hasExisting := existing[i]
+
+		// Run expressions on the nodes
+		err = e.runEvals(ctx, hasExisting, n)
+		if err != nil {
+			return
+		}
+
+		if hasExisting {
+			// On existing, we don't need to re-do identifiers and references; simply
+			// changing up the internal resource is enough.
+			//
+			// In the future, we can pass down the tree and re-do the deps like that
+			switch n.Config.MergeAlg {
+			case envoyx.OnConflictPanic:
+				err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
+				return
+
+			case envoyx.OnConflictReplace:
+				// Replace; simple ID change should do the trick
+				res.ID = existing.ID
+
+			case envoyx.OnConflictSkip:
+				// Replace the node's resource with the fetched one
+				res = &existing
+
+				// @todo merging
+			}
+		} else {
+			// @todo actually a bottleneck. As per sonyflake docs, it can at most
+			//       generate up to 2**8 (256) IDs per 10ms in a single thread.
+			//       How can we improve this?
+			res.ID = id.Next()
+		}
+
+		// We can skip validation/defaults when the resource is overwritten by
+		// the one already stored (the panic one errors out anyway) since it
+		// should already be ok.
+		if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
+			err = e.setWorkflowDefaults(res)
+			if err != nil {
+				return err
+			}
+
+			err = e.validateWorkflow(res)
+			if err != nil {
+				return err
+			}
+		}
+
+		n.Resource = res
+	}
+
+	return
+}
+
+// encodeWorkflows encodes a set of resource into the database
+func (e StoreEncoder) encodeWorkflows(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
+	for _, n := range nn {
+		err = e.encodeWorkflow(ctx, p, s, n, tree)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// encodeWorkflow encodes the resource into the database
+func (e StoreEncoder) encodeWorkflow(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
+	// Grab dependency references
+	var auxID uint64
+	err = func() (err error) {
+		for fieldLabel, ref := range n.References {
+			auxID = safeParentID(tree, n, ref)
+			if auxID == 0 {
+				continue
+			}
+
+			err = n.Resource.SetValue(fieldLabel, 0, auxID)
+			if err != nil {
+				return
+			}
+		}
+		return
+	}()
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
+		return
+	}
+
+	// Flush to the DB
+	if !n.Evaluated.Skip {
+		err = store.UpsertAutomationWorkflow(ctx, s, n.Resource.(*types.Workflow))
+		if err != nil {
+			err = errors.Wrap(err, "failed to upsert Workflow")
+			return
+		}
+	}
+
+	// Handle resources nested under it
+	//
+	// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
+	//       the root function calls?
+
+	err = func() (err error) {
+		for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
+			nn = envoyx.OmitPlaceholderNodes(nn...)
+
+			switch rt {
+
+			}
+		}
+
+		return
+	}()
+	if err != nil {
+		err = errors.Wrap(err, "failed to encode nested resources")
+		return
+	}
+
+	return
+}
+
+// matchupWorkflows returns an index with indicates what resources already exist
+func (e StoreEncoder) matchupWorkflows(ctx context.Context, s store.Storer, uu map[int]types.Workflow, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
+	// @todo might need to do it smarter then this.
+	//       Most resources won't really be that vast so this should be acceptable for now.
+	aa, _, err := store.SearchAutomationWorkflows(ctx, s, types.WorkflowFilter{})
+	if err != nil {
+		return
+	}
+
+	idMap := make(map[uint64]*types.Workflow, len(aa))
+	strMap := make(map[string]*types.Workflow, len(aa))
+
+	for _, a := range aa {
+		strMap[a.Handle] = a
+		idMap[a.ID] = a
+
+	}
+
+	var aux *types.Workflow
+	var ok bool
+	for i, n := range nn {
+
+		for _, idf := range n.Identifiers.Slice {
+			if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
+				aux, ok = idMap[id]
+				if ok {
+					uu[i] = *aux
+					// When any identifier matches we can end it
+					break
+				}
+			}
+
+			aux, ok = strMap[idf]
+			if ok {
+				uu[i] = *aux
+				// When any identifier matches we can end it
+				break
+			}
+		}
+	}
+
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource trigger
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+// prepareTrigger prepares the resources of the given type for encoding
+func (e StoreEncoder) prepareTrigger(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
+	// Grab an index of already existing resources of this type
+	// @note since these resources should be fairly low-volume and existing for
+	//       a short time (and because we batch by resource type); fetching them all
+	//       into memory shouldn't hurt too much.
+	// @todo do some benchmarks and potentially implement some smarter check such as
+	//       a bloom filter or something similar.
+
+	// Get node scopes
+	scopedNodes, err := e.getScopeNodes(ctx, s, nn)
+	if err != nil {
+		err = errors.Wrap(err, "failed to get scope nodes")
+		return
+	}
+
+	// Initializing the index here (and using a hashmap) so it's not escaped to the heap
+	existing := make(map[int]types.Trigger, len(nn))
+	err = e.matchupTriggers(ctx, s, existing, scopedNodes, nn)
+	if err != nil {
+		err = errors.Wrap(err, "failed to matchup existing Triggers")
+		return
+	}
+
+	for i, n := range nn {
+		if n.Resource == nil {
+			panic("unexpected state: cannot call prepareTrigger with nodes without a defined Resource")
+		}
+
+		res, ok := n.Resource.(*types.Trigger)
+		if !ok {
+			panic("unexpected resource type: node expecting type of trigger")
+		}
+
+		existing, hasExisting := existing[i]
+
+		// Run expressions on the nodes
+		err = e.runEvals(ctx, hasExisting, n)
+		if err != nil {
+			return
+		}
+
+		if hasExisting {
+			// On existing, we don't need to re-do identifiers and references; simply
+			// changing up the internal resource is enough.
+			//
+			// In the future, we can pass down the tree and re-do the deps like that
+			switch n.Config.MergeAlg {
+			case envoyx.OnConflictPanic:
+				err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
+				return
+
+			case envoyx.OnConflictReplace:
+				// Replace; simple ID change should do the trick
+				res.ID = existing.ID
+
+			case envoyx.OnConflictSkip:
+				// Replace the node's resource with the fetched one
+				res = &existing
+
+				// @todo merging
+			}
+		} else {
+			// @todo actually a bottleneck. As per sonyflake docs, it can at most
+			//       generate up to 2**8 (256) IDs per 10ms in a single thread.
+			//       How can we improve this?
+			res.ID = id.Next()
+		}
+
+		// We can skip validation/defaults when the resource is overwritten by
+		// the one already stored (the panic one errors out anyway) since it
+		// should already be ok.
+		if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
+			err = e.setTriggerDefaults(res)
+			if err != nil {
+				return err
+			}
+
+			err = e.validateTrigger(res)
+			if err != nil {
+				return err
+			}
+		}
+
+		n.Resource = res
+	}
+
+	return
+}
+
+// encodeTriggers encodes a set of resource into the database
+func (e StoreEncoder) encodeTriggers(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
+	for _, n := range nn {
+		err = e.encodeTrigger(ctx, p, s, n, tree)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// encodeTrigger encodes the resource into the database
+func (e StoreEncoder) encodeTrigger(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
+	// Grab dependency references
+	var auxID uint64
+	err = func() (err error) {
+		for fieldLabel, ref := range n.References {
+			auxID = safeParentID(tree, n, ref)
+			if auxID == 0 {
+				continue
+			}
+
+			err = n.Resource.SetValue(fieldLabel, 0, auxID)
+			if err != nil {
+				return
+			}
+		}
+		return
+	}()
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
+		return
+	}
+
+	// Flush to the DB
+	if !n.Evaluated.Skip {
+		err = store.UpsertAutomationTrigger(ctx, s, n.Resource.(*types.Trigger))
+		if err != nil {
+			err = errors.Wrap(err, "failed to upsert Trigger")
+			return
+		}
+	}
+
+	// Handle resources nested under it
+	//
+	// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
+	//       the root function calls?
+
+	err = func() (err error) {
+		for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
+			nn = envoyx.OmitPlaceholderNodes(nn...)
+
+			switch rt {
+
+			}
+		}
+
+		return
+	}()
+	if err != nil {
+		err = errors.Wrap(err, "failed to encode nested resources")
+		return
+	}
+
+	return
+}
+
+// matchupTriggers returns an index with indicates what resources already exist
+func (e StoreEncoder) matchupTriggers(ctx context.Context, s store.Storer, uu map[int]types.Trigger, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
+	// @todo might need to do it smarter then this.
+	//       Most resources won't really be that vast so this should be acceptable for now.
+	aa, _, err := store.SearchAutomationTriggers(ctx, s, types.TriggerFilter{})
+	if err != nil {
+		return
+	}
+
+	idMap := make(map[uint64]*types.Trigger, len(aa))
+	strMap := make(map[string]*types.Trigger, len(aa))
+
+	for _, a := range aa {
+		idMap[a.ID] = a
+
+	}
+
+	var aux *types.Trigger
+	var ok bool
+	for i, n := range nn {
+
+		for _, idf := range n.Identifiers.Slice {
+			if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
+				aux, ok = idMap[id]
+				if ok {
+					uu[i] = *aux
+					// When any identifier matches we can end it
+					break
+				}
+			}
+
+			aux, ok = strMap[idf]
+			if ok {
+				uu[i] = *aux
+				// When any identifier matches we can end it
+				break
+			}
+		}
+	}
+
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Utility functions
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (e *StoreEncoder) grabStorer(p envoyx.EncodeParams) (s store.Storer, err error) {
+	auxs, ok := p.Params[paramsKeyStorer]
+	if !ok {
+		err = errors.Errorf("store encoder expects a store conforming to store.Storer interface")
+		return
+	}
+
+	s, ok = auxs.(store.Storer)
+	if !ok {
+		err = errors.Errorf("store encoder expects a store conforming to store.Storer interface")
+		return
+	}
+
+	return
+}
+
+func (e *StoreEncoder) runEvals(ctx context.Context, existing bool, n *envoyx.Node) (err error) {
+	// Skip if
+	if n.Config.SkipIfEval == nil {
+		return
+	}
+
+	aux, err := expr.EmptyVars().Cast(map[string]any{
+		"missing": !existing,
+	})
+	if err != nil {
+		return
+	}
+
+	n.Evaluated.Skip, err = n.Config.SkipIfEval.Test(ctx, aux.(*expr.Vars))
+	return
+}
+
+func (e StoreEncoder) getScopeNodes(ctx context.Context, s store.Storer, nn envoyx.NodeSet) (scopes envoyx.NodeSet, err error) {
+	// Get all requested scopes
+	scopes = make(envoyx.NodeSet, len(nn))
+
+	// @note skipping scope logic since it's currently only supported within
+	//       Compose resources.
+
+	return
+}
+
+func safeParentID(tt envoyx.Traverser, n *envoyx.Node, ref envoyx.Ref) (out uint64) {
+	rn := tt.ParentForRef(n, ref)
+	if rn == nil {
+		return
+	}
+
+	return rn.Resource.GetID()
+}

server/automation/envoy/store_encode.go

@@ -0,0 +1,36 @@
+package envoy
+
+import (
+	"context"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/automation/types"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/store"
+)
+
+func (e StoreEncoder) encode(ctx context.Context, p envoyx.EncodeParams, s store.Storer, rt string, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
+	return
+}
+
+func (e StoreEncoder) setWorkflowDefaults(res *types.Workflow) (err error) {
+	if res.CreatedAt.IsZero() {
+		res.CreatedAt = time.Now()
+	}
+	return
+}
+
+func (e StoreEncoder) validateWorkflow(res *types.Workflow) (err error) {
+	return
+}
+
+func (e StoreEncoder) setTriggerDefaults(res *types.Trigger) (err error) {
+	if res.CreatedAt.IsZero() {
+		res.CreatedAt = time.Now()
+	}
+	return
+}
+
+func (e StoreEncoder) validateTrigger(res *types.Trigger) (err error) {
+	return
+}

server/automation/envoy/yaml_decode.go

@@ -0,0 +1,12 @@
+package envoy
+
+import (
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"gopkg.in/yaml.v3"
+)
+
+func (d *auxYamlDoc) unmarshalTriggersExtendedNode(dctx documentContext, n *yaml.Node, meta ...*yaml.Node) (out envoyx.NodeSet, err error) {
+	return d.unmarshalTriggerNode(dctx, n, meta...)
+}
+
+func (d *auxYamlDoc) unmarshalYAML(k string, n *yaml.Node) (out envoyx.NodeSet, err error) { return }

server/automation/envoy/yaml_encode.gen.go

@@ -0,0 +1,357 @@
+package envoy
+
+// This file is auto-generated.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+//
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/automation/types"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/pkg/y7s"
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v3"
+)
+
+type (
+	// YamlEncoder is responsible for encoding Corteza resources into
+	// a YAML supported format
+	YamlEncoder struct{}
+)
+
+const (
+	paramsKeyWriter = "writer"
+)
+
+// Encode encodes the given Corteza resources into some YAML supported format
+//
+// Encoding should not do any additional processing apart from matching with
+// dependencies and runtime validation
+//
+// Preparation runs validation, default value initialization, matching with
+// already existing instances, ...
+//
+// The prepare function receives a set of nodes grouped by the resource type.
+// This enables some batching optimization and simplifications when it comes to
+// matching with existing resources.
+//
+// Prepare does not receive any placeholder nodes which are used solely
+// for dependency resolution.
+func (e YamlEncoder) Encode(ctx context.Context, p envoyx.EncodeParams, rt string, nodes envoyx.NodeSet, tt envoyx.Traverser) (err error) {
+	var (
+		out *yaml.Node
+		aux *yaml.Node
+	)
+	_ = aux
+
+	w, err := e.getWriter(p)
+	if err != nil {
+		return
+	}
+
+	switch rt {
+	case types.WorkflowResourceType:
+		aux, err = e.encodeWorkflows(ctx, p, nodes, tt)
+		if err != nil {
+			return
+		}
+		// Root level resources are always encoded as a map
+		out, err = y7s.AddMap(out, "workflow", aux)
+		if err != nil {
+			return
+		}
+
+	case types.TriggerResourceType:
+		aux, err = e.encodeTriggers(ctx, p, nodes, tt)
+		if err != nil {
+			return
+		}
+		// Root level resources are always encoded as a map
+		out, err = y7s.AddMap(out, "trigger", aux)
+		if err != nil {
+			return
+		}
+	default:
+		out, err = e.encode(ctx, out, p, rt, nodes, tt)
+		if err != nil {
+			return
+		}
+	}
+
+	// Don't output nil values since that will produce broken yaml docs
+	if out == nil {
+		return
+	}
+
+	return yaml.NewEncoder(w).Encode(out)
+} // // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource workflow
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (e YamlEncoder) encodeWorkflows(ctx context.Context, p envoyx.EncodeParams, nodes envoyx.NodeSet, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	var aux *yaml.Node
+	for _, n := range nodes {
+		aux, err = e.encodeWorkflow(ctx, p, n, tt)
+		if err != nil {
+			return
+		}
+
+		out, err = y7s.AddSeq(out, aux)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// encodeWorkflow focuses on the specific resource invoked by the Encode method
+func (e YamlEncoder) encodeWorkflow(ctx context.Context, p envoyx.EncodeParams, node *envoyx.Node, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	res := node.Resource.(*types.Workflow)
+
+	// Pre-compute some map values so we can omit error checking when encoding yaml nodes
+	auxCreatedAt, err := e.encodeTimestamp(p, res.CreatedAt)
+	if err != nil {
+		return
+	}
+	auxCreatedBy, err := e.encodeRef(p, res.CreatedBy, "CreatedBy", node, tt)
+	if err != nil {
+		return
+	}
+	auxDeletedAt, err := e.encodeTimestampNil(p, res.DeletedAt)
+	if err != nil {
+		return
+	}
+	auxDeletedBy, err := e.encodeRef(p, res.DeletedBy, "DeletedBy", node, tt)
+	if err != nil {
+		return
+	}
+
+	auxOwnedBy, err := e.encodeRef(p, res.OwnedBy, "OwnedBy", node, tt)
+	if err != nil {
+		return
+	}
+
+	auxRunAs, err := e.encodeRef(p, res.RunAs, "RunAs", node, tt)
+	if err != nil {
+		return
+	}
+
+	auxUpdatedAt, err := e.encodeTimestampNil(p, res.UpdatedAt)
+	if err != nil {
+		return
+	}
+	auxUpdatedBy, err := e.encodeRef(p, res.UpdatedBy, "UpdatedBy", node, tt)
+	if err != nil {
+		return
+	}
+
+	out, err = y7s.AddMap(out,
+		"createdAt", auxCreatedAt,
+		"createdBy", auxCreatedBy,
+		"deletedAt", auxDeletedAt,
+		"deletedBy", auxDeletedBy,
+		"enabled", res.Enabled,
+		"handle", res.Handle,
+		"id", res.ID,
+		"keepSessions", res.KeepSessions,
+		"meta", res.Meta,
+		"ownedBy", auxOwnedBy,
+		"paths", res.Paths,
+		"runAs", auxRunAs,
+		"scope", res.Scope,
+		"steps", res.Steps,
+		"trace", res.Trace,
+		"updatedAt", auxUpdatedAt,
+		"updatedBy", auxUpdatedBy,
+	)
+	if err != nil {
+		return
+	}
+
+	// Handle nested resources
+	var aux *yaml.Node
+	_ = aux
+
+	aux, err = e.encodeTriggers(ctx, p, tt.ChildrenForResourceType(node, types.TriggerResourceType), tt)
+	if err != nil {
+		return
+	}
+	out, err = y7s.AddMap(out,
+		"trigger", aux,
+	)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource trigger
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (e YamlEncoder) encodeTriggers(ctx context.Context, p envoyx.EncodeParams, nodes envoyx.NodeSet, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	var aux *yaml.Node
+	for _, n := range nodes {
+		aux, err = e.encodeTrigger(ctx, p, n, tt)
+		if err != nil {
+			return
+		}
+
+		out, err = y7s.AddSeq(out, aux)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// encodeTrigger focuses on the specific resource invoked by the Encode method
+func (e YamlEncoder) encodeTrigger(ctx context.Context, p envoyx.EncodeParams, node *envoyx.Node, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	res := node.Resource.(*types.Trigger)
+
+	// Pre-compute some map values so we can omit error checking when encoding yaml nodes
+
+	auxCreatedAt, err := e.encodeTimestamp(p, res.CreatedAt)
+	if err != nil {
+		return
+	}
+	auxCreatedBy, err := e.encodeRef(p, res.CreatedBy, "CreatedBy", node, tt)
+	if err != nil {
+		return
+	}
+	auxDeletedAt, err := e.encodeTimestampNil(p, res.DeletedAt)
+	if err != nil {
+		return
+	}
+	auxDeletedBy, err := e.encodeRef(p, res.DeletedBy, "DeletedBy", node, tt)
+	if err != nil {
+		return
+	}
+
+	auxOwnedBy, err := e.encodeRef(p, res.OwnedBy, "OwnedBy", node, tt)
+	if err != nil {
+		return
+	}
+
+	auxUpdatedAt, err := e.encodeTimestampNil(p, res.UpdatedAt)
+	if err != nil {
+		return
+	}
+	auxUpdatedBy, err := e.encodeRef(p, res.UpdatedBy, "UpdatedBy", node, tt)
+	if err != nil {
+		return
+	}
+	auxWorkflowID, err := e.encodeRef(p, res.WorkflowID, "WorkflowID", node, tt)
+	if err != nil {
+		return
+	}
+
+	out, err = y7s.AddMap(out,
+		"constraints", res.Constraints,
+		"createdAt", auxCreatedAt,
+		"createdBy", auxCreatedBy,
+		"deletedAt", auxDeletedAt,
+		"deletedBy", auxDeletedBy,
+		"enabled", res.Enabled,
+		"eventType", res.EventType,
+		"id", res.ID,
+		"input", res.Input,
+		"meta", res.Meta,
+		"ownedBy", auxOwnedBy,
+		"resourceType", res.ResourceType,
+		"stepID", res.StepID,
+		"updatedAt", auxUpdatedAt,
+		"updatedBy", auxUpdatedBy,
+		"workflowID", auxWorkflowID,
+	)
+	if err != nil {
+		return
+	}
+
+	// Handle nested resources
+	var aux *yaml.Node
+	_ = aux
+
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Encoding utils
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (e YamlEncoder) encodeTimestamp(p envoyx.EncodeParams, t time.Time) (any, error) {
+	if t.IsZero() {
+		return nil, nil
+	}
+
+	tz := p.Encoder.PreferredTimezone
+	if tz != "" {
+		tzL, err := time.LoadLocation(tz)
+		if err != nil {
+			return nil, err
+		}
+		t = t.In(tzL)
+	}
+
+	ly := p.Encoder.PreferredTimeLayout
+	if ly == "" {
+		ly = time.RFC3339
+	}
+
+	return t.Format(ly), nil
+}
+
+func (e YamlEncoder) encodeTimestampNil(p envoyx.EncodeParams, t *time.Time) (any, error) {
+	if t == nil {
+		return nil, nil
+	}
+
+	return e.encodeTimestamp(p, *t)
+}
+
+func (e YamlEncoder) encodeRef(p envoyx.EncodeParams, id uint64, field string, node *envoyx.Node, tt envoyx.Traverser) (any, error) {
+	parent := tt.ParentForRef(node, node.References[field])
+
+	// @todo should we panic instead?
+	//       for now gracefully fallback to the ID
+	if parent == nil {
+		return id, nil
+	}
+
+	return parent.Identifiers.FriendlyIdentifier(), nil
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Utility functions
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (e YamlEncoder) getWriter(p envoyx.EncodeParams) (out io.Writer, err error) {
+	aux, ok := p.Params[paramsKeyWriter]
+	if ok {
+		out, ok = aux.(io.Writer)
+		if ok {
+			return
+		}
+	}
+
+	// @todo consider adding support for managing files from a location
+	err = errors.Errorf("YAML encoder expects a writer conforming to io.Writer interface")
+	return
+}
+
+func safeParentIdentifier(tt envoyx.Traverser, n *envoyx.Node, ref envoyx.Ref) (out string) {
+	aux := tt.ParentForRef(n, ref)
+	if aux == nil {
+		return ref.Identifiers.FriendlyIdentifier()
+	}
+
+	return aux.Identifiers.FriendlyIdentifier()
+}

server/automation/envoy/yaml_encode.go

@@ -0,0 +1,12 @@
+package envoy
+
+import (
+	"context"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"gopkg.in/yaml.v3"
+)
+
+func (e YamlEncoder) encode(ctx context.Context, base *yaml.Node, p envoyx.EncodeParams, rt string, nodes envoyx.NodeSet, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	return
+}