7e385fc126
Some checks failed
SonarQube / SonarQube (push) Failing after 5m28s
E2e test on main branch push / server and client test (map[name:admin port:8080]) (push) Failing after 3m33s
Server and lib tests / Library test (js) (push) Failing after 1m36s
Server and lib tests / Library test (vue) (push) Successful in 2m24s
Server and lib tests / Web client test (admin) (push) Successful in 2m52s
Server and lib tests / Web client test (compose) (push) Successful in 3m23s
Server and lib tests / Web client test (discovery) (push) Successful in 2m49s
Server and lib tests / Web client test (one) (push) Successful in 2m52s
Server and lib tests / Web client test (privacy) (push) Successful in 2m44s
Server and lib tests / Web client test (reporter) (push) Successful in 2m56s
Server and lib tests / Web client test (workflow) (push) Successful in 3m0s
Server and lib tests / Server tests (push) Failing after 4m55s
E2e test on main branch push / notify-failure (push) Failing after 9s
Server and lib tests / Send matrix testing notification (push) Successful in 2s
Mark stale issues and pull requests / stale (push) Successful in 40s
558 lines
15 KiB
Go
Generated
558 lines
15 KiB
Go
Generated
package envoy
|
|
|
|
// This file is auto-generated.
|
|
//
|
|
// Changes to this file may cause incorrect behavior and will be lost if
|
|
// the code is regenerated.
|
|
//
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strconv"
|
|
|
|
"github.com/cortezaproject/corteza/server/automation/types"
|
|
"github.com/cortezaproject/corteza/server/pkg/envoyx"
|
|
"github.com/cortezaproject/corteza/server/pkg/expr"
|
|
"github.com/cortezaproject/corteza/server/pkg/id"
|
|
"github.com/cortezaproject/corteza/server/store"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
type (
|
|
// StoreEncoder is responsible for encoding Corteza resources into the
|
|
// database via the Storer or the DAL interface
|
|
//
|
|
// @todo consider having a different encoder for the DAL resources
|
|
StoreEncoder struct{}
|
|
)
|
|
|
|
// Prepare performs some initial processing on the resource before it can be encoded
|
|
//
|
|
// Preparation runs validation, default value initialization, matching with
|
|
// already existing instances, ...
|
|
//
|
|
// The prepare function receives a set of nodes grouped by the resource type.
|
|
// This enables some batching optimization and simplifications when it comes to
|
|
// matching with existing resources.
|
|
//
|
|
// Prepare does not receive any placeholder nodes which are used solely
|
|
// for dependency resolution.
|
|
func (e StoreEncoder) Prepare(ctx context.Context, p envoyx.EncodeParams, rt string, nn envoyx.NodeSet) (err error) {
|
|
s, err := e.grabStorer(p)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
switch rt {
|
|
case types.WorkflowResourceType:
|
|
return e.prepareWorkflow(ctx, p, s, nn)
|
|
|
|
case types.TriggerResourceType:
|
|
return e.prepareTrigger(ctx, p, s, nn)
|
|
default:
|
|
return e.prepare(ctx, p, s, rt, nn)
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// Encode encodes the given Corteza resources into the primary store
|
|
//
|
|
// Encoding should not do any additional processing apart from matching with
|
|
// dependencies and runtime validation
|
|
//
|
|
// The Encode function is called for every resource type where the resource
|
|
// appears at the root of the dependency tree.
|
|
// All of the root-level resources for that resource type are passed into the function.
|
|
// The encoding function must traverse the branches to encode all of the dependencies.
|
|
//
|
|
// This flow is used to simplify the flow of how resources are encoded into YAML
|
|
// (and other documents) as well as to simplify batching.
|
|
//
|
|
// Encode does not receive any placeholder nodes which are used solely
|
|
// for dependency resolution.
|
|
func (e StoreEncoder) Encode(ctx context.Context, p envoyx.EncodeParams, rt string, nodes envoyx.NodeSet, tree envoyx.Traverser) (err error) {
|
|
s, err := e.grabStorer(p)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
switch rt {
|
|
case types.WorkflowResourceType:
|
|
return e.encodeWorkflows(ctx, p, s, nodes, tree)
|
|
|
|
case types.TriggerResourceType:
|
|
return e.encodeTriggers(ctx, p, s, nodes, tree)
|
|
default:
|
|
return e.encode(ctx, p, s, rt, nodes, tree)
|
|
}
|
|
}
|
|
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
// Functions for resource workflow
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
|
|
// prepareWorkflow prepares the resources of the given type for encoding
|
|
func (e StoreEncoder) prepareWorkflow(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
|
|
// Grab an index of already existing resources of this type
|
|
// @note since these resources should be fairly low-volume and existing for
|
|
// a short time (and because we batch by resource type); fetching them all
|
|
// into memory shouldn't hurt too much.
|
|
// @todo do some benchmarks and potentially implement some smarter check such as
|
|
// a bloom filter or something similar.
|
|
|
|
// Get node scopes
|
|
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to get scope nodes")
|
|
return
|
|
}
|
|
|
|
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
|
|
existing := make(map[int]types.Workflow, len(nn))
|
|
err = e.matchupWorkflows(ctx, s, existing, scopedNodes, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to matchup existing Workflows")
|
|
return
|
|
}
|
|
|
|
for i, n := range nn {
|
|
if n.Resource == nil {
|
|
panic("unexpected state: cannot call prepareWorkflow with nodes without a defined Resource")
|
|
}
|
|
|
|
res, ok := n.Resource.(*types.Workflow)
|
|
if !ok {
|
|
panic("unexpected resource type: node expecting type of workflow")
|
|
}
|
|
|
|
existing, hasExisting := existing[i]
|
|
|
|
// Run expressions on the nodes
|
|
err = e.runEvals(ctx, hasExisting, n)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if hasExisting {
|
|
// On existing, we don't need to re-do identifiers and references; simply
|
|
// changing up the internal resource is enough.
|
|
//
|
|
// In the future, we can pass down the tree and re-do the deps like that
|
|
switch n.Config.MergeAlg {
|
|
case envoyx.OnConflictPanic:
|
|
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
|
|
return
|
|
|
|
case envoyx.OnConflictReplace:
|
|
// Replace; simple ID change should do the trick
|
|
res.ID = existing.ID
|
|
|
|
case envoyx.OnConflictSkip:
|
|
// Replace the node's resource with the fetched one
|
|
res = &existing
|
|
|
|
// @todo merging
|
|
}
|
|
} else {
|
|
// @todo actually a bottleneck. As per sonyflake docs, it can at most
|
|
// generate up to 2**8 (256) IDs per 10ms in a single thread.
|
|
// How can we improve this?
|
|
res.ID = id.Next()
|
|
}
|
|
|
|
// We can skip validation/defaults when the resource is overwritten by
|
|
// the one already stored (the panic one errors out anyway) since it
|
|
// should already be ok.
|
|
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
|
|
err = e.setWorkflowDefaults(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = e.validateWorkflow(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
n.Resource = res
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeWorkflows encodes a set of resource into the database
|
|
func (e StoreEncoder) encodeWorkflows(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
|
|
for _, n := range nn {
|
|
err = e.encodeWorkflow(ctx, p, s, n, tree)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeWorkflow encodes the resource into the database
|
|
func (e StoreEncoder) encodeWorkflow(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
|
|
// Grab dependency references
|
|
var auxID uint64
|
|
err = func() (err error) {
|
|
for fieldLabel, ref := range n.References {
|
|
auxID = safeParentID(tree, n, ref)
|
|
if auxID == 0 {
|
|
continue
|
|
}
|
|
|
|
err = n.Resource.SetValue(fieldLabel, 0, auxID)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
|
|
return
|
|
}
|
|
|
|
// Flush to the DB
|
|
if !n.Evaluated.Skip {
|
|
err = store.UpsertAutomationWorkflow(ctx, s, n.Resource.(*types.Workflow))
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to upsert Workflow")
|
|
return
|
|
}
|
|
}
|
|
|
|
// Handle resources nested under it
|
|
//
|
|
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
|
|
// the root function calls?
|
|
|
|
err = func() (err error) {
|
|
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
|
|
nn = envoyx.OmitPlaceholderNodes(nn...)
|
|
|
|
switch rt {
|
|
|
|
}
|
|
}
|
|
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to encode nested resources")
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// matchupWorkflows returns an index with indicates what resources already exist
|
|
func (e StoreEncoder) matchupWorkflows(ctx context.Context, s store.Storer, uu map[int]types.Workflow, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
|
|
// @todo might need to do it smarter then this.
|
|
// Most resources won't really be that vast so this should be acceptable for now.
|
|
aa, _, err := store.SearchAutomationWorkflows(ctx, s, types.WorkflowFilter{})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
idMap := make(map[uint64]*types.Workflow, len(aa))
|
|
strMap := make(map[string]*types.Workflow, len(aa))
|
|
|
|
for _, a := range aa {
|
|
strMap[a.Handle] = a
|
|
idMap[a.ID] = a
|
|
|
|
}
|
|
|
|
var aux *types.Workflow
|
|
var ok bool
|
|
for i, n := range nn {
|
|
|
|
for _, idf := range n.Identifiers.Slice {
|
|
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
|
|
aux, ok = idMap[id]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
|
|
aux, ok = strMap[idf]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
// Functions for resource trigger
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
|
|
// prepareTrigger prepares the resources of the given type for encoding
|
|
func (e StoreEncoder) prepareTrigger(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
|
|
// Grab an index of already existing resources of this type
|
|
// @note since these resources should be fairly low-volume and existing for
|
|
// a short time (and because we batch by resource type); fetching them all
|
|
// into memory shouldn't hurt too much.
|
|
// @todo do some benchmarks and potentially implement some smarter check such as
|
|
// a bloom filter or something similar.
|
|
|
|
// Get node scopes
|
|
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to get scope nodes")
|
|
return
|
|
}
|
|
|
|
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
|
|
existing := make(map[int]types.Trigger, len(nn))
|
|
err = e.matchupTriggers(ctx, s, existing, scopedNodes, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to matchup existing Triggers")
|
|
return
|
|
}
|
|
|
|
for i, n := range nn {
|
|
if n.Resource == nil {
|
|
panic("unexpected state: cannot call prepareTrigger with nodes without a defined Resource")
|
|
}
|
|
|
|
res, ok := n.Resource.(*types.Trigger)
|
|
if !ok {
|
|
panic("unexpected resource type: node expecting type of trigger")
|
|
}
|
|
|
|
existing, hasExisting := existing[i]
|
|
|
|
// Run expressions on the nodes
|
|
err = e.runEvals(ctx, hasExisting, n)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if hasExisting {
|
|
// On existing, we don't need to re-do identifiers and references; simply
|
|
// changing up the internal resource is enough.
|
|
//
|
|
// In the future, we can pass down the tree and re-do the deps like that
|
|
switch n.Config.MergeAlg {
|
|
case envoyx.OnConflictPanic:
|
|
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
|
|
return
|
|
|
|
case envoyx.OnConflictReplace:
|
|
// Replace; simple ID change should do the trick
|
|
res.ID = existing.ID
|
|
|
|
case envoyx.OnConflictSkip:
|
|
// Replace the node's resource with the fetched one
|
|
res = &existing
|
|
|
|
// @todo merging
|
|
}
|
|
} else {
|
|
// @todo actually a bottleneck. As per sonyflake docs, it can at most
|
|
// generate up to 2**8 (256) IDs per 10ms in a single thread.
|
|
// How can we improve this?
|
|
res.ID = id.Next()
|
|
}
|
|
|
|
// We can skip validation/defaults when the resource is overwritten by
|
|
// the one already stored (the panic one errors out anyway) since it
|
|
// should already be ok.
|
|
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
|
|
err = e.setTriggerDefaults(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = e.validateTrigger(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
n.Resource = res
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeTriggers encodes a set of resource into the database
|
|
func (e StoreEncoder) encodeTriggers(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
|
|
for _, n := range nn {
|
|
err = e.encodeTrigger(ctx, p, s, n, tree)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeTrigger encodes the resource into the database
|
|
func (e StoreEncoder) encodeTrigger(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
|
|
// Grab dependency references
|
|
var auxID uint64
|
|
err = func() (err error) {
|
|
for fieldLabel, ref := range n.References {
|
|
auxID = safeParentID(tree, n, ref)
|
|
if auxID == 0 {
|
|
continue
|
|
}
|
|
|
|
err = n.Resource.SetValue(fieldLabel, 0, auxID)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
|
|
return
|
|
}
|
|
|
|
// Flush to the DB
|
|
if !n.Evaluated.Skip {
|
|
err = store.UpsertAutomationTrigger(ctx, s, n.Resource.(*types.Trigger))
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to upsert Trigger")
|
|
return
|
|
}
|
|
}
|
|
|
|
// Handle resources nested under it
|
|
//
|
|
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
|
|
// the root function calls?
|
|
|
|
err = func() (err error) {
|
|
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
|
|
nn = envoyx.OmitPlaceholderNodes(nn...)
|
|
|
|
switch rt {
|
|
|
|
}
|
|
}
|
|
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to encode nested resources")
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// matchupTriggers returns an index with indicates what resources already exist
|
|
func (e StoreEncoder) matchupTriggers(ctx context.Context, s store.Storer, uu map[int]types.Trigger, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
|
|
// @todo might need to do it smarter then this.
|
|
// Most resources won't really be that vast so this should be acceptable for now.
|
|
aa, _, err := store.SearchAutomationTriggers(ctx, s, types.TriggerFilter{})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
idMap := make(map[uint64]*types.Trigger, len(aa))
|
|
strMap := make(map[string]*types.Trigger, len(aa))
|
|
|
|
for _, a := range aa {
|
|
idMap[a.ID] = a
|
|
|
|
}
|
|
|
|
var aux *types.Trigger
|
|
var ok bool
|
|
for i, n := range nn {
|
|
|
|
for _, idf := range n.Identifiers.Slice {
|
|
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
|
|
aux, ok = idMap[id]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
|
|
aux, ok = strMap[idf]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
// Utility functions
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
|
|
func (e *StoreEncoder) grabStorer(p envoyx.EncodeParams) (s store.Storer, err error) {
|
|
auxs, ok := p.Params[paramsKeyStorer]
|
|
if !ok {
|
|
err = errors.Errorf("store encoder expects a store conforming to store.Storer interface")
|
|
return
|
|
}
|
|
|
|
s, ok = auxs.(store.Storer)
|
|
if !ok {
|
|
err = errors.Errorf("store encoder expects a store conforming to store.Storer interface")
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func (e *StoreEncoder) runEvals(ctx context.Context, existing bool, n *envoyx.Node) (err error) {
|
|
// Skip if
|
|
if n.Config.SkipIfEval == nil {
|
|
return
|
|
}
|
|
|
|
aux, err := expr.EmptyVars().Cast(map[string]any{
|
|
"missing": !existing,
|
|
})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
n.Evaluated.Skip, err = n.Config.SkipIfEval.Test(ctx, aux.(*expr.Vars))
|
|
return
|
|
}
|
|
|
|
func (e StoreEncoder) getScopeNodes(ctx context.Context, s store.Storer, nn envoyx.NodeSet) (scopes envoyx.NodeSet, err error) {
|
|
// Get all requested scopes
|
|
scopes = make(envoyx.NodeSet, len(nn))
|
|
|
|
// @note skipping scope logic since it's currently only supported within
|
|
// Compose resources.
|
|
|
|
return
|
|
}
|
|
|
|
func safeParentID(tt envoyx.Traverser, n *envoyx.Node, ref envoyx.Ref) (out uint64) {
|
|
rn := tt.ParentForRef(n, ref)
|
|
if rn == nil {
|
|
return
|
|
}
|
|
|
|
return rn.Resource.GetID()
|
|
}
|