Some checks failed
SonarQube / SonarQube (push) Failing after 5m28s
E2e test on main branch push / server and client test (map[name:admin port:8080]) (push) Failing after 3m33s
Server and lib tests / Library test (js) (push) Failing after 1m36s
Server and lib tests / Library test (vue) (push) Successful in 2m24s
Server and lib tests / Web client test (admin) (push) Successful in 2m52s
Server and lib tests / Web client test (compose) (push) Successful in 3m23s
Server and lib tests / Web client test (discovery) (push) Successful in 2m49s
Server and lib tests / Web client test (one) (push) Successful in 2m52s
Server and lib tests / Web client test (privacy) (push) Successful in 2m44s
Server and lib tests / Web client test (reporter) (push) Successful in 2m56s
Server and lib tests / Web client test (workflow) (push) Successful in 3m0s
Server and lib tests / Server tests (push) Failing after 4m55s
E2e test on main branch push / notify-failure (push) Failing after 9s
Server and lib tests / Send matrix testing notification (push) Successful in 2s
Mark stale issues and pull requests / stale (push) Successful in 40s
2459 lines
67 KiB
Go
Generated
2459 lines
67 KiB
Go
Generated
package envoy
|
|
|
|
// This file is auto-generated.
|
|
//
|
|
// Changes to this file may cause incorrect behavior and will be lost if
|
|
// the code is regenerated.
|
|
//
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strconv"
|
|
|
|
"github.com/cortezaproject/corteza/server/pkg/envoyx"
|
|
"github.com/cortezaproject/corteza/server/pkg/expr"
|
|
"github.com/cortezaproject/corteza/server/pkg/id"
|
|
"github.com/cortezaproject/corteza/server/store"
|
|
"github.com/cortezaproject/corteza/server/system/types"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
type (
|
|
// StoreEncoder is responsible for encoding Corteza resources into the
|
|
// database via the Storer or the DAL interface
|
|
//
|
|
// @todo consider having a different encoder for the DAL resources
|
|
StoreEncoder struct{}
|
|
)
|
|
|
|
// Prepare performs some initial processing on the resource before it can be encoded
|
|
//
|
|
// Preparation runs validation, default value initialization, matching with
|
|
// already existing instances, ...
|
|
//
|
|
// The prepare function receives a set of nodes grouped by the resource type.
|
|
// This enables some batching optimization and simplifications when it comes to
|
|
// matching with existing resources.
|
|
//
|
|
// Prepare does not receive any placeholder nodes which are used solely
|
|
// for dependency resolution.
|
|
func (e StoreEncoder) Prepare(ctx context.Context, p envoyx.EncodeParams, rt string, nn envoyx.NodeSet) (err error) {
|
|
s, err := e.grabStorer(p)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
switch rt {
|
|
case types.ApplicationResourceType:
|
|
return e.prepareApplication(ctx, p, s, nn)
|
|
case types.ApigwRouteResourceType:
|
|
return e.prepareApigwRoute(ctx, p, s, nn)
|
|
case types.ApigwFilterResourceType:
|
|
return e.prepareApigwFilter(ctx, p, s, nn)
|
|
case types.AuthClientResourceType:
|
|
return e.prepareAuthClient(ctx, p, s, nn)
|
|
|
|
case types.QueueResourceType:
|
|
return e.prepareQueue(ctx, p, s, nn)
|
|
|
|
case types.ReportResourceType:
|
|
return e.prepareReport(ctx, p, s, nn)
|
|
|
|
case types.RoleResourceType:
|
|
return e.prepareRole(ctx, p, s, nn)
|
|
|
|
case types.TemplateResourceType:
|
|
return e.prepareTemplate(ctx, p, s, nn)
|
|
case types.UserResourceType:
|
|
return e.prepareUser(ctx, p, s, nn)
|
|
case types.DalConnectionResourceType:
|
|
return e.prepareDalConnection(ctx, p, s, nn)
|
|
case types.DalSensitivityLevelResourceType:
|
|
return e.prepareDalSensitivityLevel(ctx, p, s, nn)
|
|
|
|
default:
|
|
return e.prepare(ctx, p, s, rt, nn)
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// Encode encodes the given Corteza resources into the primary store
|
|
//
|
|
// Encoding should not do any additional processing apart from matching with
|
|
// dependencies and runtime validation
|
|
//
|
|
// The Encode function is called for every resource type where the resource
|
|
// appears at the root of the dependency tree.
|
|
// All of the root-level resources for that resource type are passed into the function.
|
|
// The encoding function must traverse the branches to encode all of the dependencies.
|
|
//
|
|
// This flow is used to simplify the flow of how resources are encoded into YAML
|
|
// (and other documents) as well as to simplify batching.
|
|
//
|
|
// Encode does not receive any placeholder nodes which are used solely
|
|
// for dependency resolution.
|
|
func (e StoreEncoder) Encode(ctx context.Context, p envoyx.EncodeParams, rt string, nodes envoyx.NodeSet, tree envoyx.Traverser) (err error) {
|
|
s, err := e.grabStorer(p)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
switch rt {
|
|
case types.ApplicationResourceType:
|
|
return e.encodeApplications(ctx, p, s, nodes, tree)
|
|
|
|
case types.ApigwRouteResourceType:
|
|
return e.encodeApigwRoutes(ctx, p, s, nodes, tree)
|
|
|
|
case types.ApigwFilterResourceType:
|
|
return e.encodeApigwFilters(ctx, p, s, nodes, tree)
|
|
|
|
case types.AuthClientResourceType:
|
|
return e.encodeAuthClients(ctx, p, s, nodes, tree)
|
|
|
|
case types.QueueResourceType:
|
|
return e.encodeQueues(ctx, p, s, nodes, tree)
|
|
|
|
case types.ReportResourceType:
|
|
return e.encodeReports(ctx, p, s, nodes, tree)
|
|
|
|
case types.RoleResourceType:
|
|
return e.encodeRoles(ctx, p, s, nodes, tree)
|
|
|
|
case types.TemplateResourceType:
|
|
return e.encodeTemplates(ctx, p, s, nodes, tree)
|
|
|
|
case types.UserResourceType:
|
|
return e.encodeUsers(ctx, p, s, nodes, tree)
|
|
|
|
case types.DalConnectionResourceType:
|
|
return e.encodeDalConnections(ctx, p, s, nodes, tree)
|
|
|
|
case types.DalSensitivityLevelResourceType:
|
|
return e.encodeDalSensitivityLevels(ctx, p, s, nodes, tree)
|
|
default:
|
|
return e.encode(ctx, p, s, rt, nodes, tree)
|
|
}
|
|
}
|
|
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
// Functions for resource application
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
|
|
// prepareApplication prepares the resources of the given type for encoding
|
|
func (e StoreEncoder) prepareApplication(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
|
|
// Grab an index of already existing resources of this type
|
|
// @note since these resources should be fairly low-volume and existing for
|
|
// a short time (and because we batch by resource type); fetching them all
|
|
// into memory shouldn't hurt too much.
|
|
// @todo do some benchmarks and potentially implement some smarter check such as
|
|
// a bloom filter or something similar.
|
|
|
|
// Get node scopes
|
|
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to get scope nodes")
|
|
return
|
|
}
|
|
|
|
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
|
|
existing := make(map[int]types.Application, len(nn))
|
|
err = e.matchupApplications(ctx, s, existing, scopedNodes, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to matchup existing Applications")
|
|
return
|
|
}
|
|
|
|
for i, n := range nn {
|
|
if n.Resource == nil {
|
|
panic("unexpected state: cannot call prepareApplication with nodes without a defined Resource")
|
|
}
|
|
|
|
res, ok := n.Resource.(*types.Application)
|
|
if !ok {
|
|
panic("unexpected resource type: node expecting type of application")
|
|
}
|
|
|
|
existing, hasExisting := existing[i]
|
|
|
|
// Run expressions on the nodes
|
|
err = e.runEvals(ctx, hasExisting, n)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if hasExisting {
|
|
// On existing, we don't need to re-do identifiers and references; simply
|
|
// changing up the internal resource is enough.
|
|
//
|
|
// In the future, we can pass down the tree and re-do the deps like that
|
|
switch n.Config.MergeAlg {
|
|
case envoyx.OnConflictPanic:
|
|
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
|
|
return
|
|
|
|
case envoyx.OnConflictReplace:
|
|
// Replace; simple ID change should do the trick
|
|
res.ID = existing.ID
|
|
|
|
case envoyx.OnConflictSkip:
|
|
// Replace the node's resource with the fetched one
|
|
res = &existing
|
|
|
|
// @todo merging
|
|
}
|
|
} else {
|
|
// @todo actually a bottleneck. As per sonyflake docs, it can at most
|
|
// generate up to 2**8 (256) IDs per 10ms in a single thread.
|
|
// How can we improve this?
|
|
res.ID = id.Next()
|
|
}
|
|
|
|
// We can skip validation/defaults when the resource is overwritten by
|
|
// the one already stored (the panic one errors out anyway) since it
|
|
// should already be ok.
|
|
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
|
|
err = e.setApplicationDefaults(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = e.validateApplication(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
n.Resource = res
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeApplications encodes a set of resource into the database
|
|
func (e StoreEncoder) encodeApplications(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
|
|
for _, n := range nn {
|
|
err = e.encodeApplication(ctx, p, s, n, tree)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeApplication encodes the resource into the database
|
|
func (e StoreEncoder) encodeApplication(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
|
|
// Grab dependency references
|
|
var auxID uint64
|
|
err = func() (err error) {
|
|
for fieldLabel, ref := range n.References {
|
|
auxID = safeParentID(tree, n, ref)
|
|
if auxID == 0 {
|
|
continue
|
|
}
|
|
|
|
err = n.Resource.SetValue(fieldLabel, 0, auxID)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
|
|
return
|
|
}
|
|
|
|
// Flush to the DB
|
|
if !n.Evaluated.Skip {
|
|
err = store.UpsertApplication(ctx, s, n.Resource.(*types.Application))
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to upsert Application")
|
|
return
|
|
}
|
|
}
|
|
|
|
// Handle resources nested under it
|
|
//
|
|
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
|
|
// the root function calls?
|
|
|
|
err = func() (err error) {
|
|
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
|
|
nn = envoyx.OmitPlaceholderNodes(nn...)
|
|
|
|
switch rt {
|
|
|
|
}
|
|
}
|
|
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to encode nested resources")
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// matchupApplications returns an index with indicates what resources already exist
|
|
func (e StoreEncoder) matchupApplications(ctx context.Context, s store.Storer, uu map[int]types.Application, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
|
|
// @todo might need to do it smarter then this.
|
|
// Most resources won't really be that vast so this should be acceptable for now.
|
|
aa, _, err := store.SearchApplications(ctx, s, types.ApplicationFilter{})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
idMap := make(map[uint64]*types.Application, len(aa))
|
|
strMap := make(map[string]*types.Application, len(aa))
|
|
|
|
for _, a := range aa {
|
|
idMap[a.ID] = a
|
|
|
|
}
|
|
|
|
var aux *types.Application
|
|
var ok bool
|
|
for i, n := range nn {
|
|
|
|
for _, idf := range n.Identifiers.Slice {
|
|
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
|
|
aux, ok = idMap[id]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
|
|
aux, ok = strMap[idf]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
// Functions for resource apigwRoute
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
|
|
// prepareApigwRoute prepares the resources of the given type for encoding
|
|
func (e StoreEncoder) prepareApigwRoute(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
|
|
// Grab an index of already existing resources of this type
|
|
// @note since these resources should be fairly low-volume and existing for
|
|
// a short time (and because we batch by resource type); fetching them all
|
|
// into memory shouldn't hurt too much.
|
|
// @todo do some benchmarks and potentially implement some smarter check such as
|
|
// a bloom filter or something similar.
|
|
|
|
// Get node scopes
|
|
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to get scope nodes")
|
|
return
|
|
}
|
|
|
|
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
|
|
existing := make(map[int]types.ApigwRoute, len(nn))
|
|
err = e.matchupApigwRoutes(ctx, s, existing, scopedNodes, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to matchup existing ApigwRoutes")
|
|
return
|
|
}
|
|
|
|
for i, n := range nn {
|
|
if n.Resource == nil {
|
|
panic("unexpected state: cannot call prepareApigwRoute with nodes without a defined Resource")
|
|
}
|
|
|
|
res, ok := n.Resource.(*types.ApigwRoute)
|
|
if !ok {
|
|
panic("unexpected resource type: node expecting type of apigwRoute")
|
|
}
|
|
|
|
existing, hasExisting := existing[i]
|
|
|
|
// Run expressions on the nodes
|
|
err = e.runEvals(ctx, hasExisting, n)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if hasExisting {
|
|
// On existing, we don't need to re-do identifiers and references; simply
|
|
// changing up the internal resource is enough.
|
|
//
|
|
// In the future, we can pass down the tree and re-do the deps like that
|
|
switch n.Config.MergeAlg {
|
|
case envoyx.OnConflictPanic:
|
|
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
|
|
return
|
|
|
|
case envoyx.OnConflictReplace:
|
|
// Replace; simple ID change should do the trick
|
|
res.ID = existing.ID
|
|
|
|
case envoyx.OnConflictSkip:
|
|
// Replace the node's resource with the fetched one
|
|
res = &existing
|
|
|
|
// @todo merging
|
|
}
|
|
} else {
|
|
// @todo actually a bottleneck. As per sonyflake docs, it can at most
|
|
// generate up to 2**8 (256) IDs per 10ms in a single thread.
|
|
// How can we improve this?
|
|
res.ID = id.Next()
|
|
}
|
|
|
|
// We can skip validation/defaults when the resource is overwritten by
|
|
// the one already stored (the panic one errors out anyway) since it
|
|
// should already be ok.
|
|
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
|
|
err = e.setApigwRouteDefaults(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = e.validateApigwRoute(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
n.Resource = res
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeApigwRoutes encodes a set of resource into the database
|
|
func (e StoreEncoder) encodeApigwRoutes(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
|
|
for _, n := range nn {
|
|
err = e.encodeApigwRoute(ctx, p, s, n, tree)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeApigwRoute encodes the resource into the database
|
|
func (e StoreEncoder) encodeApigwRoute(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
|
|
// Grab dependency references
|
|
var auxID uint64
|
|
err = func() (err error) {
|
|
for fieldLabel, ref := range n.References {
|
|
auxID = safeParentID(tree, n, ref)
|
|
if auxID == 0 {
|
|
continue
|
|
}
|
|
|
|
err = n.Resource.SetValue(fieldLabel, 0, auxID)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
|
|
return
|
|
}
|
|
|
|
// Flush to the DB
|
|
if !n.Evaluated.Skip {
|
|
err = store.UpsertApigwRoute(ctx, s, n.Resource.(*types.ApigwRoute))
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to upsert ApigwRoute")
|
|
return
|
|
}
|
|
}
|
|
|
|
// Handle resources nested under it
|
|
//
|
|
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
|
|
// the root function calls?
|
|
|
|
err = func() (err error) {
|
|
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
|
|
nn = envoyx.OmitPlaceholderNodes(nn...)
|
|
|
|
switch rt {
|
|
|
|
}
|
|
}
|
|
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to encode nested resources")
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// matchupApigwRoutes returns an index with indicates what resources already exist
|
|
func (e StoreEncoder) matchupApigwRoutes(ctx context.Context, s store.Storer, uu map[int]types.ApigwRoute, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
|
|
// @todo might need to do it smarter then this.
|
|
// Most resources won't really be that vast so this should be acceptable for now.
|
|
aa, _, err := store.SearchApigwRoutes(ctx, s, types.ApigwRouteFilter{})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
idMap := make(map[uint64]*types.ApigwRoute, len(aa))
|
|
strMap := make(map[string]*types.ApigwRoute, len(aa))
|
|
|
|
for _, a := range aa {
|
|
idMap[a.ID] = a
|
|
|
|
}
|
|
|
|
var aux *types.ApigwRoute
|
|
var ok bool
|
|
for i, n := range nn {
|
|
|
|
for _, idf := range n.Identifiers.Slice {
|
|
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
|
|
aux, ok = idMap[id]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
|
|
aux, ok = strMap[idf]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
// Functions for resource apigwFilter
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
|
|
// prepareApigwFilter prepares the resources of the given type for encoding
|
|
func (e StoreEncoder) prepareApigwFilter(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
|
|
// Grab an index of already existing resources of this type
|
|
// @note since these resources should be fairly low-volume and existing for
|
|
// a short time (and because we batch by resource type); fetching them all
|
|
// into memory shouldn't hurt too much.
|
|
// @todo do some benchmarks and potentially implement some smarter check such as
|
|
// a bloom filter or something similar.
|
|
|
|
// Get node scopes
|
|
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to get scope nodes")
|
|
return
|
|
}
|
|
|
|
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
|
|
existing := make(map[int]types.ApigwFilter, len(nn))
|
|
err = e.matchupApigwFilters(ctx, s, existing, scopedNodes, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to matchup existing ApigwFilters")
|
|
return
|
|
}
|
|
|
|
for i, n := range nn {
|
|
if n.Resource == nil {
|
|
panic("unexpected state: cannot call prepareApigwFilter with nodes without a defined Resource")
|
|
}
|
|
|
|
res, ok := n.Resource.(*types.ApigwFilter)
|
|
if !ok {
|
|
panic("unexpected resource type: node expecting type of apigwFilter")
|
|
}
|
|
|
|
existing, hasExisting := existing[i]
|
|
|
|
// Run expressions on the nodes
|
|
err = e.runEvals(ctx, hasExisting, n)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if hasExisting {
|
|
// On existing, we don't need to re-do identifiers and references; simply
|
|
// changing up the internal resource is enough.
|
|
//
|
|
// In the future, we can pass down the tree and re-do the deps like that
|
|
switch n.Config.MergeAlg {
|
|
case envoyx.OnConflictPanic:
|
|
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
|
|
return
|
|
|
|
case envoyx.OnConflictReplace:
|
|
// Replace; simple ID change should do the trick
|
|
res.ID = existing.ID
|
|
|
|
case envoyx.OnConflictSkip:
|
|
// Replace the node's resource with the fetched one
|
|
res = &existing
|
|
|
|
// @todo merging
|
|
}
|
|
} else {
|
|
// @todo actually a bottleneck. As per sonyflake docs, it can at most
|
|
// generate up to 2**8 (256) IDs per 10ms in a single thread.
|
|
// How can we improve this?
|
|
res.ID = id.Next()
|
|
}
|
|
|
|
// We can skip validation/defaults when the resource is overwritten by
|
|
// the one already stored (the panic one errors out anyway) since it
|
|
// should already be ok.
|
|
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
|
|
err = e.setApigwFilterDefaults(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = e.validateApigwFilter(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
n.Resource = res
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeApigwFilters encodes a set of resource into the database
|
|
func (e StoreEncoder) encodeApigwFilters(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
|
|
for _, n := range nn {
|
|
err = e.encodeApigwFilter(ctx, p, s, n, tree)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeApigwFilter encodes the resource into the database
|
|
func (e StoreEncoder) encodeApigwFilter(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
|
|
// Grab dependency references
|
|
var auxID uint64
|
|
err = func() (err error) {
|
|
for fieldLabel, ref := range n.References {
|
|
auxID = safeParentID(tree, n, ref)
|
|
if auxID == 0 {
|
|
continue
|
|
}
|
|
|
|
err = n.Resource.SetValue(fieldLabel, 0, auxID)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
|
|
return
|
|
}
|
|
|
|
// Flush to the DB
|
|
if !n.Evaluated.Skip {
|
|
err = store.UpsertApigwFilter(ctx, s, n.Resource.(*types.ApigwFilter))
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to upsert ApigwFilter")
|
|
return
|
|
}
|
|
}
|
|
|
|
// Handle resources nested under it
|
|
//
|
|
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
|
|
// the root function calls?
|
|
|
|
err = func() (err error) {
|
|
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
|
|
nn = envoyx.OmitPlaceholderNodes(nn...)
|
|
|
|
switch rt {
|
|
|
|
}
|
|
}
|
|
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to encode nested resources")
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// matchupApigwFilters returns an index with indicates what resources already exist
|
|
func (e StoreEncoder) matchupApigwFilters(ctx context.Context, s store.Storer, uu map[int]types.ApigwFilter, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
|
|
// @todo might need to do it smarter then this.
|
|
// Most resources won't really be that vast so this should be acceptable for now.
|
|
aa, _, err := store.SearchApigwFilters(ctx, s, types.ApigwFilterFilter{})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
idMap := make(map[uint64]*types.ApigwFilter, len(aa))
|
|
strMap := make(map[string]*types.ApigwFilter, len(aa))
|
|
|
|
for _, a := range aa {
|
|
idMap[a.ID] = a
|
|
|
|
}
|
|
|
|
var aux *types.ApigwFilter
|
|
var ok bool
|
|
for i, n := range nn {
|
|
|
|
for _, idf := range n.Identifiers.Slice {
|
|
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
|
|
aux, ok = idMap[id]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
|
|
aux, ok = strMap[idf]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
// Functions for resource authClient
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
|
|
// prepareAuthClient prepares the resources of the given type for encoding
|
|
func (e StoreEncoder) prepareAuthClient(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
|
|
// Grab an index of already existing resources of this type
|
|
// @note since these resources should be fairly low-volume and existing for
|
|
// a short time (and because we batch by resource type); fetching them all
|
|
// into memory shouldn't hurt too much.
|
|
// @todo do some benchmarks and potentially implement some smarter check such as
|
|
// a bloom filter or something similar.
|
|
|
|
// Get node scopes
|
|
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to get scope nodes")
|
|
return
|
|
}
|
|
|
|
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
|
|
existing := make(map[int]types.AuthClient, len(nn))
|
|
err = e.matchupAuthClients(ctx, s, existing, scopedNodes, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to matchup existing AuthClients")
|
|
return
|
|
}
|
|
|
|
for i, n := range nn {
|
|
if n.Resource == nil {
|
|
panic("unexpected state: cannot call prepareAuthClient with nodes without a defined Resource")
|
|
}
|
|
|
|
res, ok := n.Resource.(*types.AuthClient)
|
|
if !ok {
|
|
panic("unexpected resource type: node expecting type of authClient")
|
|
}
|
|
|
|
existing, hasExisting := existing[i]
|
|
|
|
// Run expressions on the nodes
|
|
err = e.runEvals(ctx, hasExisting, n)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if hasExisting {
|
|
// On existing, we don't need to re-do identifiers and references; simply
|
|
// changing up the internal resource is enough.
|
|
//
|
|
// In the future, we can pass down the tree and re-do the deps like that
|
|
switch n.Config.MergeAlg {
|
|
case envoyx.OnConflictPanic:
|
|
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
|
|
return
|
|
|
|
case envoyx.OnConflictReplace:
|
|
// Replace; simple ID change should do the trick
|
|
res.ID = existing.ID
|
|
|
|
case envoyx.OnConflictSkip:
|
|
// Replace the node's resource with the fetched one
|
|
res = &existing
|
|
|
|
// @todo merging
|
|
}
|
|
} else {
|
|
// @todo actually a bottleneck. As per sonyflake docs, it can at most
|
|
// generate up to 2**8 (256) IDs per 10ms in a single thread.
|
|
// How can we improve this?
|
|
res.ID = id.Next()
|
|
}
|
|
|
|
// We can skip validation/defaults when the resource is overwritten by
|
|
// the one already stored (the panic one errors out anyway) since it
|
|
// should already be ok.
|
|
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
|
|
err = e.setAuthClientDefaults(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = e.validateAuthClient(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
n.Resource = res
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeAuthClients encodes a set of resource into the database
|
|
func (e StoreEncoder) encodeAuthClients(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
|
|
for _, n := range nn {
|
|
err = e.encodeAuthClient(ctx, p, s, n, tree)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeAuthClient encodes the resource into the database
|
|
func (e StoreEncoder) encodeAuthClient(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
|
|
// Grab dependency references
|
|
var auxID uint64
|
|
err = func() (err error) {
|
|
for fieldLabel, ref := range n.References {
|
|
auxID = safeParentID(tree, n, ref)
|
|
if auxID == 0 {
|
|
continue
|
|
}
|
|
|
|
err = n.Resource.SetValue(fieldLabel, 0, auxID)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
|
|
return
|
|
}
|
|
|
|
// Flush to the DB
|
|
if !n.Evaluated.Skip {
|
|
err = store.UpsertAuthClient(ctx, s, n.Resource.(*types.AuthClient))
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to upsert AuthClient")
|
|
return
|
|
}
|
|
}
|
|
|
|
// Handle resources nested under it
|
|
//
|
|
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
|
|
// the root function calls?
|
|
|
|
err = func() (err error) {
|
|
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
|
|
nn = envoyx.OmitPlaceholderNodes(nn...)
|
|
|
|
switch rt {
|
|
|
|
}
|
|
}
|
|
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to encode nested resources")
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// matchupAuthClients returns an index with indicates what resources already exist
|
|
func (e StoreEncoder) matchupAuthClients(ctx context.Context, s store.Storer, uu map[int]types.AuthClient, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
|
|
// @todo might need to do it smarter then this.
|
|
// Most resources won't really be that vast so this should be acceptable for now.
|
|
aa, _, err := store.SearchAuthClients(ctx, s, types.AuthClientFilter{})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
idMap := make(map[uint64]*types.AuthClient, len(aa))
|
|
strMap := make(map[string]*types.AuthClient, len(aa))
|
|
|
|
for _, a := range aa {
|
|
strMap[a.Handle] = a
|
|
idMap[a.ID] = a
|
|
|
|
}
|
|
|
|
var aux *types.AuthClient
|
|
var ok bool
|
|
for i, n := range nn {
|
|
|
|
for _, idf := range n.Identifiers.Slice {
|
|
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
|
|
aux, ok = idMap[id]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
|
|
aux, ok = strMap[idf]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
// Functions for resource queue
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
|
|
// prepareQueue prepares the resources of the given type for encoding
|
|
func (e StoreEncoder) prepareQueue(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
|
|
// Grab an index of already existing resources of this type
|
|
// @note since these resources should be fairly low-volume and existing for
|
|
// a short time (and because we batch by resource type); fetching them all
|
|
// into memory shouldn't hurt too much.
|
|
// @todo do some benchmarks and potentially implement some smarter check such as
|
|
// a bloom filter or something similar.
|
|
|
|
// Get node scopes
|
|
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to get scope nodes")
|
|
return
|
|
}
|
|
|
|
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
|
|
existing := make(map[int]types.Queue, len(nn))
|
|
err = e.matchupQueues(ctx, s, existing, scopedNodes, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to matchup existing Queues")
|
|
return
|
|
}
|
|
|
|
for i, n := range nn {
|
|
if n.Resource == nil {
|
|
panic("unexpected state: cannot call prepareQueue with nodes without a defined Resource")
|
|
}
|
|
|
|
res, ok := n.Resource.(*types.Queue)
|
|
if !ok {
|
|
panic("unexpected resource type: node expecting type of queue")
|
|
}
|
|
|
|
existing, hasExisting := existing[i]
|
|
|
|
// Run expressions on the nodes
|
|
err = e.runEvals(ctx, hasExisting, n)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if hasExisting {
|
|
// On existing, we don't need to re-do identifiers and references; simply
|
|
// changing up the internal resource is enough.
|
|
//
|
|
// In the future, we can pass down the tree and re-do the deps like that
|
|
switch n.Config.MergeAlg {
|
|
case envoyx.OnConflictPanic:
|
|
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
|
|
return
|
|
|
|
case envoyx.OnConflictReplace:
|
|
// Replace; simple ID change should do the trick
|
|
res.ID = existing.ID
|
|
|
|
case envoyx.OnConflictSkip:
|
|
// Replace the node's resource with the fetched one
|
|
res = &existing
|
|
|
|
// @todo merging
|
|
}
|
|
} else {
|
|
// @todo actually a bottleneck. As per sonyflake docs, it can at most
|
|
// generate up to 2**8 (256) IDs per 10ms in a single thread.
|
|
// How can we improve this?
|
|
res.ID = id.Next()
|
|
}
|
|
|
|
// We can skip validation/defaults when the resource is overwritten by
|
|
// the one already stored (the panic one errors out anyway) since it
|
|
// should already be ok.
|
|
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
|
|
err = e.setQueueDefaults(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = e.validateQueue(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
n.Resource = res
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeQueues encodes a set of resource into the database
|
|
func (e StoreEncoder) encodeQueues(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
|
|
for _, n := range nn {
|
|
err = e.encodeQueue(ctx, p, s, n, tree)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeQueue encodes the resource into the database
|
|
func (e StoreEncoder) encodeQueue(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
|
|
// Grab dependency references
|
|
var auxID uint64
|
|
err = func() (err error) {
|
|
for fieldLabel, ref := range n.References {
|
|
auxID = safeParentID(tree, n, ref)
|
|
if auxID == 0 {
|
|
continue
|
|
}
|
|
|
|
err = n.Resource.SetValue(fieldLabel, 0, auxID)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
|
|
return
|
|
}
|
|
|
|
// Flush to the DB
|
|
if !n.Evaluated.Skip {
|
|
err = store.UpsertQueue(ctx, s, n.Resource.(*types.Queue))
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to upsert Queue")
|
|
return
|
|
}
|
|
}
|
|
|
|
// Handle resources nested under it
|
|
//
|
|
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
|
|
// the root function calls?
|
|
|
|
err = func() (err error) {
|
|
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
|
|
nn = envoyx.OmitPlaceholderNodes(nn...)
|
|
|
|
switch rt {
|
|
|
|
}
|
|
}
|
|
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to encode nested resources")
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// matchupQueues returns an index with indicates what resources already exist
|
|
func (e StoreEncoder) matchupQueues(ctx context.Context, s store.Storer, uu map[int]types.Queue, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
|
|
// @todo might need to do it smarter then this.
|
|
// Most resources won't really be that vast so this should be acceptable for now.
|
|
aa, _, err := store.SearchQueues(ctx, s, types.QueueFilter{})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
idMap := make(map[uint64]*types.Queue, len(aa))
|
|
strMap := make(map[string]*types.Queue, len(aa))
|
|
|
|
for _, a := range aa {
|
|
idMap[a.ID] = a
|
|
strMap[a.Queue] = a
|
|
|
|
}
|
|
|
|
var aux *types.Queue
|
|
var ok bool
|
|
for i, n := range nn {
|
|
|
|
for _, idf := range n.Identifiers.Slice {
|
|
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
|
|
aux, ok = idMap[id]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
|
|
aux, ok = strMap[idf]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
// Functions for resource report
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
|
|
// prepareReport prepares the resources of the given type for encoding
|
|
func (e StoreEncoder) prepareReport(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
|
|
// Grab an index of already existing resources of this type
|
|
// @note since these resources should be fairly low-volume and existing for
|
|
// a short time (and because we batch by resource type); fetching them all
|
|
// into memory shouldn't hurt too much.
|
|
// @todo do some benchmarks and potentially implement some smarter check such as
|
|
// a bloom filter or something similar.
|
|
|
|
// Get node scopes
|
|
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to get scope nodes")
|
|
return
|
|
}
|
|
|
|
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
|
|
existing := make(map[int]types.Report, len(nn))
|
|
err = e.matchupReports(ctx, s, existing, scopedNodes, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to matchup existing Reports")
|
|
return
|
|
}
|
|
|
|
for i, n := range nn {
|
|
if n.Resource == nil {
|
|
panic("unexpected state: cannot call prepareReport with nodes without a defined Resource")
|
|
}
|
|
|
|
res, ok := n.Resource.(*types.Report)
|
|
if !ok {
|
|
panic("unexpected resource type: node expecting type of report")
|
|
}
|
|
|
|
existing, hasExisting := existing[i]
|
|
|
|
// Run expressions on the nodes
|
|
err = e.runEvals(ctx, hasExisting, n)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if hasExisting {
|
|
// On existing, we don't need to re-do identifiers and references; simply
|
|
// changing up the internal resource is enough.
|
|
//
|
|
// In the future, we can pass down the tree and re-do the deps like that
|
|
switch n.Config.MergeAlg {
|
|
case envoyx.OnConflictPanic:
|
|
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
|
|
return
|
|
|
|
case envoyx.OnConflictReplace:
|
|
// Replace; simple ID change should do the trick
|
|
res.ID = existing.ID
|
|
|
|
case envoyx.OnConflictSkip:
|
|
// Replace the node's resource with the fetched one
|
|
res = &existing
|
|
|
|
// @todo merging
|
|
}
|
|
} else {
|
|
// @todo actually a bottleneck. As per sonyflake docs, it can at most
|
|
// generate up to 2**8 (256) IDs per 10ms in a single thread.
|
|
// How can we improve this?
|
|
res.ID = id.Next()
|
|
}
|
|
|
|
// We can skip validation/defaults when the resource is overwritten by
|
|
// the one already stored (the panic one errors out anyway) since it
|
|
// should already be ok.
|
|
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
|
|
err = e.setReportDefaults(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = e.validateReport(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
n.Resource = res
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeReports encodes a set of resource into the database
|
|
func (e StoreEncoder) encodeReports(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
|
|
for _, n := range nn {
|
|
err = e.encodeReport(ctx, p, s, n, tree)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeReport encodes the resource into the database
|
|
func (e StoreEncoder) encodeReport(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
|
|
// Grab dependency references
|
|
var auxID uint64
|
|
err = func() (err error) {
|
|
for fieldLabel, ref := range n.References {
|
|
auxID = safeParentID(tree, n, ref)
|
|
if auxID == 0 {
|
|
continue
|
|
}
|
|
|
|
err = n.Resource.SetValue(fieldLabel, 0, auxID)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
|
|
return
|
|
}
|
|
|
|
// Flush to the DB
|
|
if !n.Evaluated.Skip {
|
|
err = store.UpsertReport(ctx, s, n.Resource.(*types.Report))
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to upsert Report")
|
|
return
|
|
}
|
|
}
|
|
|
|
// Handle resources nested under it
|
|
//
|
|
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
|
|
// the root function calls?
|
|
|
|
err = func() (err error) {
|
|
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
|
|
nn = envoyx.OmitPlaceholderNodes(nn...)
|
|
|
|
switch rt {
|
|
|
|
}
|
|
}
|
|
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to encode nested resources")
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// matchupReports returns an index with indicates what resources already exist
|
|
func (e StoreEncoder) matchupReports(ctx context.Context, s store.Storer, uu map[int]types.Report, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
|
|
// @todo might need to do it smarter then this.
|
|
// Most resources won't really be that vast so this should be acceptable for now.
|
|
aa, _, err := store.SearchReports(ctx, s, types.ReportFilter{})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
idMap := make(map[uint64]*types.Report, len(aa))
|
|
strMap := make(map[string]*types.Report, len(aa))
|
|
|
|
for _, a := range aa {
|
|
strMap[a.Handle] = a
|
|
idMap[a.ID] = a
|
|
|
|
}
|
|
|
|
var aux *types.Report
|
|
var ok bool
|
|
for i, n := range nn {
|
|
|
|
for _, idf := range n.Identifiers.Slice {
|
|
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
|
|
aux, ok = idMap[id]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
|
|
aux, ok = strMap[idf]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
// Functions for resource role
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
|
|
// prepareRole prepares the resources of the given type for encoding
|
|
func (e StoreEncoder) prepareRole(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
|
|
// Grab an index of already existing resources of this type
|
|
// @note since these resources should be fairly low-volume and existing for
|
|
// a short time (and because we batch by resource type); fetching them all
|
|
// into memory shouldn't hurt too much.
|
|
// @todo do some benchmarks and potentially implement some smarter check such as
|
|
// a bloom filter or something similar.
|
|
|
|
// Get node scopes
|
|
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to get scope nodes")
|
|
return
|
|
}
|
|
|
|
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
|
|
existing := make(map[int]types.Role, len(nn))
|
|
err = e.matchupRoles(ctx, s, existing, scopedNodes, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to matchup existing Roles")
|
|
return
|
|
}
|
|
|
|
for i, n := range nn {
|
|
if n.Resource == nil {
|
|
panic("unexpected state: cannot call prepareRole with nodes without a defined Resource")
|
|
}
|
|
|
|
res, ok := n.Resource.(*types.Role)
|
|
if !ok {
|
|
panic("unexpected resource type: node expecting type of role")
|
|
}
|
|
|
|
existing, hasExisting := existing[i]
|
|
|
|
// Run expressions on the nodes
|
|
err = e.runEvals(ctx, hasExisting, n)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if hasExisting {
|
|
// On existing, we don't need to re-do identifiers and references; simply
|
|
// changing up the internal resource is enough.
|
|
//
|
|
// In the future, we can pass down the tree and re-do the deps like that
|
|
switch n.Config.MergeAlg {
|
|
case envoyx.OnConflictPanic:
|
|
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
|
|
return
|
|
|
|
case envoyx.OnConflictReplace:
|
|
// Replace; simple ID change should do the trick
|
|
res.ID = existing.ID
|
|
|
|
case envoyx.OnConflictSkip:
|
|
// Replace the node's resource with the fetched one
|
|
res = &existing
|
|
|
|
// @todo merging
|
|
}
|
|
} else {
|
|
// @todo actually a bottleneck. As per sonyflake docs, it can at most
|
|
// generate up to 2**8 (256) IDs per 10ms in a single thread.
|
|
// How can we improve this?
|
|
res.ID = id.Next()
|
|
}
|
|
|
|
// We can skip validation/defaults when the resource is overwritten by
|
|
// the one already stored (the panic one errors out anyway) since it
|
|
// should already be ok.
|
|
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
|
|
err = e.setRoleDefaults(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = e.validateRole(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
n.Resource = res
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeRoles encodes a set of resource into the database
|
|
func (e StoreEncoder) encodeRoles(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
|
|
for _, n := range nn {
|
|
err = e.encodeRole(ctx, p, s, n, tree)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeRole encodes the resource into the database
|
|
func (e StoreEncoder) encodeRole(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
|
|
// Grab dependency references
|
|
var auxID uint64
|
|
err = func() (err error) {
|
|
for fieldLabel, ref := range n.References {
|
|
auxID = safeParentID(tree, n, ref)
|
|
if auxID == 0 {
|
|
continue
|
|
}
|
|
|
|
err = n.Resource.SetValue(fieldLabel, 0, auxID)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
|
|
return
|
|
}
|
|
|
|
// Flush to the DB
|
|
if !n.Evaluated.Skip {
|
|
err = store.UpsertRole(ctx, s, n.Resource.(*types.Role))
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to upsert Role")
|
|
return
|
|
}
|
|
}
|
|
|
|
// Handle resources nested under it
|
|
//
|
|
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
|
|
// the root function calls?
|
|
|
|
err = func() (err error) {
|
|
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
|
|
nn = envoyx.OmitPlaceholderNodes(nn...)
|
|
|
|
switch rt {
|
|
|
|
}
|
|
}
|
|
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to encode nested resources")
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// matchupRoles returns an index with indicates what resources already exist
|
|
func (e StoreEncoder) matchupRoles(ctx context.Context, s store.Storer, uu map[int]types.Role, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
|
|
// @todo might need to do it smarter then this.
|
|
// Most resources won't really be that vast so this should be acceptable for now.
|
|
aa, _, err := store.SearchRoles(ctx, s, types.RoleFilter{})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
idMap := make(map[uint64]*types.Role, len(aa))
|
|
strMap := make(map[string]*types.Role, len(aa))
|
|
|
|
for _, a := range aa {
|
|
strMap[a.Handle] = a
|
|
idMap[a.ID] = a
|
|
|
|
}
|
|
|
|
var aux *types.Role
|
|
var ok bool
|
|
for i, n := range nn {
|
|
|
|
for _, idf := range n.Identifiers.Slice {
|
|
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
|
|
aux, ok = idMap[id]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
|
|
aux, ok = strMap[idf]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
// Functions for resource template
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
|
|
// prepareTemplate prepares the resources of the given type for encoding
|
|
func (e StoreEncoder) prepareTemplate(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
|
|
// Grab an index of already existing resources of this type
|
|
// @note since these resources should be fairly low-volume and existing for
|
|
// a short time (and because we batch by resource type); fetching them all
|
|
// into memory shouldn't hurt too much.
|
|
// @todo do some benchmarks and potentially implement some smarter check such as
|
|
// a bloom filter or something similar.
|
|
|
|
// Get node scopes
|
|
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to get scope nodes")
|
|
return
|
|
}
|
|
|
|
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
|
|
existing := make(map[int]types.Template, len(nn))
|
|
err = e.matchupTemplates(ctx, s, existing, scopedNodes, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to matchup existing Templates")
|
|
return
|
|
}
|
|
|
|
for i, n := range nn {
|
|
if n.Resource == nil {
|
|
panic("unexpected state: cannot call prepareTemplate with nodes without a defined Resource")
|
|
}
|
|
|
|
res, ok := n.Resource.(*types.Template)
|
|
if !ok {
|
|
panic("unexpected resource type: node expecting type of template")
|
|
}
|
|
|
|
existing, hasExisting := existing[i]
|
|
|
|
// Run expressions on the nodes
|
|
err = e.runEvals(ctx, hasExisting, n)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if hasExisting {
|
|
// On existing, we don't need to re-do identifiers and references; simply
|
|
// changing up the internal resource is enough.
|
|
//
|
|
// In the future, we can pass down the tree and re-do the deps like that
|
|
switch n.Config.MergeAlg {
|
|
case envoyx.OnConflictPanic:
|
|
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
|
|
return
|
|
|
|
case envoyx.OnConflictReplace:
|
|
// Replace; simple ID change should do the trick
|
|
res.ID = existing.ID
|
|
|
|
case envoyx.OnConflictSkip:
|
|
// Replace the node's resource with the fetched one
|
|
res = &existing
|
|
|
|
// @todo merging
|
|
}
|
|
} else {
|
|
// @todo actually a bottleneck. As per sonyflake docs, it can at most
|
|
// generate up to 2**8 (256) IDs per 10ms in a single thread.
|
|
// How can we improve this?
|
|
res.ID = id.Next()
|
|
}
|
|
|
|
// We can skip validation/defaults when the resource is overwritten by
|
|
// the one already stored (the panic one errors out anyway) since it
|
|
// should already be ok.
|
|
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
|
|
err = e.setTemplateDefaults(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = e.validateTemplate(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
n.Resource = res
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeTemplates encodes a set of resource into the database
|
|
func (e StoreEncoder) encodeTemplates(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
|
|
for _, n := range nn {
|
|
err = e.encodeTemplate(ctx, p, s, n, tree)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeTemplate encodes the resource into the database
|
|
func (e StoreEncoder) encodeTemplate(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
|
|
// Grab dependency references
|
|
var auxID uint64
|
|
err = func() (err error) {
|
|
for fieldLabel, ref := range n.References {
|
|
auxID = safeParentID(tree, n, ref)
|
|
if auxID == 0 {
|
|
continue
|
|
}
|
|
|
|
err = n.Resource.SetValue(fieldLabel, 0, auxID)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
|
|
return
|
|
}
|
|
|
|
// Flush to the DB
|
|
if !n.Evaluated.Skip {
|
|
err = store.UpsertTemplate(ctx, s, n.Resource.(*types.Template))
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to upsert Template")
|
|
return
|
|
}
|
|
}
|
|
|
|
// Handle resources nested under it
|
|
//
|
|
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
|
|
// the root function calls?
|
|
|
|
err = func() (err error) {
|
|
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
|
|
nn = envoyx.OmitPlaceholderNodes(nn...)
|
|
|
|
switch rt {
|
|
|
|
}
|
|
}
|
|
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to encode nested resources")
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// matchupTemplates returns an index with indicates what resources already exist
|
|
func (e StoreEncoder) matchupTemplates(ctx context.Context, s store.Storer, uu map[int]types.Template, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
|
|
// @todo might need to do it smarter then this.
|
|
// Most resources won't really be that vast so this should be acceptable for now.
|
|
aa, _, err := store.SearchTemplates(ctx, s, types.TemplateFilter{})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
idMap := make(map[uint64]*types.Template, len(aa))
|
|
strMap := make(map[string]*types.Template, len(aa))
|
|
|
|
for _, a := range aa {
|
|
strMap[a.Handle] = a
|
|
idMap[a.ID] = a
|
|
|
|
}
|
|
|
|
var aux *types.Template
|
|
var ok bool
|
|
for i, n := range nn {
|
|
|
|
for _, idf := range n.Identifiers.Slice {
|
|
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
|
|
aux, ok = idMap[id]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
|
|
aux, ok = strMap[idf]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
// Functions for resource user
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
|
|
// prepareUser prepares the resources of the given type for encoding
|
|
func (e StoreEncoder) prepareUser(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
|
|
// Grab an index of already existing resources of this type
|
|
// @note since these resources should be fairly low-volume and existing for
|
|
// a short time (and because we batch by resource type); fetching them all
|
|
// into memory shouldn't hurt too much.
|
|
// @todo do some benchmarks and potentially implement some smarter check such as
|
|
// a bloom filter or something similar.
|
|
|
|
// Get node scopes
|
|
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to get scope nodes")
|
|
return
|
|
}
|
|
|
|
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
|
|
existing := make(map[int]types.User, len(nn))
|
|
err = e.matchupUsers(ctx, s, existing, scopedNodes, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to matchup existing Users")
|
|
return
|
|
}
|
|
|
|
for i, n := range nn {
|
|
if n.Resource == nil {
|
|
panic("unexpected state: cannot call prepareUser with nodes without a defined Resource")
|
|
}
|
|
|
|
res, ok := n.Resource.(*types.User)
|
|
if !ok {
|
|
panic("unexpected resource type: node expecting type of user")
|
|
}
|
|
|
|
existing, hasExisting := existing[i]
|
|
|
|
// Run expressions on the nodes
|
|
err = e.runEvals(ctx, hasExisting, n)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if hasExisting {
|
|
// On existing, we don't need to re-do identifiers and references; simply
|
|
// changing up the internal resource is enough.
|
|
//
|
|
// In the future, we can pass down the tree and re-do the deps like that
|
|
switch n.Config.MergeAlg {
|
|
case envoyx.OnConflictPanic:
|
|
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
|
|
return
|
|
|
|
case envoyx.OnConflictReplace:
|
|
// Replace; simple ID change should do the trick
|
|
res.ID = existing.ID
|
|
|
|
case envoyx.OnConflictSkip:
|
|
// Replace the node's resource with the fetched one
|
|
res = &existing
|
|
|
|
// @todo merging
|
|
}
|
|
} else {
|
|
// @todo actually a bottleneck. As per sonyflake docs, it can at most
|
|
// generate up to 2**8 (256) IDs per 10ms in a single thread.
|
|
// How can we improve this?
|
|
res.ID = id.Next()
|
|
}
|
|
|
|
// We can skip validation/defaults when the resource is overwritten by
|
|
// the one already stored (the panic one errors out anyway) since it
|
|
// should already be ok.
|
|
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
|
|
err = e.setUserDefaults(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = e.validateUser(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
n.Resource = res
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeUsers encodes a set of resource into the database
|
|
func (e StoreEncoder) encodeUsers(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
|
|
for _, n := range nn {
|
|
err = e.encodeUser(ctx, p, s, n, tree)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeUser encodes the resource into the database
|
|
func (e StoreEncoder) encodeUser(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
|
|
// Grab dependency references
|
|
var auxID uint64
|
|
err = func() (err error) {
|
|
for fieldLabel, ref := range n.References {
|
|
auxID = safeParentID(tree, n, ref)
|
|
if auxID == 0 {
|
|
continue
|
|
}
|
|
|
|
err = n.Resource.SetValue(fieldLabel, 0, auxID)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
|
|
return
|
|
}
|
|
|
|
// Flush to the DB
|
|
if !n.Evaluated.Skip {
|
|
err = store.UpsertUser(ctx, s, n.Resource.(*types.User))
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to upsert User")
|
|
return
|
|
}
|
|
}
|
|
|
|
// Handle resources nested under it
|
|
//
|
|
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
|
|
// the root function calls?
|
|
|
|
err = func() (err error) {
|
|
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
|
|
nn = envoyx.OmitPlaceholderNodes(nn...)
|
|
|
|
switch rt {
|
|
|
|
}
|
|
}
|
|
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to encode nested resources")
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// matchupUsers returns an index with indicates what resources already exist
|
|
func (e StoreEncoder) matchupUsers(ctx context.Context, s store.Storer, uu map[int]types.User, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
|
|
// @todo might need to do it smarter then this.
|
|
// Most resources won't really be that vast so this should be acceptable for now.
|
|
aa, _, err := store.SearchUsers(ctx, s, types.UserFilter{})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
idMap := make(map[uint64]*types.User, len(aa))
|
|
strMap := make(map[string]*types.User, len(aa))
|
|
|
|
for _, a := range aa {
|
|
strMap[a.Handle] = a
|
|
idMap[a.ID] = a
|
|
|
|
}
|
|
|
|
var aux *types.User
|
|
var ok bool
|
|
for i, n := range nn {
|
|
|
|
for _, idf := range n.Identifiers.Slice {
|
|
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
|
|
aux, ok = idMap[id]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
|
|
aux, ok = strMap[idf]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
// Functions for resource dalConnection
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
|
|
// prepareDalConnection prepares the resources of the given type for encoding
|
|
func (e StoreEncoder) prepareDalConnection(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
|
|
// Grab an index of already existing resources of this type
|
|
// @note since these resources should be fairly low-volume and existing for
|
|
// a short time (and because we batch by resource type); fetching them all
|
|
// into memory shouldn't hurt too much.
|
|
// @todo do some benchmarks and potentially implement some smarter check such as
|
|
// a bloom filter or something similar.
|
|
|
|
// Get node scopes
|
|
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to get scope nodes")
|
|
return
|
|
}
|
|
|
|
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
|
|
existing := make(map[int]types.DalConnection, len(nn))
|
|
err = e.matchupDalConnections(ctx, s, existing, scopedNodes, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to matchup existing DalConnections")
|
|
return
|
|
}
|
|
|
|
for i, n := range nn {
|
|
if n.Resource == nil {
|
|
panic("unexpected state: cannot call prepareDalConnection with nodes without a defined Resource")
|
|
}
|
|
|
|
res, ok := n.Resource.(*types.DalConnection)
|
|
if !ok {
|
|
panic("unexpected resource type: node expecting type of dalConnection")
|
|
}
|
|
|
|
existing, hasExisting := existing[i]
|
|
|
|
// Run expressions on the nodes
|
|
err = e.runEvals(ctx, hasExisting, n)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if hasExisting {
|
|
// On existing, we don't need to re-do identifiers and references; simply
|
|
// changing up the internal resource is enough.
|
|
//
|
|
// In the future, we can pass down the tree and re-do the deps like that
|
|
switch n.Config.MergeAlg {
|
|
case envoyx.OnConflictPanic:
|
|
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
|
|
return
|
|
|
|
case envoyx.OnConflictReplace:
|
|
// Replace; simple ID change should do the trick
|
|
res.ID = existing.ID
|
|
|
|
case envoyx.OnConflictSkip:
|
|
// Replace the node's resource with the fetched one
|
|
res = &existing
|
|
|
|
// @todo merging
|
|
}
|
|
} else {
|
|
// @todo actually a bottleneck. As per sonyflake docs, it can at most
|
|
// generate up to 2**8 (256) IDs per 10ms in a single thread.
|
|
// How can we improve this?
|
|
res.ID = id.Next()
|
|
}
|
|
|
|
// We can skip validation/defaults when the resource is overwritten by
|
|
// the one already stored (the panic one errors out anyway) since it
|
|
// should already be ok.
|
|
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
|
|
err = e.setDalConnectionDefaults(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = e.validateDalConnection(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
n.Resource = res
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeDalConnections encodes a set of resource into the database
|
|
func (e StoreEncoder) encodeDalConnections(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
|
|
for _, n := range nn {
|
|
err = e.encodeDalConnection(ctx, p, s, n, tree)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeDalConnection encodes the resource into the database
|
|
func (e StoreEncoder) encodeDalConnection(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
|
|
// Grab dependency references
|
|
var auxID uint64
|
|
err = func() (err error) {
|
|
for fieldLabel, ref := range n.References {
|
|
auxID = safeParentID(tree, n, ref)
|
|
if auxID == 0 {
|
|
continue
|
|
}
|
|
|
|
err = n.Resource.SetValue(fieldLabel, 0, auxID)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
|
|
return
|
|
}
|
|
|
|
// Flush to the DB
|
|
if !n.Evaluated.Skip {
|
|
err = store.UpsertDalConnection(ctx, s, n.Resource.(*types.DalConnection))
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to upsert DalConnection")
|
|
return
|
|
}
|
|
}
|
|
|
|
// Handle resources nested under it
|
|
//
|
|
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
|
|
// the root function calls?
|
|
|
|
err = func() (err error) {
|
|
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
|
|
nn = envoyx.OmitPlaceholderNodes(nn...)
|
|
|
|
switch rt {
|
|
|
|
}
|
|
}
|
|
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to encode nested resources")
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// matchupDalConnections returns an index with indicates what resources already exist
|
|
func (e StoreEncoder) matchupDalConnections(ctx context.Context, s store.Storer, uu map[int]types.DalConnection, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
|
|
// @todo might need to do it smarter then this.
|
|
// Most resources won't really be that vast so this should be acceptable for now.
|
|
aa, _, err := store.SearchDalConnections(ctx, s, types.DalConnectionFilter{})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
idMap := make(map[uint64]*types.DalConnection, len(aa))
|
|
strMap := make(map[string]*types.DalConnection, len(aa))
|
|
|
|
for _, a := range aa {
|
|
strMap[a.Handle] = a
|
|
idMap[a.ID] = a
|
|
|
|
}
|
|
|
|
var aux *types.DalConnection
|
|
var ok bool
|
|
for i, n := range nn {
|
|
|
|
for _, idf := range n.Identifiers.Slice {
|
|
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
|
|
aux, ok = idMap[id]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
|
|
aux, ok = strMap[idf]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
// Functions for resource dalSensitivityLevel
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
|
|
// prepareDalSensitivityLevel prepares the resources of the given type for encoding
|
|
func (e StoreEncoder) prepareDalSensitivityLevel(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
|
|
// Grab an index of already existing resources of this type
|
|
// @note since these resources should be fairly low-volume and existing for
|
|
// a short time (and because we batch by resource type); fetching them all
|
|
// into memory shouldn't hurt too much.
|
|
// @todo do some benchmarks and potentially implement some smarter check such as
|
|
// a bloom filter or something similar.
|
|
|
|
// Get node scopes
|
|
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to get scope nodes")
|
|
return
|
|
}
|
|
|
|
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
|
|
existing := make(map[int]types.DalSensitivityLevel, len(nn))
|
|
err = e.matchupDalSensitivityLevels(ctx, s, existing, scopedNodes, nn)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to matchup existing DalSensitivityLevels")
|
|
return
|
|
}
|
|
|
|
for i, n := range nn {
|
|
if n.Resource == nil {
|
|
panic("unexpected state: cannot call prepareDalSensitivityLevel with nodes without a defined Resource")
|
|
}
|
|
|
|
res, ok := n.Resource.(*types.DalSensitivityLevel)
|
|
if !ok {
|
|
panic("unexpected resource type: node expecting type of dalSensitivityLevel")
|
|
}
|
|
|
|
existing, hasExisting := existing[i]
|
|
|
|
// Run expressions on the nodes
|
|
err = e.runEvals(ctx, hasExisting, n)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if hasExisting {
|
|
// On existing, we don't need to re-do identifiers and references; simply
|
|
// changing up the internal resource is enough.
|
|
//
|
|
// In the future, we can pass down the tree and re-do the deps like that
|
|
switch n.Config.MergeAlg {
|
|
case envoyx.OnConflictPanic:
|
|
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
|
|
return
|
|
|
|
case envoyx.OnConflictReplace:
|
|
// Replace; simple ID change should do the trick
|
|
res.ID = existing.ID
|
|
|
|
case envoyx.OnConflictSkip:
|
|
// Replace the node's resource with the fetched one
|
|
res = &existing
|
|
|
|
// @todo merging
|
|
}
|
|
} else {
|
|
// @todo actually a bottleneck. As per sonyflake docs, it can at most
|
|
// generate up to 2**8 (256) IDs per 10ms in a single thread.
|
|
// How can we improve this?
|
|
res.ID = id.Next()
|
|
}
|
|
|
|
// We can skip validation/defaults when the resource is overwritten by
|
|
// the one already stored (the panic one errors out anyway) since it
|
|
// should already be ok.
|
|
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
|
|
err = e.setDalSensitivityLevelDefaults(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = e.validateDalSensitivityLevel(res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
n.Resource = res
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeDalSensitivityLevels encodes a set of resource into the database
|
|
func (e StoreEncoder) encodeDalSensitivityLevels(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
|
|
for _, n := range nn {
|
|
err = e.encodeDalSensitivityLevel(ctx, p, s, n, tree)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// encodeDalSensitivityLevel encodes the resource into the database
|
|
func (e StoreEncoder) encodeDalSensitivityLevel(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
|
|
// Grab dependency references
|
|
var auxID uint64
|
|
err = func() (err error) {
|
|
for fieldLabel, ref := range n.References {
|
|
auxID = safeParentID(tree, n, ref)
|
|
if auxID == 0 {
|
|
continue
|
|
}
|
|
|
|
err = n.Resource.SetValue(fieldLabel, 0, auxID)
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
|
|
return
|
|
}
|
|
|
|
// Flush to the DB
|
|
if !n.Evaluated.Skip {
|
|
err = store.UpsertDalSensitivityLevel(ctx, s, n.Resource.(*types.DalSensitivityLevel))
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to upsert DalSensitivityLevel")
|
|
return
|
|
}
|
|
}
|
|
|
|
// Handle resources nested under it
|
|
//
|
|
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
|
|
// the root function calls?
|
|
|
|
err = func() (err error) {
|
|
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
|
|
nn = envoyx.OmitPlaceholderNodes(nn...)
|
|
|
|
switch rt {
|
|
|
|
}
|
|
}
|
|
|
|
return
|
|
}()
|
|
if err != nil {
|
|
err = errors.Wrap(err, "failed to encode nested resources")
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// matchupDalSensitivityLevels returns an index with indicates what resources already exist
|
|
func (e StoreEncoder) matchupDalSensitivityLevels(ctx context.Context, s store.Storer, uu map[int]types.DalSensitivityLevel, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
|
|
// @todo might need to do it smarter then this.
|
|
// Most resources won't really be that vast so this should be acceptable for now.
|
|
aa, _, err := store.SearchDalSensitivityLevels(ctx, s, types.DalSensitivityLevelFilter{})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
idMap := make(map[uint64]*types.DalSensitivityLevel, len(aa))
|
|
strMap := make(map[string]*types.DalSensitivityLevel, len(aa))
|
|
|
|
for _, a := range aa {
|
|
strMap[a.Handle] = a
|
|
idMap[a.ID] = a
|
|
|
|
}
|
|
|
|
var aux *types.DalSensitivityLevel
|
|
var ok bool
|
|
for i, n := range nn {
|
|
|
|
for _, idf := range n.Identifiers.Slice {
|
|
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
|
|
aux, ok = idMap[id]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
|
|
aux, ok = strMap[idf]
|
|
if ok {
|
|
uu[i] = *aux
|
|
// When any identifier matches we can end it
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
// Utility functions
|
|
// // // // // // // // // // // // // // // // // // // // // // // // //
|
|
|
|
func (e *StoreEncoder) grabStorer(p envoyx.EncodeParams) (s store.Storer, err error) {
|
|
auxs, ok := p.Params[paramsKeyStorer]
|
|
if !ok {
|
|
err = errors.Errorf("store encoder expects a store conforming to store.Storer interface")
|
|
return
|
|
}
|
|
|
|
s, ok = auxs.(store.Storer)
|
|
if !ok {
|
|
err = errors.Errorf("store encoder expects a store conforming to store.Storer interface")
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func (e *StoreEncoder) runEvals(ctx context.Context, existing bool, n *envoyx.Node) (err error) {
|
|
// Skip if
|
|
if n.Config.SkipIfEval == nil {
|
|
return
|
|
}
|
|
|
|
aux, err := expr.EmptyVars().Cast(map[string]any{
|
|
"missing": !existing,
|
|
})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
n.Evaluated.Skip, err = n.Config.SkipIfEval.Test(ctx, aux.(*expr.Vars))
|
|
return
|
|
}
|
|
|
|
func (e StoreEncoder) getScopeNodes(ctx context.Context, s store.Storer, nn envoyx.NodeSet) (scopes envoyx.NodeSet, err error) {
|
|
// Get all requested scopes
|
|
scopes = make(envoyx.NodeSet, len(nn))
|
|
|
|
// @note skipping scope logic since it's currently only supported within
|
|
// Compose resources.
|
|
|
|
return
|
|
}
|
|
|
|
func safeParentID(tt envoyx.Traverser, n *envoyx.Node, ref envoyx.Ref) (out uint64) {
|
|
rn := tt.ParentForRef(n, ref)
|
|
if rn == nil {
|
|
return
|
|
}
|
|
|
|
return rn.Resource.GetID()
|
|
}
|