Files
upstream-corteza/server/system/envoy/store_encode.gen.go
upstream-mirror 7e385fc126
Some checks failed
SonarQube / SonarQube (push) Failing after 5m28s
E2e test on main branch push / server and client test (map[name:admin port:8080]) (push) Failing after 3m33s
Server and lib tests / Library test (js) (push) Failing after 1m36s
Server and lib tests / Library test (vue) (push) Successful in 2m24s
Server and lib tests / Web client test (admin) (push) Successful in 2m52s
Server and lib tests / Web client test (compose) (push) Successful in 3m23s
Server and lib tests / Web client test (discovery) (push) Successful in 2m49s
Server and lib tests / Web client test (one) (push) Successful in 2m52s
Server and lib tests / Web client test (privacy) (push) Successful in 2m44s
Server and lib tests / Web client test (reporter) (push) Successful in 2m56s
Server and lib tests / Web client test (workflow) (push) Successful in 3m0s
Server and lib tests / Server tests (push) Failing after 4m55s
E2e test on main branch push / notify-failure (push) Failing after 9s
Server and lib tests / Send matrix testing notification (push) Successful in 2s
Mark stale issues and pull requests / stale (push) Successful in 40s
snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)
2026-09-18 01:22:23 +05:00

2459 lines
67 KiB
Go
Generated

package envoy
// This file is auto-generated.
//
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
//
import (
"context"
"fmt"
"strconv"
"github.com/cortezaproject/corteza/server/pkg/envoyx"
"github.com/cortezaproject/corteza/server/pkg/expr"
"github.com/cortezaproject/corteza/server/pkg/id"
"github.com/cortezaproject/corteza/server/store"
"github.com/cortezaproject/corteza/server/system/types"
"github.com/pkg/errors"
)
type (
// StoreEncoder is responsible for encoding Corteza resources into the
// database via the Storer or the DAL interface
//
// @todo consider having a different encoder for the DAL resources
StoreEncoder struct{}
)
// Prepare performs some initial processing on the resource before it can be encoded
//
// Preparation runs validation, default value initialization, matching with
// already existing instances, ...
//
// The prepare function receives a set of nodes grouped by the resource type.
// This enables some batching optimization and simplifications when it comes to
// matching with existing resources.
//
// Prepare does not receive any placeholder nodes which are used solely
// for dependency resolution.
func (e StoreEncoder) Prepare(ctx context.Context, p envoyx.EncodeParams, rt string, nn envoyx.NodeSet) (err error) {
s, err := e.grabStorer(p)
if err != nil {
return
}
switch rt {
case types.ApplicationResourceType:
return e.prepareApplication(ctx, p, s, nn)
case types.ApigwRouteResourceType:
return e.prepareApigwRoute(ctx, p, s, nn)
case types.ApigwFilterResourceType:
return e.prepareApigwFilter(ctx, p, s, nn)
case types.AuthClientResourceType:
return e.prepareAuthClient(ctx, p, s, nn)
case types.QueueResourceType:
return e.prepareQueue(ctx, p, s, nn)
case types.ReportResourceType:
return e.prepareReport(ctx, p, s, nn)
case types.RoleResourceType:
return e.prepareRole(ctx, p, s, nn)
case types.TemplateResourceType:
return e.prepareTemplate(ctx, p, s, nn)
case types.UserResourceType:
return e.prepareUser(ctx, p, s, nn)
case types.DalConnectionResourceType:
return e.prepareDalConnection(ctx, p, s, nn)
case types.DalSensitivityLevelResourceType:
return e.prepareDalSensitivityLevel(ctx, p, s, nn)
default:
return e.prepare(ctx, p, s, rt, nn)
}
return
}
// Encode encodes the given Corteza resources into the primary store
//
// Encoding should not do any additional processing apart from matching with
// dependencies and runtime validation
//
// The Encode function is called for every resource type where the resource
// appears at the root of the dependency tree.
// All of the root-level resources for that resource type are passed into the function.
// The encoding function must traverse the branches to encode all of the dependencies.
//
// This flow is used to simplify the flow of how resources are encoded into YAML
// (and other documents) as well as to simplify batching.
//
// Encode does not receive any placeholder nodes which are used solely
// for dependency resolution.
func (e StoreEncoder) Encode(ctx context.Context, p envoyx.EncodeParams, rt string, nodes envoyx.NodeSet, tree envoyx.Traverser) (err error) {
s, err := e.grabStorer(p)
if err != nil {
return
}
switch rt {
case types.ApplicationResourceType:
return e.encodeApplications(ctx, p, s, nodes, tree)
case types.ApigwRouteResourceType:
return e.encodeApigwRoutes(ctx, p, s, nodes, tree)
case types.ApigwFilterResourceType:
return e.encodeApigwFilters(ctx, p, s, nodes, tree)
case types.AuthClientResourceType:
return e.encodeAuthClients(ctx, p, s, nodes, tree)
case types.QueueResourceType:
return e.encodeQueues(ctx, p, s, nodes, tree)
case types.ReportResourceType:
return e.encodeReports(ctx, p, s, nodes, tree)
case types.RoleResourceType:
return e.encodeRoles(ctx, p, s, nodes, tree)
case types.TemplateResourceType:
return e.encodeTemplates(ctx, p, s, nodes, tree)
case types.UserResourceType:
return e.encodeUsers(ctx, p, s, nodes, tree)
case types.DalConnectionResourceType:
return e.encodeDalConnections(ctx, p, s, nodes, tree)
case types.DalSensitivityLevelResourceType:
return e.encodeDalSensitivityLevels(ctx, p, s, nodes, tree)
default:
return e.encode(ctx, p, s, rt, nodes, tree)
}
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Functions for resource application
// // // // // // // // // // // // // // // // // // // // // // // // //
// prepareApplication prepares the resources of the given type for encoding
func (e StoreEncoder) prepareApplication(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
// Grab an index of already existing resources of this type
// @note since these resources should be fairly low-volume and existing for
// a short time (and because we batch by resource type); fetching them all
// into memory shouldn't hurt too much.
// @todo do some benchmarks and potentially implement some smarter check such as
// a bloom filter or something similar.
// Get node scopes
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
if err != nil {
err = errors.Wrap(err, "failed to get scope nodes")
return
}
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
existing := make(map[int]types.Application, len(nn))
err = e.matchupApplications(ctx, s, existing, scopedNodes, nn)
if err != nil {
err = errors.Wrap(err, "failed to matchup existing Applications")
return
}
for i, n := range nn {
if n.Resource == nil {
panic("unexpected state: cannot call prepareApplication with nodes without a defined Resource")
}
res, ok := n.Resource.(*types.Application)
if !ok {
panic("unexpected resource type: node expecting type of application")
}
existing, hasExisting := existing[i]
// Run expressions on the nodes
err = e.runEvals(ctx, hasExisting, n)
if err != nil {
return
}
if hasExisting {
// On existing, we don't need to re-do identifiers and references; simply
// changing up the internal resource is enough.
//
// In the future, we can pass down the tree and re-do the deps like that
switch n.Config.MergeAlg {
case envoyx.OnConflictPanic:
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
return
case envoyx.OnConflictReplace:
// Replace; simple ID change should do the trick
res.ID = existing.ID
case envoyx.OnConflictSkip:
// Replace the node's resource with the fetched one
res = &existing
// @todo merging
}
} else {
// @todo actually a bottleneck. As per sonyflake docs, it can at most
// generate up to 2**8 (256) IDs per 10ms in a single thread.
// How can we improve this?
res.ID = id.Next()
}
// We can skip validation/defaults when the resource is overwritten by
// the one already stored (the panic one errors out anyway) since it
// should already be ok.
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
err = e.setApplicationDefaults(res)
if err != nil {
return err
}
err = e.validateApplication(res)
if err != nil {
return err
}
}
n.Resource = res
}
return
}
// encodeApplications encodes a set of resource into the database
func (e StoreEncoder) encodeApplications(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
for _, n := range nn {
err = e.encodeApplication(ctx, p, s, n, tree)
if err != nil {
return
}
}
return
}
// encodeApplication encodes the resource into the database
func (e StoreEncoder) encodeApplication(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
// Grab dependency references
var auxID uint64
err = func() (err error) {
for fieldLabel, ref := range n.References {
auxID = safeParentID(tree, n, ref)
if auxID == 0 {
continue
}
err = n.Resource.SetValue(fieldLabel, 0, auxID)
if err != nil {
return
}
}
return
}()
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
return
}
// Flush to the DB
if !n.Evaluated.Skip {
err = store.UpsertApplication(ctx, s, n.Resource.(*types.Application))
if err != nil {
err = errors.Wrap(err, "failed to upsert Application")
return
}
}
// Handle resources nested under it
//
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
// the root function calls?
err = func() (err error) {
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
nn = envoyx.OmitPlaceholderNodes(nn...)
switch rt {
}
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to encode nested resources")
return
}
return
}
// matchupApplications returns an index with indicates what resources already exist
func (e StoreEncoder) matchupApplications(ctx context.Context, s store.Storer, uu map[int]types.Application, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
// @todo might need to do it smarter then this.
// Most resources won't really be that vast so this should be acceptable for now.
aa, _, err := store.SearchApplications(ctx, s, types.ApplicationFilter{})
if err != nil {
return
}
idMap := make(map[uint64]*types.Application, len(aa))
strMap := make(map[string]*types.Application, len(aa))
for _, a := range aa {
idMap[a.ID] = a
}
var aux *types.Application
var ok bool
for i, n := range nn {
for _, idf := range n.Identifiers.Slice {
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
aux, ok = idMap[id]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
aux, ok = strMap[idf]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
}
return
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Functions for resource apigwRoute
// // // // // // // // // // // // // // // // // // // // // // // // //
// prepareApigwRoute prepares the resources of the given type for encoding
func (e StoreEncoder) prepareApigwRoute(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
// Grab an index of already existing resources of this type
// @note since these resources should be fairly low-volume and existing for
// a short time (and because we batch by resource type); fetching them all
// into memory shouldn't hurt too much.
// @todo do some benchmarks and potentially implement some smarter check such as
// a bloom filter or something similar.
// Get node scopes
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
if err != nil {
err = errors.Wrap(err, "failed to get scope nodes")
return
}
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
existing := make(map[int]types.ApigwRoute, len(nn))
err = e.matchupApigwRoutes(ctx, s, existing, scopedNodes, nn)
if err != nil {
err = errors.Wrap(err, "failed to matchup existing ApigwRoutes")
return
}
for i, n := range nn {
if n.Resource == nil {
panic("unexpected state: cannot call prepareApigwRoute with nodes without a defined Resource")
}
res, ok := n.Resource.(*types.ApigwRoute)
if !ok {
panic("unexpected resource type: node expecting type of apigwRoute")
}
existing, hasExisting := existing[i]
// Run expressions on the nodes
err = e.runEvals(ctx, hasExisting, n)
if err != nil {
return
}
if hasExisting {
// On existing, we don't need to re-do identifiers and references; simply
// changing up the internal resource is enough.
//
// In the future, we can pass down the tree and re-do the deps like that
switch n.Config.MergeAlg {
case envoyx.OnConflictPanic:
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
return
case envoyx.OnConflictReplace:
// Replace; simple ID change should do the trick
res.ID = existing.ID
case envoyx.OnConflictSkip:
// Replace the node's resource with the fetched one
res = &existing
// @todo merging
}
} else {
// @todo actually a bottleneck. As per sonyflake docs, it can at most
// generate up to 2**8 (256) IDs per 10ms in a single thread.
// How can we improve this?
res.ID = id.Next()
}
// We can skip validation/defaults when the resource is overwritten by
// the one already stored (the panic one errors out anyway) since it
// should already be ok.
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
err = e.setApigwRouteDefaults(res)
if err != nil {
return err
}
err = e.validateApigwRoute(res)
if err != nil {
return err
}
}
n.Resource = res
}
return
}
// encodeApigwRoutes encodes a set of resource into the database
func (e StoreEncoder) encodeApigwRoutes(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
for _, n := range nn {
err = e.encodeApigwRoute(ctx, p, s, n, tree)
if err != nil {
return
}
}
return
}
// encodeApigwRoute encodes the resource into the database
func (e StoreEncoder) encodeApigwRoute(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
// Grab dependency references
var auxID uint64
err = func() (err error) {
for fieldLabel, ref := range n.References {
auxID = safeParentID(tree, n, ref)
if auxID == 0 {
continue
}
err = n.Resource.SetValue(fieldLabel, 0, auxID)
if err != nil {
return
}
}
return
}()
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
return
}
// Flush to the DB
if !n.Evaluated.Skip {
err = store.UpsertApigwRoute(ctx, s, n.Resource.(*types.ApigwRoute))
if err != nil {
err = errors.Wrap(err, "failed to upsert ApigwRoute")
return
}
}
// Handle resources nested under it
//
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
// the root function calls?
err = func() (err error) {
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
nn = envoyx.OmitPlaceholderNodes(nn...)
switch rt {
}
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to encode nested resources")
return
}
return
}
// matchupApigwRoutes returns an index with indicates what resources already exist
func (e StoreEncoder) matchupApigwRoutes(ctx context.Context, s store.Storer, uu map[int]types.ApigwRoute, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
// @todo might need to do it smarter then this.
// Most resources won't really be that vast so this should be acceptable for now.
aa, _, err := store.SearchApigwRoutes(ctx, s, types.ApigwRouteFilter{})
if err != nil {
return
}
idMap := make(map[uint64]*types.ApigwRoute, len(aa))
strMap := make(map[string]*types.ApigwRoute, len(aa))
for _, a := range aa {
idMap[a.ID] = a
}
var aux *types.ApigwRoute
var ok bool
for i, n := range nn {
for _, idf := range n.Identifiers.Slice {
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
aux, ok = idMap[id]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
aux, ok = strMap[idf]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
}
return
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Functions for resource apigwFilter
// // // // // // // // // // // // // // // // // // // // // // // // //
// prepareApigwFilter prepares the resources of the given type for encoding
func (e StoreEncoder) prepareApigwFilter(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
// Grab an index of already existing resources of this type
// @note since these resources should be fairly low-volume and existing for
// a short time (and because we batch by resource type); fetching them all
// into memory shouldn't hurt too much.
// @todo do some benchmarks and potentially implement some smarter check such as
// a bloom filter or something similar.
// Get node scopes
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
if err != nil {
err = errors.Wrap(err, "failed to get scope nodes")
return
}
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
existing := make(map[int]types.ApigwFilter, len(nn))
err = e.matchupApigwFilters(ctx, s, existing, scopedNodes, nn)
if err != nil {
err = errors.Wrap(err, "failed to matchup existing ApigwFilters")
return
}
for i, n := range nn {
if n.Resource == nil {
panic("unexpected state: cannot call prepareApigwFilter with nodes without a defined Resource")
}
res, ok := n.Resource.(*types.ApigwFilter)
if !ok {
panic("unexpected resource type: node expecting type of apigwFilter")
}
existing, hasExisting := existing[i]
// Run expressions on the nodes
err = e.runEvals(ctx, hasExisting, n)
if err != nil {
return
}
if hasExisting {
// On existing, we don't need to re-do identifiers and references; simply
// changing up the internal resource is enough.
//
// In the future, we can pass down the tree and re-do the deps like that
switch n.Config.MergeAlg {
case envoyx.OnConflictPanic:
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
return
case envoyx.OnConflictReplace:
// Replace; simple ID change should do the trick
res.ID = existing.ID
case envoyx.OnConflictSkip:
// Replace the node's resource with the fetched one
res = &existing
// @todo merging
}
} else {
// @todo actually a bottleneck. As per sonyflake docs, it can at most
// generate up to 2**8 (256) IDs per 10ms in a single thread.
// How can we improve this?
res.ID = id.Next()
}
// We can skip validation/defaults when the resource is overwritten by
// the one already stored (the panic one errors out anyway) since it
// should already be ok.
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
err = e.setApigwFilterDefaults(res)
if err != nil {
return err
}
err = e.validateApigwFilter(res)
if err != nil {
return err
}
}
n.Resource = res
}
return
}
// encodeApigwFilters encodes a set of resource into the database
func (e StoreEncoder) encodeApigwFilters(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
for _, n := range nn {
err = e.encodeApigwFilter(ctx, p, s, n, tree)
if err != nil {
return
}
}
return
}
// encodeApigwFilter encodes the resource into the database
func (e StoreEncoder) encodeApigwFilter(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
// Grab dependency references
var auxID uint64
err = func() (err error) {
for fieldLabel, ref := range n.References {
auxID = safeParentID(tree, n, ref)
if auxID == 0 {
continue
}
err = n.Resource.SetValue(fieldLabel, 0, auxID)
if err != nil {
return
}
}
return
}()
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
return
}
// Flush to the DB
if !n.Evaluated.Skip {
err = store.UpsertApigwFilter(ctx, s, n.Resource.(*types.ApigwFilter))
if err != nil {
err = errors.Wrap(err, "failed to upsert ApigwFilter")
return
}
}
// Handle resources nested under it
//
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
// the root function calls?
err = func() (err error) {
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
nn = envoyx.OmitPlaceholderNodes(nn...)
switch rt {
}
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to encode nested resources")
return
}
return
}
// matchupApigwFilters returns an index with indicates what resources already exist
func (e StoreEncoder) matchupApigwFilters(ctx context.Context, s store.Storer, uu map[int]types.ApigwFilter, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
// @todo might need to do it smarter then this.
// Most resources won't really be that vast so this should be acceptable for now.
aa, _, err := store.SearchApigwFilters(ctx, s, types.ApigwFilterFilter{})
if err != nil {
return
}
idMap := make(map[uint64]*types.ApigwFilter, len(aa))
strMap := make(map[string]*types.ApigwFilter, len(aa))
for _, a := range aa {
idMap[a.ID] = a
}
var aux *types.ApigwFilter
var ok bool
for i, n := range nn {
for _, idf := range n.Identifiers.Slice {
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
aux, ok = idMap[id]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
aux, ok = strMap[idf]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
}
return
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Functions for resource authClient
// // // // // // // // // // // // // // // // // // // // // // // // //
// prepareAuthClient prepares the resources of the given type for encoding
func (e StoreEncoder) prepareAuthClient(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
// Grab an index of already existing resources of this type
// @note since these resources should be fairly low-volume and existing for
// a short time (and because we batch by resource type); fetching them all
// into memory shouldn't hurt too much.
// @todo do some benchmarks and potentially implement some smarter check such as
// a bloom filter or something similar.
// Get node scopes
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
if err != nil {
err = errors.Wrap(err, "failed to get scope nodes")
return
}
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
existing := make(map[int]types.AuthClient, len(nn))
err = e.matchupAuthClients(ctx, s, existing, scopedNodes, nn)
if err != nil {
err = errors.Wrap(err, "failed to matchup existing AuthClients")
return
}
for i, n := range nn {
if n.Resource == nil {
panic("unexpected state: cannot call prepareAuthClient with nodes without a defined Resource")
}
res, ok := n.Resource.(*types.AuthClient)
if !ok {
panic("unexpected resource type: node expecting type of authClient")
}
existing, hasExisting := existing[i]
// Run expressions on the nodes
err = e.runEvals(ctx, hasExisting, n)
if err != nil {
return
}
if hasExisting {
// On existing, we don't need to re-do identifiers and references; simply
// changing up the internal resource is enough.
//
// In the future, we can pass down the tree and re-do the deps like that
switch n.Config.MergeAlg {
case envoyx.OnConflictPanic:
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
return
case envoyx.OnConflictReplace:
// Replace; simple ID change should do the trick
res.ID = existing.ID
case envoyx.OnConflictSkip:
// Replace the node's resource with the fetched one
res = &existing
// @todo merging
}
} else {
// @todo actually a bottleneck. As per sonyflake docs, it can at most
// generate up to 2**8 (256) IDs per 10ms in a single thread.
// How can we improve this?
res.ID = id.Next()
}
// We can skip validation/defaults when the resource is overwritten by
// the one already stored (the panic one errors out anyway) since it
// should already be ok.
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
err = e.setAuthClientDefaults(res)
if err != nil {
return err
}
err = e.validateAuthClient(res)
if err != nil {
return err
}
}
n.Resource = res
}
return
}
// encodeAuthClients encodes a set of resource into the database
func (e StoreEncoder) encodeAuthClients(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
for _, n := range nn {
err = e.encodeAuthClient(ctx, p, s, n, tree)
if err != nil {
return
}
}
return
}
// encodeAuthClient encodes the resource into the database
func (e StoreEncoder) encodeAuthClient(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
// Grab dependency references
var auxID uint64
err = func() (err error) {
for fieldLabel, ref := range n.References {
auxID = safeParentID(tree, n, ref)
if auxID == 0 {
continue
}
err = n.Resource.SetValue(fieldLabel, 0, auxID)
if err != nil {
return
}
}
return
}()
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
return
}
// Flush to the DB
if !n.Evaluated.Skip {
err = store.UpsertAuthClient(ctx, s, n.Resource.(*types.AuthClient))
if err != nil {
err = errors.Wrap(err, "failed to upsert AuthClient")
return
}
}
// Handle resources nested under it
//
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
// the root function calls?
err = func() (err error) {
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
nn = envoyx.OmitPlaceholderNodes(nn...)
switch rt {
}
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to encode nested resources")
return
}
return
}
// matchupAuthClients returns an index with indicates what resources already exist
func (e StoreEncoder) matchupAuthClients(ctx context.Context, s store.Storer, uu map[int]types.AuthClient, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
// @todo might need to do it smarter then this.
// Most resources won't really be that vast so this should be acceptable for now.
aa, _, err := store.SearchAuthClients(ctx, s, types.AuthClientFilter{})
if err != nil {
return
}
idMap := make(map[uint64]*types.AuthClient, len(aa))
strMap := make(map[string]*types.AuthClient, len(aa))
for _, a := range aa {
strMap[a.Handle] = a
idMap[a.ID] = a
}
var aux *types.AuthClient
var ok bool
for i, n := range nn {
for _, idf := range n.Identifiers.Slice {
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
aux, ok = idMap[id]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
aux, ok = strMap[idf]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
}
return
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Functions for resource queue
// // // // // // // // // // // // // // // // // // // // // // // // //
// prepareQueue prepares the resources of the given type for encoding
func (e StoreEncoder) prepareQueue(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
// Grab an index of already existing resources of this type
// @note since these resources should be fairly low-volume and existing for
// a short time (and because we batch by resource type); fetching them all
// into memory shouldn't hurt too much.
// @todo do some benchmarks and potentially implement some smarter check such as
// a bloom filter or something similar.
// Get node scopes
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
if err != nil {
err = errors.Wrap(err, "failed to get scope nodes")
return
}
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
existing := make(map[int]types.Queue, len(nn))
err = e.matchupQueues(ctx, s, existing, scopedNodes, nn)
if err != nil {
err = errors.Wrap(err, "failed to matchup existing Queues")
return
}
for i, n := range nn {
if n.Resource == nil {
panic("unexpected state: cannot call prepareQueue with nodes without a defined Resource")
}
res, ok := n.Resource.(*types.Queue)
if !ok {
panic("unexpected resource type: node expecting type of queue")
}
existing, hasExisting := existing[i]
// Run expressions on the nodes
err = e.runEvals(ctx, hasExisting, n)
if err != nil {
return
}
if hasExisting {
// On existing, we don't need to re-do identifiers and references; simply
// changing up the internal resource is enough.
//
// In the future, we can pass down the tree and re-do the deps like that
switch n.Config.MergeAlg {
case envoyx.OnConflictPanic:
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
return
case envoyx.OnConflictReplace:
// Replace; simple ID change should do the trick
res.ID = existing.ID
case envoyx.OnConflictSkip:
// Replace the node's resource with the fetched one
res = &existing
// @todo merging
}
} else {
// @todo actually a bottleneck. As per sonyflake docs, it can at most
// generate up to 2**8 (256) IDs per 10ms in a single thread.
// How can we improve this?
res.ID = id.Next()
}
// We can skip validation/defaults when the resource is overwritten by
// the one already stored (the panic one errors out anyway) since it
// should already be ok.
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
err = e.setQueueDefaults(res)
if err != nil {
return err
}
err = e.validateQueue(res)
if err != nil {
return err
}
}
n.Resource = res
}
return
}
// encodeQueues encodes a set of resource into the database
func (e StoreEncoder) encodeQueues(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
for _, n := range nn {
err = e.encodeQueue(ctx, p, s, n, tree)
if err != nil {
return
}
}
return
}
// encodeQueue encodes the resource into the database
func (e StoreEncoder) encodeQueue(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
// Grab dependency references
var auxID uint64
err = func() (err error) {
for fieldLabel, ref := range n.References {
auxID = safeParentID(tree, n, ref)
if auxID == 0 {
continue
}
err = n.Resource.SetValue(fieldLabel, 0, auxID)
if err != nil {
return
}
}
return
}()
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
return
}
// Flush to the DB
if !n.Evaluated.Skip {
err = store.UpsertQueue(ctx, s, n.Resource.(*types.Queue))
if err != nil {
err = errors.Wrap(err, "failed to upsert Queue")
return
}
}
// Handle resources nested under it
//
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
// the root function calls?
err = func() (err error) {
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
nn = envoyx.OmitPlaceholderNodes(nn...)
switch rt {
}
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to encode nested resources")
return
}
return
}
// matchupQueues returns an index with indicates what resources already exist
func (e StoreEncoder) matchupQueues(ctx context.Context, s store.Storer, uu map[int]types.Queue, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
// @todo might need to do it smarter then this.
// Most resources won't really be that vast so this should be acceptable for now.
aa, _, err := store.SearchQueues(ctx, s, types.QueueFilter{})
if err != nil {
return
}
idMap := make(map[uint64]*types.Queue, len(aa))
strMap := make(map[string]*types.Queue, len(aa))
for _, a := range aa {
idMap[a.ID] = a
strMap[a.Queue] = a
}
var aux *types.Queue
var ok bool
for i, n := range nn {
for _, idf := range n.Identifiers.Slice {
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
aux, ok = idMap[id]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
aux, ok = strMap[idf]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
}
return
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Functions for resource report
// // // // // // // // // // // // // // // // // // // // // // // // //
// prepareReport prepares the resources of the given type for encoding
func (e StoreEncoder) prepareReport(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
// Grab an index of already existing resources of this type
// @note since these resources should be fairly low-volume and existing for
// a short time (and because we batch by resource type); fetching them all
// into memory shouldn't hurt too much.
// @todo do some benchmarks and potentially implement some smarter check such as
// a bloom filter or something similar.
// Get node scopes
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
if err != nil {
err = errors.Wrap(err, "failed to get scope nodes")
return
}
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
existing := make(map[int]types.Report, len(nn))
err = e.matchupReports(ctx, s, existing, scopedNodes, nn)
if err != nil {
err = errors.Wrap(err, "failed to matchup existing Reports")
return
}
for i, n := range nn {
if n.Resource == nil {
panic("unexpected state: cannot call prepareReport with nodes without a defined Resource")
}
res, ok := n.Resource.(*types.Report)
if !ok {
panic("unexpected resource type: node expecting type of report")
}
existing, hasExisting := existing[i]
// Run expressions on the nodes
err = e.runEvals(ctx, hasExisting, n)
if err != nil {
return
}
if hasExisting {
// On existing, we don't need to re-do identifiers and references; simply
// changing up the internal resource is enough.
//
// In the future, we can pass down the tree and re-do the deps like that
switch n.Config.MergeAlg {
case envoyx.OnConflictPanic:
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
return
case envoyx.OnConflictReplace:
// Replace; simple ID change should do the trick
res.ID = existing.ID
case envoyx.OnConflictSkip:
// Replace the node's resource with the fetched one
res = &existing
// @todo merging
}
} else {
// @todo actually a bottleneck. As per sonyflake docs, it can at most
// generate up to 2**8 (256) IDs per 10ms in a single thread.
// How can we improve this?
res.ID = id.Next()
}
// We can skip validation/defaults when the resource is overwritten by
// the one already stored (the panic one errors out anyway) since it
// should already be ok.
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
err = e.setReportDefaults(res)
if err != nil {
return err
}
err = e.validateReport(res)
if err != nil {
return err
}
}
n.Resource = res
}
return
}
// encodeReports encodes a set of resource into the database
func (e StoreEncoder) encodeReports(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
for _, n := range nn {
err = e.encodeReport(ctx, p, s, n, tree)
if err != nil {
return
}
}
return
}
// encodeReport encodes the resource into the database
func (e StoreEncoder) encodeReport(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
// Grab dependency references
var auxID uint64
err = func() (err error) {
for fieldLabel, ref := range n.References {
auxID = safeParentID(tree, n, ref)
if auxID == 0 {
continue
}
err = n.Resource.SetValue(fieldLabel, 0, auxID)
if err != nil {
return
}
}
return
}()
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
return
}
// Flush to the DB
if !n.Evaluated.Skip {
err = store.UpsertReport(ctx, s, n.Resource.(*types.Report))
if err != nil {
err = errors.Wrap(err, "failed to upsert Report")
return
}
}
// Handle resources nested under it
//
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
// the root function calls?
err = func() (err error) {
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
nn = envoyx.OmitPlaceholderNodes(nn...)
switch rt {
}
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to encode nested resources")
return
}
return
}
// matchupReports returns an index with indicates what resources already exist
func (e StoreEncoder) matchupReports(ctx context.Context, s store.Storer, uu map[int]types.Report, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
// @todo might need to do it smarter then this.
// Most resources won't really be that vast so this should be acceptable for now.
aa, _, err := store.SearchReports(ctx, s, types.ReportFilter{})
if err != nil {
return
}
idMap := make(map[uint64]*types.Report, len(aa))
strMap := make(map[string]*types.Report, len(aa))
for _, a := range aa {
strMap[a.Handle] = a
idMap[a.ID] = a
}
var aux *types.Report
var ok bool
for i, n := range nn {
for _, idf := range n.Identifiers.Slice {
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
aux, ok = idMap[id]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
aux, ok = strMap[idf]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
}
return
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Functions for resource role
// // // // // // // // // // // // // // // // // // // // // // // // //
// prepareRole prepares the resources of the given type for encoding
func (e StoreEncoder) prepareRole(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
// Grab an index of already existing resources of this type
// @note since these resources should be fairly low-volume and existing for
// a short time (and because we batch by resource type); fetching them all
// into memory shouldn't hurt too much.
// @todo do some benchmarks and potentially implement some smarter check such as
// a bloom filter or something similar.
// Get node scopes
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
if err != nil {
err = errors.Wrap(err, "failed to get scope nodes")
return
}
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
existing := make(map[int]types.Role, len(nn))
err = e.matchupRoles(ctx, s, existing, scopedNodes, nn)
if err != nil {
err = errors.Wrap(err, "failed to matchup existing Roles")
return
}
for i, n := range nn {
if n.Resource == nil {
panic("unexpected state: cannot call prepareRole with nodes without a defined Resource")
}
res, ok := n.Resource.(*types.Role)
if !ok {
panic("unexpected resource type: node expecting type of role")
}
existing, hasExisting := existing[i]
// Run expressions on the nodes
err = e.runEvals(ctx, hasExisting, n)
if err != nil {
return
}
if hasExisting {
// On existing, we don't need to re-do identifiers and references; simply
// changing up the internal resource is enough.
//
// In the future, we can pass down the tree and re-do the deps like that
switch n.Config.MergeAlg {
case envoyx.OnConflictPanic:
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
return
case envoyx.OnConflictReplace:
// Replace; simple ID change should do the trick
res.ID = existing.ID
case envoyx.OnConflictSkip:
// Replace the node's resource with the fetched one
res = &existing
// @todo merging
}
} else {
// @todo actually a bottleneck. As per sonyflake docs, it can at most
// generate up to 2**8 (256) IDs per 10ms in a single thread.
// How can we improve this?
res.ID = id.Next()
}
// We can skip validation/defaults when the resource is overwritten by
// the one already stored (the panic one errors out anyway) since it
// should already be ok.
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
err = e.setRoleDefaults(res)
if err != nil {
return err
}
err = e.validateRole(res)
if err != nil {
return err
}
}
n.Resource = res
}
return
}
// encodeRoles encodes a set of resource into the database
func (e StoreEncoder) encodeRoles(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
for _, n := range nn {
err = e.encodeRole(ctx, p, s, n, tree)
if err != nil {
return
}
}
return
}
// encodeRole encodes the resource into the database
func (e StoreEncoder) encodeRole(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
// Grab dependency references
var auxID uint64
err = func() (err error) {
for fieldLabel, ref := range n.References {
auxID = safeParentID(tree, n, ref)
if auxID == 0 {
continue
}
err = n.Resource.SetValue(fieldLabel, 0, auxID)
if err != nil {
return
}
}
return
}()
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
return
}
// Flush to the DB
if !n.Evaluated.Skip {
err = store.UpsertRole(ctx, s, n.Resource.(*types.Role))
if err != nil {
err = errors.Wrap(err, "failed to upsert Role")
return
}
}
// Handle resources nested under it
//
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
// the root function calls?
err = func() (err error) {
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
nn = envoyx.OmitPlaceholderNodes(nn...)
switch rt {
}
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to encode nested resources")
return
}
return
}
// matchupRoles returns an index with indicates what resources already exist
func (e StoreEncoder) matchupRoles(ctx context.Context, s store.Storer, uu map[int]types.Role, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
// @todo might need to do it smarter then this.
// Most resources won't really be that vast so this should be acceptable for now.
aa, _, err := store.SearchRoles(ctx, s, types.RoleFilter{})
if err != nil {
return
}
idMap := make(map[uint64]*types.Role, len(aa))
strMap := make(map[string]*types.Role, len(aa))
for _, a := range aa {
strMap[a.Handle] = a
idMap[a.ID] = a
}
var aux *types.Role
var ok bool
for i, n := range nn {
for _, idf := range n.Identifiers.Slice {
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
aux, ok = idMap[id]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
aux, ok = strMap[idf]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
}
return
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Functions for resource template
// // // // // // // // // // // // // // // // // // // // // // // // //
// prepareTemplate prepares the resources of the given type for encoding
func (e StoreEncoder) prepareTemplate(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
// Grab an index of already existing resources of this type
// @note since these resources should be fairly low-volume and existing for
// a short time (and because we batch by resource type); fetching them all
// into memory shouldn't hurt too much.
// @todo do some benchmarks and potentially implement some smarter check such as
// a bloom filter or something similar.
// Get node scopes
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
if err != nil {
err = errors.Wrap(err, "failed to get scope nodes")
return
}
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
existing := make(map[int]types.Template, len(nn))
err = e.matchupTemplates(ctx, s, existing, scopedNodes, nn)
if err != nil {
err = errors.Wrap(err, "failed to matchup existing Templates")
return
}
for i, n := range nn {
if n.Resource == nil {
panic("unexpected state: cannot call prepareTemplate with nodes without a defined Resource")
}
res, ok := n.Resource.(*types.Template)
if !ok {
panic("unexpected resource type: node expecting type of template")
}
existing, hasExisting := existing[i]
// Run expressions on the nodes
err = e.runEvals(ctx, hasExisting, n)
if err != nil {
return
}
if hasExisting {
// On existing, we don't need to re-do identifiers and references; simply
// changing up the internal resource is enough.
//
// In the future, we can pass down the tree and re-do the deps like that
switch n.Config.MergeAlg {
case envoyx.OnConflictPanic:
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
return
case envoyx.OnConflictReplace:
// Replace; simple ID change should do the trick
res.ID = existing.ID
case envoyx.OnConflictSkip:
// Replace the node's resource with the fetched one
res = &existing
// @todo merging
}
} else {
// @todo actually a bottleneck. As per sonyflake docs, it can at most
// generate up to 2**8 (256) IDs per 10ms in a single thread.
// How can we improve this?
res.ID = id.Next()
}
// We can skip validation/defaults when the resource is overwritten by
// the one already stored (the panic one errors out anyway) since it
// should already be ok.
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
err = e.setTemplateDefaults(res)
if err != nil {
return err
}
err = e.validateTemplate(res)
if err != nil {
return err
}
}
n.Resource = res
}
return
}
// encodeTemplates encodes a set of resource into the database
func (e StoreEncoder) encodeTemplates(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
for _, n := range nn {
err = e.encodeTemplate(ctx, p, s, n, tree)
if err != nil {
return
}
}
return
}
// encodeTemplate encodes the resource into the database
func (e StoreEncoder) encodeTemplate(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
// Grab dependency references
var auxID uint64
err = func() (err error) {
for fieldLabel, ref := range n.References {
auxID = safeParentID(tree, n, ref)
if auxID == 0 {
continue
}
err = n.Resource.SetValue(fieldLabel, 0, auxID)
if err != nil {
return
}
}
return
}()
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
return
}
// Flush to the DB
if !n.Evaluated.Skip {
err = store.UpsertTemplate(ctx, s, n.Resource.(*types.Template))
if err != nil {
err = errors.Wrap(err, "failed to upsert Template")
return
}
}
// Handle resources nested under it
//
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
// the root function calls?
err = func() (err error) {
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
nn = envoyx.OmitPlaceholderNodes(nn...)
switch rt {
}
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to encode nested resources")
return
}
return
}
// matchupTemplates returns an index with indicates what resources already exist
func (e StoreEncoder) matchupTemplates(ctx context.Context, s store.Storer, uu map[int]types.Template, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
// @todo might need to do it smarter then this.
// Most resources won't really be that vast so this should be acceptable for now.
aa, _, err := store.SearchTemplates(ctx, s, types.TemplateFilter{})
if err != nil {
return
}
idMap := make(map[uint64]*types.Template, len(aa))
strMap := make(map[string]*types.Template, len(aa))
for _, a := range aa {
strMap[a.Handle] = a
idMap[a.ID] = a
}
var aux *types.Template
var ok bool
for i, n := range nn {
for _, idf := range n.Identifiers.Slice {
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
aux, ok = idMap[id]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
aux, ok = strMap[idf]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
}
return
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Functions for resource user
// // // // // // // // // // // // // // // // // // // // // // // // //
// prepareUser prepares the resources of the given type for encoding
func (e StoreEncoder) prepareUser(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
// Grab an index of already existing resources of this type
// @note since these resources should be fairly low-volume and existing for
// a short time (and because we batch by resource type); fetching them all
// into memory shouldn't hurt too much.
// @todo do some benchmarks and potentially implement some smarter check such as
// a bloom filter or something similar.
// Get node scopes
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
if err != nil {
err = errors.Wrap(err, "failed to get scope nodes")
return
}
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
existing := make(map[int]types.User, len(nn))
err = e.matchupUsers(ctx, s, existing, scopedNodes, nn)
if err != nil {
err = errors.Wrap(err, "failed to matchup existing Users")
return
}
for i, n := range nn {
if n.Resource == nil {
panic("unexpected state: cannot call prepareUser with nodes without a defined Resource")
}
res, ok := n.Resource.(*types.User)
if !ok {
panic("unexpected resource type: node expecting type of user")
}
existing, hasExisting := existing[i]
// Run expressions on the nodes
err = e.runEvals(ctx, hasExisting, n)
if err != nil {
return
}
if hasExisting {
// On existing, we don't need to re-do identifiers and references; simply
// changing up the internal resource is enough.
//
// In the future, we can pass down the tree and re-do the deps like that
switch n.Config.MergeAlg {
case envoyx.OnConflictPanic:
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
return
case envoyx.OnConflictReplace:
// Replace; simple ID change should do the trick
res.ID = existing.ID
case envoyx.OnConflictSkip:
// Replace the node's resource with the fetched one
res = &existing
// @todo merging
}
} else {
// @todo actually a bottleneck. As per sonyflake docs, it can at most
// generate up to 2**8 (256) IDs per 10ms in a single thread.
// How can we improve this?
res.ID = id.Next()
}
// We can skip validation/defaults when the resource is overwritten by
// the one already stored (the panic one errors out anyway) since it
// should already be ok.
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
err = e.setUserDefaults(res)
if err != nil {
return err
}
err = e.validateUser(res)
if err != nil {
return err
}
}
n.Resource = res
}
return
}
// encodeUsers encodes a set of resource into the database
func (e StoreEncoder) encodeUsers(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
for _, n := range nn {
err = e.encodeUser(ctx, p, s, n, tree)
if err != nil {
return
}
}
return
}
// encodeUser encodes the resource into the database
func (e StoreEncoder) encodeUser(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
// Grab dependency references
var auxID uint64
err = func() (err error) {
for fieldLabel, ref := range n.References {
auxID = safeParentID(tree, n, ref)
if auxID == 0 {
continue
}
err = n.Resource.SetValue(fieldLabel, 0, auxID)
if err != nil {
return
}
}
return
}()
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
return
}
// Flush to the DB
if !n.Evaluated.Skip {
err = store.UpsertUser(ctx, s, n.Resource.(*types.User))
if err != nil {
err = errors.Wrap(err, "failed to upsert User")
return
}
}
// Handle resources nested under it
//
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
// the root function calls?
err = func() (err error) {
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
nn = envoyx.OmitPlaceholderNodes(nn...)
switch rt {
}
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to encode nested resources")
return
}
return
}
// matchupUsers returns an index with indicates what resources already exist
func (e StoreEncoder) matchupUsers(ctx context.Context, s store.Storer, uu map[int]types.User, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
// @todo might need to do it smarter then this.
// Most resources won't really be that vast so this should be acceptable for now.
aa, _, err := store.SearchUsers(ctx, s, types.UserFilter{})
if err != nil {
return
}
idMap := make(map[uint64]*types.User, len(aa))
strMap := make(map[string]*types.User, len(aa))
for _, a := range aa {
strMap[a.Handle] = a
idMap[a.ID] = a
}
var aux *types.User
var ok bool
for i, n := range nn {
for _, idf := range n.Identifiers.Slice {
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
aux, ok = idMap[id]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
aux, ok = strMap[idf]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
}
return
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Functions for resource dalConnection
// // // // // // // // // // // // // // // // // // // // // // // // //
// prepareDalConnection prepares the resources of the given type for encoding
func (e StoreEncoder) prepareDalConnection(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
// Grab an index of already existing resources of this type
// @note since these resources should be fairly low-volume and existing for
// a short time (and because we batch by resource type); fetching them all
// into memory shouldn't hurt too much.
// @todo do some benchmarks and potentially implement some smarter check such as
// a bloom filter or something similar.
// Get node scopes
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
if err != nil {
err = errors.Wrap(err, "failed to get scope nodes")
return
}
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
existing := make(map[int]types.DalConnection, len(nn))
err = e.matchupDalConnections(ctx, s, existing, scopedNodes, nn)
if err != nil {
err = errors.Wrap(err, "failed to matchup existing DalConnections")
return
}
for i, n := range nn {
if n.Resource == nil {
panic("unexpected state: cannot call prepareDalConnection with nodes without a defined Resource")
}
res, ok := n.Resource.(*types.DalConnection)
if !ok {
panic("unexpected resource type: node expecting type of dalConnection")
}
existing, hasExisting := existing[i]
// Run expressions on the nodes
err = e.runEvals(ctx, hasExisting, n)
if err != nil {
return
}
if hasExisting {
// On existing, we don't need to re-do identifiers and references; simply
// changing up the internal resource is enough.
//
// In the future, we can pass down the tree and re-do the deps like that
switch n.Config.MergeAlg {
case envoyx.OnConflictPanic:
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
return
case envoyx.OnConflictReplace:
// Replace; simple ID change should do the trick
res.ID = existing.ID
case envoyx.OnConflictSkip:
// Replace the node's resource with the fetched one
res = &existing
// @todo merging
}
} else {
// @todo actually a bottleneck. As per sonyflake docs, it can at most
// generate up to 2**8 (256) IDs per 10ms in a single thread.
// How can we improve this?
res.ID = id.Next()
}
// We can skip validation/defaults when the resource is overwritten by
// the one already stored (the panic one errors out anyway) since it
// should already be ok.
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
err = e.setDalConnectionDefaults(res)
if err != nil {
return err
}
err = e.validateDalConnection(res)
if err != nil {
return err
}
}
n.Resource = res
}
return
}
// encodeDalConnections encodes a set of resource into the database
func (e StoreEncoder) encodeDalConnections(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
for _, n := range nn {
err = e.encodeDalConnection(ctx, p, s, n, tree)
if err != nil {
return
}
}
return
}
// encodeDalConnection encodes the resource into the database
func (e StoreEncoder) encodeDalConnection(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
// Grab dependency references
var auxID uint64
err = func() (err error) {
for fieldLabel, ref := range n.References {
auxID = safeParentID(tree, n, ref)
if auxID == 0 {
continue
}
err = n.Resource.SetValue(fieldLabel, 0, auxID)
if err != nil {
return
}
}
return
}()
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
return
}
// Flush to the DB
if !n.Evaluated.Skip {
err = store.UpsertDalConnection(ctx, s, n.Resource.(*types.DalConnection))
if err != nil {
err = errors.Wrap(err, "failed to upsert DalConnection")
return
}
}
// Handle resources nested under it
//
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
// the root function calls?
err = func() (err error) {
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
nn = envoyx.OmitPlaceholderNodes(nn...)
switch rt {
}
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to encode nested resources")
return
}
return
}
// matchupDalConnections returns an index with indicates what resources already exist
func (e StoreEncoder) matchupDalConnections(ctx context.Context, s store.Storer, uu map[int]types.DalConnection, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
// @todo might need to do it smarter then this.
// Most resources won't really be that vast so this should be acceptable for now.
aa, _, err := store.SearchDalConnections(ctx, s, types.DalConnectionFilter{})
if err != nil {
return
}
idMap := make(map[uint64]*types.DalConnection, len(aa))
strMap := make(map[string]*types.DalConnection, len(aa))
for _, a := range aa {
strMap[a.Handle] = a
idMap[a.ID] = a
}
var aux *types.DalConnection
var ok bool
for i, n := range nn {
for _, idf := range n.Identifiers.Slice {
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
aux, ok = idMap[id]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
aux, ok = strMap[idf]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
}
return
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Functions for resource dalSensitivityLevel
// // // // // // // // // // // // // // // // // // // // // // // // //
// prepareDalSensitivityLevel prepares the resources of the given type for encoding
func (e StoreEncoder) prepareDalSensitivityLevel(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
// Grab an index of already existing resources of this type
// @note since these resources should be fairly low-volume and existing for
// a short time (and because we batch by resource type); fetching them all
// into memory shouldn't hurt too much.
// @todo do some benchmarks and potentially implement some smarter check such as
// a bloom filter or something similar.
// Get node scopes
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
if err != nil {
err = errors.Wrap(err, "failed to get scope nodes")
return
}
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
existing := make(map[int]types.DalSensitivityLevel, len(nn))
err = e.matchupDalSensitivityLevels(ctx, s, existing, scopedNodes, nn)
if err != nil {
err = errors.Wrap(err, "failed to matchup existing DalSensitivityLevels")
return
}
for i, n := range nn {
if n.Resource == nil {
panic("unexpected state: cannot call prepareDalSensitivityLevel with nodes without a defined Resource")
}
res, ok := n.Resource.(*types.DalSensitivityLevel)
if !ok {
panic("unexpected resource type: node expecting type of dalSensitivityLevel")
}
existing, hasExisting := existing[i]
// Run expressions on the nodes
err = e.runEvals(ctx, hasExisting, n)
if err != nil {
return
}
if hasExisting {
// On existing, we don't need to re-do identifiers and references; simply
// changing up the internal resource is enough.
//
// In the future, we can pass down the tree and re-do the deps like that
switch n.Config.MergeAlg {
case envoyx.OnConflictPanic:
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
return
case envoyx.OnConflictReplace:
// Replace; simple ID change should do the trick
res.ID = existing.ID
case envoyx.OnConflictSkip:
// Replace the node's resource with the fetched one
res = &existing
// @todo merging
}
} else {
// @todo actually a bottleneck. As per sonyflake docs, it can at most
// generate up to 2**8 (256) IDs per 10ms in a single thread.
// How can we improve this?
res.ID = id.Next()
}
// We can skip validation/defaults when the resource is overwritten by
// the one already stored (the panic one errors out anyway) since it
// should already be ok.
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
err = e.setDalSensitivityLevelDefaults(res)
if err != nil {
return err
}
err = e.validateDalSensitivityLevel(res)
if err != nil {
return err
}
}
n.Resource = res
}
return
}
// encodeDalSensitivityLevels encodes a set of resource into the database
func (e StoreEncoder) encodeDalSensitivityLevels(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
for _, n := range nn {
err = e.encodeDalSensitivityLevel(ctx, p, s, n, tree)
if err != nil {
return
}
}
return
}
// encodeDalSensitivityLevel encodes the resource into the database
func (e StoreEncoder) encodeDalSensitivityLevel(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
// Grab dependency references
var auxID uint64
err = func() (err error) {
for fieldLabel, ref := range n.References {
auxID = safeParentID(tree, n, ref)
if auxID == 0 {
continue
}
err = n.Resource.SetValue(fieldLabel, 0, auxID)
if err != nil {
return
}
}
return
}()
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
return
}
// Flush to the DB
if !n.Evaluated.Skip {
err = store.UpsertDalSensitivityLevel(ctx, s, n.Resource.(*types.DalSensitivityLevel))
if err != nil {
err = errors.Wrap(err, "failed to upsert DalSensitivityLevel")
return
}
}
// Handle resources nested under it
//
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
// the root function calls?
err = func() (err error) {
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
nn = envoyx.OmitPlaceholderNodes(nn...)
switch rt {
}
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to encode nested resources")
return
}
return
}
// matchupDalSensitivityLevels returns an index with indicates what resources already exist
func (e StoreEncoder) matchupDalSensitivityLevels(ctx context.Context, s store.Storer, uu map[int]types.DalSensitivityLevel, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
// @todo might need to do it smarter then this.
// Most resources won't really be that vast so this should be acceptable for now.
aa, _, err := store.SearchDalSensitivityLevels(ctx, s, types.DalSensitivityLevelFilter{})
if err != nil {
return
}
idMap := make(map[uint64]*types.DalSensitivityLevel, len(aa))
strMap := make(map[string]*types.DalSensitivityLevel, len(aa))
for _, a := range aa {
strMap[a.Handle] = a
idMap[a.ID] = a
}
var aux *types.DalSensitivityLevel
var ok bool
for i, n := range nn {
for _, idf := range n.Identifiers.Slice {
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
aux, ok = idMap[id]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
aux, ok = strMap[idf]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
}
return
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Utility functions
// // // // // // // // // // // // // // // // // // // // // // // // //
func (e *StoreEncoder) grabStorer(p envoyx.EncodeParams) (s store.Storer, err error) {
auxs, ok := p.Params[paramsKeyStorer]
if !ok {
err = errors.Errorf("store encoder expects a store conforming to store.Storer interface")
return
}
s, ok = auxs.(store.Storer)
if !ok {
err = errors.Errorf("store encoder expects a store conforming to store.Storer interface")
return
}
return
}
func (e *StoreEncoder) runEvals(ctx context.Context, existing bool, n *envoyx.Node) (err error) {
// Skip if
if n.Config.SkipIfEval == nil {
return
}
aux, err := expr.EmptyVars().Cast(map[string]any{
"missing": !existing,
})
if err != nil {
return
}
n.Evaluated.Skip, err = n.Config.SkipIfEval.Test(ctx, aux.(*expr.Vars))
return
}
func (e StoreEncoder) getScopeNodes(ctx context.Context, s store.Storer, nn envoyx.NodeSet) (scopes envoyx.NodeSet, err error) {
// Get all requested scopes
scopes = make(envoyx.NodeSet, len(nn))
// @note skipping scope logic since it's currently only supported within
// Compose resources.
return
}
func safeParentID(tt envoyx.Traverser, n *envoyx.Node, ref envoyx.Ref) (out uint64) {
rn := tt.ParentForRef(n, ref)
if rn == nil {
return
}
return rn.Resource.GetID()
}