snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)
Some checks failed
SonarQube / SonarQube (push) Failing after 5m28s
E2e test on main branch push / server and client test (map[name:admin port:8080]) (push) Failing after 3m33s
Server and lib tests / Library test (js) (push) Failing after 1m36s
Server and lib tests / Library test (vue) (push) Successful in 2m24s
Server and lib tests / Web client test (admin) (push) Successful in 2m52s
Server and lib tests / Web client test (compose) (push) Successful in 3m23s
Server and lib tests / Web client test (discovery) (push) Successful in 2m49s
Server and lib tests / Web client test (one) (push) Successful in 2m52s
Server and lib tests / Web client test (privacy) (push) Successful in 2m44s
Server and lib tests / Web client test (reporter) (push) Successful in 2m56s
Server and lib tests / Web client test (workflow) (push) Successful in 3m0s
Server and lib tests / Server tests (push) Failing after 4m55s
E2e test on main branch push / notify-failure (push) Failing after 9s
Server and lib tests / Send matrix testing notification (push) Successful in 2s
Mark stale issues and pull requests / stale (push) Successful in 40s

This commit is contained in:
upstream-mirror
2026-09-18 01:22:23 +05:00
commit 7e385fc126
8391 changed files with 1766236 additions and 0 deletions

View File

@@ -0,0 +1,32 @@
package {{ .package }}
{{ template "gocode/header-gentext.tpl" }}
import (
{{- range .imports }}
{{ . }}
{{- end }}
)
{{- range .resources }}
// {{ .rbacRefFunc }} generates RBAC references
//
// Resources with "envoy: false" are skipped
//
// This function is auto-generated
func {{ .rbacRefFunc }}({{- range .references }}{{ .param }} string, {{- end }}) (res *Ref, pp []*Ref, err error) {
{{- range .references }}
{{- if eq .refField "ID" }}
if {{ .param }} != "*" {
res = &Ref{ResourceType: types.{{ .expIdent }}ResourceType, Identifiers: MakeIdentifiers({{ .param }})}
}
{{- else }}
if {{ .param }} != "*" {
pp = append(pp, &Ref{ResourceType: types.{{ .expIdent }}ResourceType, Identifiers: MakeIdentifiers({{ .param }})})
}
{{- end }}
{{- end }}
return
}
{{- end }}

View File

@@ -0,0 +1,74 @@
package {{ .package }}
{{ template "gocode/header-gentext.tpl" }}
import (
"fmt"
"strings"
{{- range .imports }}
{{ . }}
{{- end }}
)
// Parse generates resource setting logic for each resource
//
// Resources with "envoy: false" are skipped
//
// This function is auto-generated
func ParseRule(res string) (string, *Ref, []*Ref, error) {
if res == "" {
return "", nil, nil, fmt.Errorf("empty resource")
}
sp := "/"
res = strings.TrimSpace(res)
res = strings.TrimRight(res, sp)
rr := strings.Split(res, sp)
// only service defined (corteza::system, corteza::compose, ...)
if len(rr) == 1 {
return res, nil, nil, nil
}
// full thing
resourceType, path := rr[0], rr[1:]
for p := 1; p < len(path); p++ {
if path[p] != "*" && path[p-1] == "*" {
return "", nil, nil, fmt.Errorf("invalid path wildcard combination for '%s'", res)
}
}
// @todo this is to support res. tr. resources also.
// Split it into a separate function and remove this.
if !strings.HasPrefix(resourceType, "corteza::") {
resourceType = "corteza::" + resourceType
}
// make the resource provide the slice of parent resources we should nest under
switch resourceType {
{{- range .resources }}
case {{ .typeConst }}:
if len(path) != {{ len .references }} {
return "", nil, nil, fmt.Errorf("expecting {{ len .references }} reference components in path, got %d", len(path))
}
{{- if gt (len .references) 0 }}
ref, pp, err := {{ .rbacRefFunc }}(
{{- range $i, $r := .references }}
path[{{ $i }}],
{{- end }}
)
return resourceType, ref, pp, err
{{ else }}
// Component resource, no path
return resourceType, nil, nil, nil
{{- end }}
{{- end}}
}
// return unhandled resource as-is
return resourceType, nil, nil, nil
}

View File

@@ -0,0 +1,25 @@
package {{ .package }}
{{ template "gocode/header-gentext.tpl" }}
import (
systemTypes "github.com/cortezaproject/corteza/server/system/types"
)
{{- range .resources }}
func (r *{{ .expIdent }}) EncodeTranslations() ([]*ResourceTranslation, error) {
out := make([]*ResourceTranslation, 0, 10)
rr := r.Res.EncodeTranslations()
rr.SetLanguage(defaultLanguage)
res, ref, pp := r.ResourceTranslationParts()
out = append(out, NewResourceTranslation(systemTypes.FromLocale(rr), res, ref, pp...))
{{ if .extended }}
tmp, err := r.encodeTranslations()
return append(out, tmp...), err
{{ else }}
return out, nil
{{- end }}
}
{{- end }}

View File

@@ -0,0 +1,69 @@
package {{ .package }}
{{ template "gocode/header-gentext.tpl" }}
import (
"fmt"
"strings"
{{- range .imports }}
{{ . }}
{{- end }}
)
// ParseResourceTranslation generates resource setting logic for each resource
//
// Resources with "envoy: false" are skipped
//
// This function is auto-generated
func ParseResourceTranslation(res string) (string, *Ref, []*Ref, error) {
if res == "" {
return "", nil, nil, fmt.Errorf("empty resource")
}
sp := "/"
if strings.Index(res, "corteza::") == 0 {
res = res[9:]
}
res = strings.TrimSpace(res)
res = strings.TrimRight(res, sp)
rr := strings.Split(res, sp)
// only service defined (corteza::system, corteza::compose, ...)
if len(rr) == 1 {
return "", nil, nil, fmt.Errorf("only service defined: %s", res)
}
// full thing
resourceType, path := rr[0], rr[1:]
for p := 1; p < len(path); p++ {
if path[p] == "*" {
return "", nil, nil, fmt.Errorf("path wildcard not allowed for locale resources: '%s'", res)
}
}
// make the resource provide the slice of parent resources we should nest under
switch resourceType {
{{- range .resources }}
case {{ .typeConst }}:
if len(path) != {{ len .references }} {
return "", nil, nil, fmt.Errorf("expecting {{ len .references }} reference components in path, got %d", len(path))
}
ref, pp, err := {{ .resTrRefFunc }}(
{{- range $i, $r := .references }}
path[{{ $i }}],
{{- end }}
)
return {{ .typeConst }}, ref, pp, err
{{ else }}
// Component resource, no path
return {{ .typeConst }}, nil, nil, nil
{{- end }}
}
// return unhandled resource as-is
return resourceType, nil, nil, nil
}

View File

@@ -0,0 +1,24 @@
package {{ .package }}
{{ template "gocode/header-gentext.tpl" }}
import (
{{- range .imports }}
{{ . }}
{{- end }}
)
{{- range .resources }}
// {{ .resTrRefFunc }} generates Locale references
//
// This function is auto-generated
func {{ .resTrRefFunc }}({{- range .references }}{{ .param }} string, {{- end }} self string) (res *Ref, pp []*Ref, err error) {
res = &Ref{ResourceType: types.{{ .expIdent }}ResourceType, Identifiers: MakeIdentifiers(self)}
{{- range .references }}
pp = append(pp, &Ref{ResourceType: types.{{ .expIdent }}ResourceType, Identifiers: MakeIdentifiers({{ .param }})})
{{- end }}
return
}
{{- end }}

View File

@@ -0,0 +1,401 @@
package {{ .package }}
{{ template "gocode/header-gentext.tpl" }}
import (
"context"
"fmt"
"strings"
"github.com/cortezaproject/corteza/server/pkg/id"
"github.com/cortezaproject/corteza/server/pkg/dal"
"github.com/cortezaproject/corteza/server/pkg/envoyx"
"github.com/cortezaproject/corteza/server/store"
"github.com/pkg/errors"
{{- range .imports }}
"{{ . }}"
{{- end }}
)
type (
// StoreDecoder is responsible for generating Envoy nodes from already stored
// resources which can then be managed by Envoy and imported via an encoder.
StoreDecoder struct{}
filterWrap struct {
rt string
f envoyx.ResourceFilter
}
)
const (
paramsKeyStorer = "storer"
paramsKeyDAL = "dal"
)
var (
// @todo temporary fix to make unused pkg/id not throw errors
_ = id.Next
)
// Decode returns a set of envoy nodes based on the provided params
//
// StoreDecoder expects the DecodeParam of `storer` and `dal` which conform
// to the store.Storer and dal.FullService interfaces.
func (d StoreDecoder) Decode(ctx context.Context, p envoyx.DecodeParams) (out envoyx.NodeSet, err error) {
// @todo we can optionally not require them based on what we're doing
s, err := d.getStorer(p)
if err != nil {
return
}
dl, err := d.getDal(p)
if err != nil {
return
}
return d.decode(ctx, s, dl, p)
}
func (d StoreDecoder) decode(ctx context.Context, s store.Storer, dl dal.FullService, p envoyx.DecodeParams) (out envoyx.NodeSet, err error) {
// Preprocessing and basic filtering (to omit what this decoder can't handle)
wrappedFilters := d.prepFilters(p.Filter)
// Get all scoped nodes
scopedNodes, err := d.getScopeNodes(ctx, s, dl, wrappedFilters)
if err != nil {
return
}
// Get all reference nodes
refNodes, refRefs, err := d.getReferenceNodes(ctx, s, dl, wrappedFilters)
if err != nil {
return
}
// Process filters to get the envoy nodes
err = func() (err error) {
var aux envoyx.NodeSet
for i, wf := range wrappedFilters {
switch wf.rt {
{{ range .resources -}}
{{- if .envoy.omit}}{{continue}}{{ end -}}
case types.{{.expIdent}}ResourceType:
aux, err = d.decode{{.expIdent}}(ctx, s, dl, d.make{{.expIdent}}Filter(scopedNodes[i], refNodes[i], wf.f))
if err != nil {
return
}
for _, a := range aux {
a.Identifiers = a.Identifiers.Merge(wf.f.Identifiers)
a.References = envoyx.MergeRefs(a.References, refRefs[i])
}
out = append(out, aux...)
{{ end }}
default:
aux, err = d.extendDecoder(ctx, s, dl, p, wf.rt, refNodes[i], wf.f)
if err!= nil {
return
}
for _, a := range aux {
a.Identifiers = a.Identifiers.Merge(wf.f.Identifiers)
a.References = envoyx.MergeRefs(a.References, refRefs[i])
}
out = append(out, aux...)
}
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to decode filters")
return
}
return
}
{{- range .resources }}
{{- if .envoy.omit}}
{{continue}}
{{ end -}}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Functions for resource {{.ident}}
// // // // // // // // // // // // // // // // // // // // // // // // //
func (d StoreDecoder) decode{{.expIdent}}(ctx context.Context, s store.Storer, dl dal.FullService, f types.{{.expIdent}}Filter) (out envoyx.NodeSet, err error) {
// @todo this might need to be improved.
// Currently, no resource is vast enough to pose a problem.
rr, _, err := store.Search{{.store.expIdentPlural}}(ctx, s, f)
if err != nil {
return
}
for _, r := range rr {
var n *envoyx.Node
n, err = {{.expIdent}}ToEnvoyNode(r)
if err != nil {
return
}
out = append(out, n)
}
{{ if .envoy.store.extendedDecoder -}}
aux, err := d.extended{{.expIdent}}Decoder(ctx, s, dl, f, out)
if err != nil {
return
}
out = append(out, aux...)
{{- end }}
return
}
func {{.expIdent}}ToEnvoyNode(r *types.{{.expIdent}}) (node *envoyx.Node, err error) {
// Identifiers
ii := envoyx.MakeIdentifiers(
{{- range .model.attributes -}}
{{- if not .envoy.identifier -}}
{{continue}}
{{ end }}
r.{{.expIdent}},
{{- end }}
)
// Handle references
// Omit any non-defined values
refs := map[string]envoyx.Ref{}
{{- range .model.attributes -}}
{{- if eq .dal.type "Ref" }}
if r.{{.expIdent}} > 0 {
refs["{{ .expIdent }}"] = envoyx.Ref{
ResourceType: "{{ .dal.refModelResType }}",
Identifiers: envoyx.MakeIdentifiers(r.{{.expIdent}}),
}
}
{{- end }}
{{- end }}
{{ if .envoy.store.extendedRefDecoder }}
refs = envoyx.MergeRefs(refs, decode{{.expIdent}}Refs(r))
{{ end }}
var scope envoyx.Scope
{{if and .envoy.scoped .parents}}
scope = envoyx.Scope{
ResourceType: refs["{{(index .parents 0).refField}}"].ResourceType,
Identifiers: refs["{{(index .parents 0).refField}}"].Identifiers,
}
for k, ref := range refs {
// @todo temporary solution to not needlessly scope resources.
// Optimally, this would be selectively handled by codegen.
if !strings.HasPrefix(ref.ResourceType, "corteza::compose") {
continue
}
ref.Scope = scope
refs[k] = ref
}
{{end}}
{{if and .envoy.scoped (not .parents)}}
scope = envoyx.Scope{
ResourceType: types.{{ .expIdent }}ResourceType,
Identifiers: ii,
}
{{end}}
node = &envoyx.Node{
Resource: r,
ResourceType: types.{{.expIdent}}ResourceType,
Identifiers: ii,
References: refs,
Scope: scope,
}
return
}
{{ if not .envoy.store.customFilterBuilder }}
func (d StoreDecoder) make{{.expIdent}}Filter(scope *envoyx.Node, refs map[string]*envoyx.Node, auxf envoyx.ResourceFilter) (out types.{{.expIdent}}Filter) {
out.Limit = auxf.Limit
ids, hh := auxf.Identifiers.Idents()
_ = ids
_ = hh
out.{{.expIdent}}ID = id.Strings(ids...)
{{ if .envoy.store.handleField }}
if len(hh) > 0 {
out.{{ .envoy.store.handleField }} = hh[0]
}
{{ end }}
// Refs
var (
ar *envoyx.Node
ok bool
)
_ = ar
_ = ok
{{ range .model.attributes }}
{{- if .envoy.store.omitRefFilter }}{{continue}}{{ end }}
{{ if eq .dal.type "Ref" }}
ar, ok = refs["{{ .expIdent }}"]
if ok {
{{ if .envoy.store.filterRefField -}}
out.{{ .envoy.store.filterRefField }} = ar.Resource.GetID()
{{- else -}}
out.{{ .expIdent }} = ar.Resource.GetID()
{{- end }}
}
{{ end }}
{{ end }}
{{- if .envoy.store.extendedFilterBuilder }}
out = d.extend{{.expIdent}}Filter(scope, refs, auxf, out)
{{ end -}}
return
}
{{ else }}
// Resource should define a custom filter builder
{{ end }}
{{end}}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Utilities
// // // // // // // // // // // // // // // // // // // // // // // // //
func (d StoreDecoder) getStorer(p envoyx.DecodeParams) (s store.Storer, err error) {
aux, ok := p.Params[paramsKeyStorer]
if ok {
s, ok = aux.(store.Storer)
if ok {
return
}
}
err = errors.Errorf("store decoder expects a storer conforming to store.Storer interface")
return
}
func (d StoreDecoder) getDal(p envoyx.DecodeParams) (dl dal.FullService, err error) {
aux, ok := p.Params[paramsKeyDAL]
if ok {
dl, ok = aux.(dal.FullService)
if ok {
return
}
}
err = errors.Errorf("store decoder expects a DAL conforming to dal.FullService interface")
return
}
func (d StoreDecoder) prepFilters(ff map[string]envoyx.ResourceFilter) (out []filterWrap) {
out = make([]filterWrap, 0, len(ff))
for rt, f := range ff {
// Handle resources that don't belong to this decoder
if !strings.HasPrefix(rt, "corteza::{{.componentIdent}}") {
continue
}
out = append(out, filterWrap{rt: rt, f: f})
}
return
}
func (d StoreDecoder) getScopeNodes(ctx context.Context, s store.Storer, dl dal.FullService, ff []filterWrap) (scopes envoyx.NodeSet, err error) {
// Get all requested scopes
scopes = make(envoyx.NodeSet, len(ff))
{{ if eq .componentIdent "compose" }}
err = func() (err error) {
for i, fw := range ff {
if fw.f.Scope.ResourceType == "" {
continue
}
// For now the scope can only point to namespace so this will do
var nn envoyx.NodeSet
nn, err = d.decodeNamespace(ctx, s, dl, d.makeNamespaceFilter(nil, nil, envoyx.ResourceFilter{Identifiers: fw.f.Scope.Identifiers}))
if err != nil {
return
}
if len(nn) > 1 {
err = fmt.Errorf("ambiguous scope %v: matches multiple resources", fw.f.Scope)
return
}
if len(nn) == 0 {
err = fmt.Errorf("invalid scope %v: resource not found", fw.f)
return
}
scopes[i] = nn[0]
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to decode node scopes")
return
}
{{ else }}
// @note skipping scope logic since it's currently only supported within
// Compose resources.
{{ end }}
return
}
// getReferenceNodes returns all of the nodes referenced by the nodes defined by the filters
//
// The nodes are provided as a slice (the same order as the filters) and as a map for easier lookups.
func (d StoreDecoder) getReferenceNodes(ctx context.Context, s store.Storer, dl dal.FullService, ff []filterWrap) (nodes []map[string]*envoyx.Node, refs []map[string]envoyx.Ref, err error) {
nodes = make([]map[string]*envoyx.Node, len(ff))
refs = make([]map[string]envoyx.Ref, len(ff))
err = func() (err error) {
for i, a := range ff {
if len(a.f.Refs) == 0 {
continue
}
auxr := make(map[string]*envoyx.Node, len(a.f.Refs))
auxa := make(map[string]envoyx.Ref)
for field, ref := range a.f.Refs {
f := ref.ResourceFilter()
aux, err := d.decode(ctx, s, dl, envoyx.DecodeParams{
Type: envoyx.DecodeTypeStore,
Filter: f,
})
if err != nil {
return err
}
// @todo consider changing this.
// Currently it's required because the .decode may return some
// nested nodes as well.
// Consider a flag or a new function.
aux = envoyx.NodesForResourceType(ref.ResourceType, aux...)
if len(aux) == 0 {
return fmt.Errorf("invalid reference %v", ref)
}
if len(aux) > 1 {
return fmt.Errorf("ambiguous reference: too many resources returned %v", a.f)
}
auxr[field] = aux[0]
auxa[field] = aux[0].ToRef()
}
nodes[i] = auxr
refs[i] = auxa
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to decode node references")
return
}
return
}

View File

@@ -0,0 +1,513 @@
package {{ .package }}
{{ template "gocode/header-gentext.tpl" }}
import (
"context"
"fmt"
"strconv"
"github.com/cortezaproject/corteza/server/pkg/envoyx"
"github.com/cortezaproject/corteza/server/pkg/expr"
"github.com/cortezaproject/corteza/server/pkg/id"
"github.com/cortezaproject/corteza/server/store"
"github.com/pkg/errors"
{{- range .imports }}
"{{ . }}"
{{- end }}
)
type (
// StoreEncoder is responsible for encoding Corteza resources into the
// database via the Storer or the DAL interface
//
// @todo consider having a different encoder for the DAL resources
StoreEncoder struct{}
)
{{ $rootRes := .resources }}
// Prepare performs some initial processing on the resource before it can be encoded
//
// Preparation runs validation, default value initialization, matching with
// already existing instances, ...
//
// The prepare function receives a set of nodes grouped by the resource type.
// This enables some batching optimization and simplifications when it comes to
// matching with existing resources.
//
// Prepare does not receive any placeholder nodes which are used solely
// for dependency resolution.
func (e StoreEncoder) Prepare(ctx context.Context, p envoyx.EncodeParams, rt string, nn envoyx.NodeSet) (err error) {
s, err := e.grabStorer(p)
if err != nil {
return
}
switch rt {
{{- range .resources }}
{{- if .envoy.omit}}
{{continue}}
{{end -}}
case types.{{.expIdent}}ResourceType:
return e.prepare{{.expIdent}}(ctx, p, s, nn)
{{ end -}}
default:
return e.prepare(ctx, p, s, rt, nn)
}
return
}
// Encode encodes the given Corteza resources into the primary store
//
// Encoding should not do any additional processing apart from matching with
// dependencies and runtime validation
//
// The Encode function is called for every resource type where the resource
// appears at the root of the dependency tree.
// All of the root-level resources for that resource type are passed into the function.
// The encoding function must traverse the branches to encode all of the dependencies.
//
// This flow is used to simplify the flow of how resources are encoded into YAML
// (and other documents) as well as to simplify batching.
//
// Encode does not receive any placeholder nodes which are used solely
// for dependency resolution.
func (e StoreEncoder) Encode(ctx context.Context, p envoyx.EncodeParams, rt string, nodes envoyx.NodeSet, tree envoyx.Traverser) (err error) {
s, err := e.grabStorer(p)
if err != nil {
return
}
switch rt {
{{- range .resources }}
{{- if .envoy.omit -}}
{{continue}}
{{end}}
case types.{{.expIdent}}ResourceType:
return e.encode{{.expIdent}}s(ctx, p, s, nodes, tree)
{{ end -}}
default:
return e.encode(ctx, p, s, rt, nodes, tree)
}
}
{{- range .resources }}
{{- if .envoy.omit}}
{{continue}}
{{end}}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Functions for resource {{.ident}}
// // // // // // // // // // // // // // // // // // // // // // // // //
// prepare{{.expIdent}} prepares the resources of the given type for encoding
func (e StoreEncoder) prepare{{.expIdent}}(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
// Grab an index of already existing resources of this type
// @note since these resources should be fairly low-volume and existing for
// a short time (and because we batch by resource type); fetching them all
// into memory shouldn't hurt too much.
// @todo do some benchmarks and potentially implement some smarter check such as
// a bloom filter or something similar.
// Get node scopes
scopedNodes, err := e.getScopeNodes(ctx, s, nn)
if err != nil {
err = errors.Wrap(err, "failed to get scope nodes")
return
}
// Initializing the index here (and using a hashmap) so it's not escaped to the heap
existing := make(map[int]types.{{.expIdent}}, len(nn))
err = e.matchup{{.expIdent}}s(ctx, s, existing, scopedNodes, nn)
if err != nil {
err = errors.Wrap(err, "failed to matchup existing {{.expIdent}}s")
return
}
for i, n := range nn {
if n.Resource == nil {
panic("unexpected state: cannot call prepare{{.expIdent}} with nodes without a defined Resource")
}
res, ok := n.Resource.(*types.{{.expIdent}})
if !ok {
panic("unexpected resource type: node expecting type of {{.ident}}")
}
existing, hasExisting := existing[i]
// Run expressions on the nodes
err = e.runEvals(ctx, hasExisting, n)
if err != nil {
return
}
if hasExisting {
// On existing, we don't need to re-do identifiers and references; simply
// changing up the internal resource is enough.
//
// In the future, we can pass down the tree and re-do the deps like that
switch n.Config.MergeAlg {
case envoyx.OnConflictPanic:
err = errors.Errorf("resource %v already exists", n.Identifiers.Slice)
return
case envoyx.OnConflictReplace:
// Replace; simple ID change should do the trick
res.ID = existing.ID
case envoyx.OnConflictSkip:
// Replace the node's resource with the fetched one
res = &existing
// @todo merging
}
} else {
// @todo actually a bottleneck. As per sonyflake docs, it can at most
// generate up to 2**8 (256) IDs per 10ms in a single thread.
// How can we improve this?
res.ID = id.Next()
}
// We can skip validation/defaults when the resource is overwritten by
// the one already stored (the panic one errors out anyway) since it
// should already be ok.
if !hasExisting || n.Config.MergeAlg != envoyx.OnConflictSkip {
err = e.set{{.expIdent}}Defaults(res)
if err != nil {
return err
}
err = e.validate{{.expIdent}}(res)
if err != nil {
return err
}
}
n.Resource = res
}
return
}
// encode{{.expIdent}}s encodes a set of resource into the database
func (e StoreEncoder) encode{{.expIdent}}s(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
for _, n := range nn {
err = e.encode{{.expIdent}}(ctx, p, s, n, tree)
if err != nil {
return
}
}
{{- if .envoy.store.postSetEncoder }}
err = e.post{{.expIdent}}sEncode(ctx, p, s, tree, nn)
{{end }}
return
}
// encode{{.expIdent}} encodes the resource into the database
func (e StoreEncoder) encode{{.expIdent}}(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
// Grab dependency references
var auxID uint64
err = func() (err error) {
for fieldLabel, ref := range n.References {
auxID = safeParentID(tree, n, ref)
if auxID == 0 {
continue
}
err = n.Resource.SetValue(fieldLabel, 0, auxID)
if err != nil {
return
}
}
return
}()
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("failed to set dependency references for %s %v", n.ResourceType, n.Identifiers.Slice))
return
}
{{ if .envoy.store.sanitizeBeforeSave }}
// Custom resource sanitization before saving.
// This can be used to cleanup arbitrary config fields.
e.sanitize{{.expIdent}}BeforeSave(n.Resource.(*types.{{.expIdent}}))
{{ end }}
// Flush to the DB
if !n.Evaluated.Skip {
err = store.Upsert{{.store.expIdent}}(ctx, s, n.Resource.(*types.{{.expIdent}}))
if err != nil {
err = errors.Wrap(err, "failed to upsert {{.expIdent}}")
return
}
}
{{ $a := . }}
// Handle resources nested under it
//
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
// the root function calls?
{{/*
@note this setup will not duplicate encode calls since we only take the
most specific parent resource.
*/}}
{{ $extendedEncoder := .envoy.store.extendedEncoder}}
{{ if $extendedEncoder }}
nested := make(envoyx.NodeSet, 0, 10)
{{ end }}
err = func() (err error) {
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
nn = envoyx.OmitPlaceholderNodes(nn...)
switch rt {
{{- range $cmp := $rootRes }}
{{ if $cmp.envoy.omit }}
{{continue}}
{{ end }}
{{ if not $cmp.parents }}
{{continue}}
{{ end }}
{{ $p := index $cmp.parents (sub (len $cmp.parents) 1)}}
{{ if not (eq $p.handle $a.ident) }}
{{continue}}
{{ end }}
case types.{{$cmp.expIdent}}ResourceType:
err = e.encode{{$cmp.expIdent}}s(ctx, p, s, nn, tree)
if err != nil {
return
}
{{ if $extendedEncoder }}
nested = append(nested, nn...)
{{ end }}
{{- end }}
}
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to encode nested resources")
return
}
{{ if .envoy.store.extendedEncoder }}
err = e.encode{{.expIdent}}Extend(ctx, p, s, n, nested, tree)
if err != nil {
err = errors.Wrap(err, "post encode logic failed with errors")
return
}
{{ end }}
{{ if .envoy.store.extendedSubResources }}
err = e.encode{{.expIdent}}ExtendSubResources(ctx, p, s, n, tree)
if err != nil {
err = errors.Wrap(err, "failed to encode extended sub resources")
return
}
{{ end }}
return
}
// matchup{{.expIdent}}s returns an index with indicates what resources already exist
func (e StoreEncoder) matchup{{.expIdent}}s(ctx context.Context, s store.Storer, uu map[int]types.{{.expIdent}}, scopes envoyx.NodeSet, nn envoyx.NodeSet) (err error) {
// @todo might need to do it smarter then this.
// Most resources won't really be that vast so this should be acceptable for now.
aa, _, err := store.Search{{.store.expIdentPlural}}(ctx, s, types.{{.expIdent}}Filter{})
if err != nil {
return
}
idMap := make(map[uint64]*types.{{.expIdent}}, len(aa))
strMap := make(map[string]*types.{{.expIdent}}, len(aa))
for _, a := range aa {
{{ range .model.attributes }}
{{- if not .envoy.identifier -}}
{{continue}}
{{- end -}}
{{- if eq .goType "uint64" -}}
idMap[a.{{.expIdent}}] = a
{{- else -}}
strMap[a.{{.expIdent}}] = a
{{- end}}
{{ end }}
}
var aux *types.{{.expIdent}}
var ok bool
for i, n := range nn {
{{ if eq .component "compose" }}
scope := scopes[i]
if scope == nil {
continue
}
{{ end }}
for _, idf := range n.Identifiers.Slice {
if id, err := strconv.ParseUint(idf, 10, 64); err == nil {
aux, ok = idMap[id]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
aux, ok = strMap[idf]
if ok {
uu[i] = *aux
// When any identifier matches we can end it
break
}
}
}
return
}
{{end}}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Utility functions
// // // // // // // // // // // // // // // // // // // // // // // // //
func (e *StoreEncoder) grabStorer(p envoyx.EncodeParams) (s store.Storer, err error) {
auxs, ok := p.Params[paramsKeyStorer]
if !ok {
err = errors.Errorf("store encoder expects a store conforming to store.Storer interface")
return
}
s, ok = auxs.(store.Storer)
if !ok {
err = errors.Errorf("store encoder expects a store conforming to store.Storer interface")
return
}
return
}
func (e *StoreEncoder) runEvals(ctx context.Context, existing bool, n *envoyx.Node) (err error) {
// Skip if
if n.Config.SkipIfEval == nil {
return
}
aux, err := expr.EmptyVars().Cast(map[string]any{
"missing": !existing,
})
if err != nil {
return
}
n.Evaluated.Skip, err = n.Config.SkipIfEval.Test(ctx, aux.(*expr.Vars))
return
}
func (e StoreEncoder) getScopeNodes(ctx context.Context, s store.Storer, nn envoyx.NodeSet) (scopes envoyx.NodeSet, err error) {
// Get all requested scopes
scopes = make(envoyx.NodeSet, len(nn))
{{ if eq .componentIdent "compose" }}
err = func() (err error) {
for i, n := range nn {
if n.Scope.ResourceType == "" {
continue
}
// For now the scope can only point to namespace so this will do
var nn envoyx.NodeSet
nn, err = e.decodeNamespace(ctx, s, e.makeNamespaceFilter(nil, nil, envoyx.ResourceFilter{Identifiers: n.Scope.Identifiers}))
if err != nil {
return
}
if len(nn) > 1 {
err = fmt.Errorf("ambiguous scope %v: matches multiple resources", n.Scope)
return
}
// when encoding, it could be missing
if len(nn) == 0 {
return
}
scopes[i] = nn[0]
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to decode node scopes")
return
}
{{ else }}
// @note skipping scope logic since it's currently only supported within
// Compose resources.
{{ end }}
return
}
{{ if eq .componentIdent "compose" }}
func (e StoreEncoder) decodeNamespace(ctx context.Context, s store.Storer, f types.NamespaceFilter) (out envoyx.NodeSet, err error) {
// @todo this might need to be improved.
// Currently, no resource is vast enough to pose a problem.
rr, _, err := store.SearchComposeNamespaces(ctx, s, f)
if err != nil {
return
}
for _, r := range rr {
var n *envoyx.Node
n, err = NamespaceToEnvoyNode(r)
if err != nil {
return
}
out = append(out, n)
}
return
}
func (e StoreEncoder) makeNamespaceFilter(scope *envoyx.Node, refs map[string]*envoyx.Node, auxf envoyx.ResourceFilter) (out types.NamespaceFilter) {
out.Limit = auxf.Limit
ids, hh := auxf.Identifiers.Idents()
_ = ids
_ = hh
out.NamespaceID = id.Strings(ids...)
if len(hh) > 0 {
out.Slug = hh[0]
}
if scope == nil {
return
}
if scope.ResourceType == "" {
return
}
// Overwrite it
out.NamespaceID = id.Strings(scope.Resource.GetID())
return
}
{{ end }}
func safeParentID(tt envoyx.Traverser, n *envoyx.Node, ref envoyx.Ref) (out uint64) {
rn := tt.ParentForRef(n, ref)
if rn == nil {
return
}
return rn.Resource.GetID()
}

View File

@@ -0,0 +1,92 @@
package {{ .package }}
{{ template "gocode/header-gentext.tpl" }}
import (
"strings"
{{- range .imports }}
"{{ . }}"
{{- end }}
)
// SplitResourceIdentifier takes an identifier string and splices it into path
// identifiers as defined by the resource
func SplitResourceIdentifier(ref string) (out map[string]Ref) {
out = make(map[string]Ref, 3)
ref = strings.TrimRight(ref, "/")
pp := strings.Split(ref, "/")
rt := pp[0]
pp = pp[1:]
// @todo this is to support res. tr. resources also.
// Split it into a separate function and remove this.
if !strings.HasPrefix(rt, "corteza::") {
rt = "corteza::" + rt
}
gRef := func(pp []string, i int) string {
if pp[i] == "*" {
return ""
}
return pp[i]
}
switch rt {
{{ range .components }}
{{ $rootCmp := . }}
{{range .resources}}
{{ $a := . }}
case "{{.fqrt}}":
scope := Scope{}
{{$res := .}}
{{range $i, $p := .parents}}
if gRef(pp, {{$i}}) == "" {
return
}
{{- range $cmp := $rootCmp.resources }}
{{ if $cmp.envoy.omit }}
{{continue}}
{{ end }}
{{ if eq $p.handle $cmp.ident }}
{{ if and (eq $rootCmp.ident "compose") (eq $i 0) }}
aux := gRef(pp, {{ $i }})
if aux != "" {
scope.ResourceType = "corteza::compose:namespace"
scope.Identifiers = MakeIdentifiers(aux)
}
{{ end }}
out["Path.{{$i}}"] = Ref{
ResourceType: "{{$cmp.fqrt}}",
Identifiers: MakeIdentifiers(gRef(pp, {{ $i }})),
Scope: scope,
}
{{break}}
{{ end }}
{{- end }}
{{end}}
if gRef(pp, {{len .parents}}) == "" {
return
}
{{if eq .ident "namespace"}}
scope.ResourceType = "{{.fqrt}}"
scope.Identifiers = MakeIdentifiers(gRef(pp, {{len .parents}}))
{{end}}
out["Path.{{len .parents}}"] = Ref{
ResourceType: "{{.fqrt}}",
Identifiers: MakeIdentifiers(gRef(pp, {{len .parents}})),
Scope: scope,
}
{{ end }}
{{ end }}
}
return
}

View File

@@ -0,0 +1,33 @@
package {{ .package }}
{{ template "gocode/header-gentext.tpl" }}
import (
{{- range .imports }}
"{{ . }}"
{{- end }}
)
var (
// needyResources is a list of resources that require a parent resource
//
// This list is primarily used when figuring out what nodes the dep. graph
// should return when traversing.
needyResources = map[string]bool{
{{- range .components -}}
{{- range .resources -}}
{{- if not .parents }}
{{continue}}
{{- end }}
"{{ .fqrt }}": true,
{{- end }}
{{- end }}
"corteza::compose:record-datasource": true,
}
// superNeedyResources is the second level of filtering in case the first
// pass removes everything
superNeedyResources = map[string]bool{
"corteza::compose:module-field": true,
}
)

View File

@@ -0,0 +1,872 @@
package {{ .package }}
{{ template "gocode/header-gentext.tpl" }}
import (
"strings"
"context"
"io"
"os"
systemTypes "github.com/cortezaproject/corteza/server/system/types"
"github.com/cortezaproject/corteza/server/pkg/envoyx"
"github.com/cortezaproject/corteza/server/pkg/rbac"
"github.com/cortezaproject/corteza/server/pkg/y7s"
"golang.org/x/text/language"
"github.com/spf13/cast"
"gopkg.in/yaml.v3"
"github.com/pkg/errors"
{{- range .imports }}
"{{ . }}"
{{- end }}
)
{{$cmpIdent := .componentIdent}}
{{$rootRes := .resources}}
type (
// YamlDecoder is responsible for decoding YAML documents into Corteza resources
// which are then managed by envoy and imported via an encoder.
YamlDecoder struct{}
// documentContext provides a bit of metadata to the decoder such as
// root-level reference definitions (such as the namespace)
documentContext struct {
// references holds any references defined on the root of the document
//
// Example of defining a namespace reference:
// namespace: test_import_namespace
// modules:
// - name: Test Import Module
// handle: test_import_module
references map[string]string
// parentIdent holds the identifier of the parent resource (if nested).
// The parent ident can be handy if the resource requires the info before the
// original handling is finished (such as record datasource nodes).
parentIdent envoyx.Identifiers
}
// auxYamlDoc is a helper struct that registers custom YAML decoders
// to assist the package
auxYamlDoc struct {
nodes envoyx.NodeSet
}
)
const (
paramsKeyStream = "stream"
)
// CanFile returns true if the provided file can be decoded with this decoder
func (d YamlDecoder) CanFile(f *os.File) (ok bool) {
// @todo improve this check; for now a simple extension check should do the trick
return d.CanExt(f.Name())
}
// CanExt returns true if the provided file extension can be decoded with this decoder
func (d YamlDecoder) CanExt(name string) (ok bool) {
var (
pt = strings.Split(name, ".")
ext = strings.TrimSpace(pt[len(pt)-1])
)
return ext == "yaml" || ext == "yml"
}
// Decode returns a set of envoy nodes based on the provided params
//
// YamlDecoder expects the DecodeParam of `stream` which conforms
// to the io.Reader interface.
func (d YamlDecoder) Decode(ctx context.Context, p envoyx.DecodeParams) (out envoyx.NodeSet, err error) {
r, err := d.getReader(ctx, p)
if err != nil {
return
}
// Offload decoding to the aux document
doc := &auxYamlDoc{}
err = yaml.NewDecoder(r).Decode(doc)
if err != nil {
err = errors.Wrap(err, "{{$cmpIdent}} yaml decoder: failed to decode document")
return
}
return doc.nodes, nil
}
func (d *auxYamlDoc) UnmarshalYAML(n *yaml.Node) (err error) {
// Get the document context from the root level
dctx, err := d.getDocumentContext(n)
if err != nil {
return
}
var aux envoyx.NodeSet
return y7s.EachMap(n, func(k, v *yaml.Node) error {
kv := strings.ToLower(k.Value)
switch kv {
// Decode all resources under the {{$cmpIdent}} component
{{- range .resources -}}
{{- if .envoy.omit -}}
{{continue}}
{{- end -}}
{{ $identKeys := .envoy.yaml.identKeys }}
case {{ range $i, $l := $identKeys -}}
"{{ $l }}"{{if not (eq $i (sub (len $identKeys) 1))}},{{end}}
{{- end}}:
{{- if .envoy.yaml.supportMappedInput }}
if y7s.IsMapping(v) {
aux, err = d.unmarshal{{.expIdent}}Map(dctx, v)
d.nodes = append(d.nodes, aux...)
}
{{ else }}
// @note {{ .ident }} doesn't support mapped inputs. This can be
// changed in the .cue definition under the
// .envoy.yaml.supportMappedInput field.
{{- end }}
if y7s.IsSeq(v) {
aux, err = d.unmarshal{{.expIdent}}Seq(dctx, v)
d.nodes = append(d.nodes, aux...)
}
if err != nil {
err = errors.Wrap(err, "failed to unmarshal node: {{.ident}}")
}
return err
{{ end }}
{{/* @todo this should be improved and offloaded to the cue instead of template sys */}}
{{- if eq .componentIdent "system" }}
// Decode access control nodes
case "allow":
aux, err = unmarshalAllowNode(v)
d.nodes = append(d.nodes, aux...)
if err != nil {
err = errors.Wrap(err, "failed to unmarshal node: RBAC allow")
}
case "deny":
aux, err = unmarshalDenyNode(v)
d.nodes = append(d.nodes, aux...)
if err != nil {
err = errors.Wrap(err, "failed to unmarshal node: RBAC deny")
}
// Resource translation nodes
case "locale", "translation", "translations", "i18n":
aux, err = unmarshalLocaleNode(v)
d.nodes = append(d.nodes, aux...)
if err != nil {
err = errors.Wrap(err, "failed to unmarshal node: locale")
}
{{ end }}
// Offload to custom handlers
default:
aux, err = d.unmarshalYAML(kv, v)
d.nodes = append(d.nodes, aux...)
if err != nil {
err = errors.Wrap(err, "failed to unmarshal node")
}
}
return nil
})
}
{{- range .resources }}
{{- if .envoy.omit}}
{{continue}}
{{ end -}}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Functions for resource {{.ident}}
// // // // // // // // // // // // // // // // // // // // // // // // //
// unmarshal{{.expIdent}}Seq unmarshals {{.expIdent}} when provided as a sequence node
func (d *auxYamlDoc) unmarshal{{.expIdent}}Seq(dctx documentContext, n *yaml.Node) (out envoyx.NodeSet, err error) {
var aux envoyx.NodeSet
err = y7s.EachSeq(n, func(n *yaml.Node) error {
aux, err = d.unmarshal{{ .expIdent }}Node(dctx, n)
if err != nil {
return err
}
out = append(out, aux...)
return nil
})
return
}
// unmarshal{{.expIdent}}Map unmarshals {{.expIdent}} when provided as a mapping node
//
// When map encoded, the map key is used as a preset identifier.
// The identifier is passed to the node function as a meta node
{{- if not .envoy.yaml.supportMappedInput }}
// @note this resource does not support map encoding.
// Refer to the corresponding definition files to adjust if needed.
{{ else }}
func (d *auxYamlDoc) unmarshal{{ .expIdent }}Map(dctx documentContext, n *yaml.Node) (out envoyx.NodeSet, err error) {
var aux envoyx.NodeSet
err = y7s.EachMap(n, func(k, n *yaml.Node) error {
aux, err = d.unmarshal{{ .expIdent }}Node(dctx, n, k)
if err != nil {
return err
}
out = append(out, aux...)
return nil
})
return
}
{{ end }}
{{ range .envoy.yaml.extendedResourceDecoders -}}
// unmarshal{{.expIdent}}ExtendedSeq unmarshals {{.expIdent}} when provided as a sequence node
func (d *auxYamlDoc) unmarshalExtended{{.expIdent}}Seq(dctx documentContext, n *yaml.Node) (out envoyx.NodeSet, err error) {
var aux envoyx.NodeSet
err = y7s.EachSeq(n, func(n *yaml.Node) error {
aux, err = d.unmarshal{{ .expIdent }}ExtendedNode(dctx, n)
if err != nil {
return err
}
out = append(out, aux...)
return nil
})
return
}
{{if not .supportMappedInput}}{{continue}}{{end}}
// unmarshal{{.expIdent}}ExtendedMap unmarshals {{.expIdent}} when provided as a mapping node
//
// When map encoded, the map key is used as a preset identifier.
// The identifier is passed to the node function as a meta node
func (d *auxYamlDoc) unmarshalExtended{{ .expIdent }}Map(dctx documentContext, n *yaml.Node) (out envoyx.NodeSet, err error) {
var aux envoyx.NodeSet
err = y7s.EachMap(n, func(k, n *yaml.Node) error {
aux, err = d.unmarshal{{ .expIdent }}ExtendedNode(dctx, n, k)
if err != nil {
return err
}
out = append(out, aux...)
return nil
})
return
}
{{ end }}
// unmarshal{{ .expIdent }}Node is a cookie-cutter function to unmarshal
// the yaml node into the corresponding Corteza type & Node
func (d *auxYamlDoc) unmarshal{{ .expIdent }}Node(dctx documentContext, n *yaml.Node, meta ...*yaml.Node) (out envoyx.NodeSet, err error) {
var r *types.{{ .expIdent }}
// @todo we're omitting errors because there will be a bunch due to invalid
// resource field types. This might be a bit unstable as other errors may
// also get ignored.
//
// A potential fix would be to firstly unmarshal into an any, check errors
// and then unmarshal into the resource while omitting errors.
n.Decode(&r)
// Identifiers are determined manually when iterating the yaml node.
// This is to help assure there are no duplicates and everything
// was accounted for especially when working with aliases such as
// user_name instead of userName.
ii := envoyx.Identifiers{}
{{ if .envoy.yaml.supportMappedInput}}
// When a resource supports mapped input, the key is passed as meta which
// needs to be registered as an identifier (since it is)
if len(meta) > 0 {
y7s.DecodeScalar(meta[0], "{{ .envoy.yaml.mappedField }}", &r.{{ .envoy.yaml.mappedField }})
ii = ii.Add(r.{{ .envoy.yaml.mappedField }})
}
{{ end }}
var (
refs = make(map[string]envoyx.Ref)
auxOut envoyx.NodeSet
nestedNodes envoyx.NodeSet
scope envoyx.Scope
envoyConfig envoyx.EnvoyConfig
{{- if .rbac }}
rbacNodes envoyx.NodeSet
{{- end }}
)
_ = auxOut
_ = refs
err = y7s.EachMap(n, func(k, n *yaml.Node) error {
var auxNodeValue any
_ = auxNodeValue
switch strings.ToLower(k.Value) {
{{ $resource := . }}
{{/*
Iterate over all model attributes and handle
- attribute aliases
- identifiers
- reference
- custom decoding logic
Generic field decoding is already handled at the top with the generic
node.Decode(...) function call.
*/}}
{{- range $attr := .model.attributes }}
{{/*
We can skip the attributes when...
- there aren't any ident key aliases
- not an identifier
- not a reference
- no custom logic
*/}}
{{
if and
(or (not $attr.envoy.yaml.identKeyAlias) (eq (len $attr.envoy.yaml.identKeyAlias) 0))
(not $attr.envoy.identifier)
(not (eq $attr.dal.type "Ref"))
(not $attr.envoy.yaml.customDecoder)
}}
{{continue}}
{{ end }}
{{ $identKeys := .envoy.yaml.identKeys }}
case {{ range $i, $l := $identKeys -}}
"{{ $l }}"{{if not (eq $i (sub (len $identKeys) 1))}},{{end}}
{{- end}}:
{{- if and $attr.envoy.yaml.identKeyAlias (gt (len $attr.envoy.yaml.identKeyAlias) 0) (not (eq $attr.dal.type "Ref")) }}
// Handle field alias
//
// @todo consider adding an is empty check before overwriting
err = y7s.DecodeScalar(n, "{{ $attr.ident }}", &r.{{ $attr.expIdent }})
if err != nil {
return err
}
{{- end }}
{{- if $attr.envoy.identifier }}
// Handle identifiers
err = y7s.DecodeScalar(n, "{{ $attr.ident }}", &auxNodeValue)
if err != nil {
return err
}
ii = ii.Add(auxNodeValue)
{{- end }}
{{- if eq $attr.dal.type "Ref" }}
// Handle references
err = y7s.DecodeScalar(n, "{{ $attr.ident }}", &auxNodeValue)
if err != nil {
return err
}
// Omit if not defined
tmp := cast.ToString(auxNodeValue)
if tmp == "0" || tmp == "" {
break
}
refs["{{ $attr.expIdent }}"] = envoyx.Ref{
ResourceType: "{{ $attr.dal.refModelResType }}",
Identifiers: envoyx.MakeIdentifiers(auxNodeValue),
}
{{- end }}
{{ if $attr.envoy.yaml.customDecoder }}
// Handle custom node decoder
//
// The decoder may update the passed resource with arbitrary values
// as well as provide additional references and identifiers for the node.
var (
auxRefs map[string]envoyx.Ref
auxIdents envoyx.Identifiers
)
auxRefs, auxIdents, err = d.unmarshal{{ $resource.expIdent }}{{ $attr.expIdent }}Node(r, n)
if err != nil {
return err
}
refs = envoyx.MergeRefs(refs, auxRefs)
ii = ii.Merge(auxIdents)
{{ end }}
break
{{- end }}
{{ if .rbac }}
// Handle RBAC rules
case "allow":
auxOut, err = unmarshalAllowNode(n)
if err != nil {
return err
}
rbacNodes = append(rbacNodes, auxOut...)
auxOut = nil
case "deny":
auxOut, err = unmarshalDenyNode(n)
if err != nil {
return err
}
rbacNodes = append(rbacNodes, auxOut...)
auxOut = nil
{{- end }}
case "(envoy)":
envoyConfig = d.decodeEnvoyConfig(n)
}
return nil
})
if err != nil {
return
}
// Make parent identifiers available through the dctx
dctx.parentIdent = ii
{{ if eq $cmpIdent "compose" }}
// Handle global namespace reference which can be provided as the doc. context
//
// @todo this is a temporary solution and should be extended when the document
// context needs to be extended.
// Limit this only to the compose resource since that is the only scenario
// the previous implementation supports.
if ref, ok := dctx.references["namespace"]; ok {
refs["NamespaceID"] = envoyx.Ref{
ResourceType: types.NamespaceResourceType,
Identifiers: envoyx.MakeIdentifiers(ref),
}
}
{{ end }}
{{- if and .envoy.scoped .parents}}
// Define the scope
//
// This resource is scoped to the first parent (generally the namespace)
// when talking about Compose resources (the only supported scenario at the moment).
scope = envoyx.Scope{
ResourceType: refs["{{(index .parents 0).refField}}"].ResourceType,
Identifiers: refs["{{(index .parents 0).refField}}"].Identifiers,
}
{{end}}
{{if and .envoy.scoped (not .parents)}}
// Define the scope
//
// This resource is scoped with no parent resources so this resource is the
// root itself (generally the namespace -- the only currently supported scenario).
scope = envoyx.Scope{
ResourceType: types.{{ .expIdent }}ResourceType,
Identifiers: ii,
}
{{end}}
// Apply the scope to all of the references of the same type
for k, ref := range refs {
if !strings.HasPrefix(ref.ResourceType, "corteza::compose") {
continue
}
ref.Scope = scope
refs[k] = ref
}
// Handle any resources that could be inserted under {{.ident}} such as a module inside a namespace
//
// This operation is done in the second pass of the document so we have
// the complete context of the current resource; such as the identifier,
// references, and scope.
var auxNestedNodes envoyx.NodeSet
err = y7s.EachMap(n, func(k, n *yaml.Node) error {
nestedNodes = nil
switch strings.ToLower(k.Value) {
{{/*
@note each resource can only nest resources from the same component.
Iterate resources of the current component and if this one appears as any
of the parent resources, attempt to unmarshal it.
@todo consider limiting the supported set to only the root parent
(like we do with store encoders) to limit strange cases and reduce
potential problems.
*/}}
{{ $a := . }}
{{- range $cmp := $rootRes }}
{{ if $cmp.envoy.omit }}
{{continue}}
{{ end }}
{{- range $p := $cmp.parents }}
{{ if not (eq $p.handle $a.ident) }}
{{continue}}
{{ end }}
{{ $identKeys := $cmp.envoy.yaml.identKeys }}
case {{ range $i, $l := $identKeys -}}
"{{ $l }}"{{if not (eq $i (sub (len $identKeys) 1))}},{{end}}
{{- end}}:
if y7s.IsSeq(n) {
nestedNodes, err = d.unmarshal{{$cmp.expIdent}}Seq(dctx, n)
if err != nil {
return err
}
} else {
nestedNodes, err = d.unmarshal{{$cmp.expIdent}}Map(dctx, n)
if err != nil {
return err
}
}
break
{{/* As long as one parent matches we're golden; avoid potential duplicates */}}
{{break}}
{{- end }}
{{- end -}}
{{- range .envoy.yaml.extendedResourceDecoders }}
{{ $identKeys := .identKeys }}
case {{ range $i, $l := $identKeys -}}
"{{ $l }}"{{if not (eq $i (sub (len $identKeys) 1))}},{{end}}
{{- end}}:
if y7s.IsSeq(n) {
nestedNodes, err = d.unmarshalExtended{{.expIdent}}Seq(dctx, n)
if err != nil {
return err
}
} {{- if .supportMappedInput }} else {
nestedNodes, err = d.unmarshalExtended{{.expIdent}}Map(dctx, n)
if err != nil {
return err
}
}{{ end }}
break
{{ end -}}
}
// Iterate nested nodes and update their reference to the current resource
//
// Any reference to the parent resource from the child resource is overwritten
// to avoid potential user-error edge cases.
for _, a := range nestedNodes {
// @note all nested resources fall under the same component and the same scope.
// Simply assign the same scope to all -- if it shouldn't be scoped
// the parent won't have it (saving CPU ticks :)
a.Scope = scope
if a.References == nil {
a.References = make(map[string]envoyx.Ref)
}
if _, ok := a.References["{{if .envoy.yaml.extendedResourceRefIdent}}{{.envoy.yaml.extendedResourceRefIdent}}{{else}}{{.expIdent}}ID{{end}}"]; !ok {
a.References["{{if .envoy.yaml.extendedResourceRefIdent}}{{.envoy.yaml.extendedResourceRefIdent}}{{else}}{{.expIdent}}ID{{end}}"] = envoyx.Ref{
ResourceType: types.{{ .expIdent }}ResourceType,
Identifiers: ii,
Scope: scope,
}
}
for f, ref := range a.References {
if !strings.HasPrefix(ref.ResourceType, "corteza::compose") {
continue
}
ref.Scope = scope
a.References[f] = ref
}
for f, ref := range refs {
// Only inherit root references
// @todo improve; this is a hack
if strings.Contains(f, ".") {
continue
}
// Only assume refs if they're not yet set
if _, ok := a.References[f]; ok {
continue
}
a.References[f] = ref
}
}
auxNestedNodes = append(auxNestedNodes, nestedNodes...)
return nil
})
if err != nil {
return
}
{{ if .envoy.yaml.extendedResourcePostProcess}}
auxNestedNodes, err = d.postProcessNested{{.expIdent}}Nodes(auxNestedNodes)
if err!= nil {
return
}
{{ end }}
out = append(out, auxNestedNodes...)
a := &envoyx.Node{
Resource: r,
ResourceType: types.{{ .expIdent }}ResourceType,
Identifiers: ii,
References: refs,
{{if or .envoy.scoped}}
Scope: scope,
{{end}}
Config: envoyConfig,
}
{{- if .rbac }}
// Update RBAC resource nodes with references regarding the resource
rres := r.RbacResource()
for _, rn := range rbacNodes {
aux := rn.Resource.(*rbac.Rule)
if aux.Resource == "" {
aux.Resource = rres
}
// Since the rule belongs to the resource, it will have the same
// subset of references as the parent resource.
rn.References = envoyx.MergeRefs(rn.References, a.References)
// The RBAC rule's most specific identifier is the resource itself.
// Using this we can hardcode it to point to the location after the parent resource.
//
// @todo consider using a more descriptive identifier for the position
// such as `index-%d`.
rn.References["{{len .parents}}"] = envoyx.Ref{
ResourceType: a.ResourceType,
Identifiers: a.Identifiers,
Scope: scope,
}
for _, r := range rn.References {
if r.Scope.IsEmpty() {
continue
}
rn.Scope = r.Scope
break
}
}
{{ end }}
// Put it all together...
out = append(out, a)
out = append(out, auxOut...)
{{- if .rbac }}
out = append(out, rbacNodes...)
{{ end }}
return
}
{{ end }}
// // // // // // // // // // // // // // // // // // // // // // // // //
// RBAC unmarshal logic
// // // // // // // // // // // // // // // // // // // // // // // // //
func unmarshalAllowNode(n *yaml.Node) (out envoyx.NodeSet, err error) {
return unmarshalRBACNode(n, rbac.Allow)
}
func unmarshalDenyNode(n *yaml.Node) (out envoyx.NodeSet, err error) {
return unmarshalRBACNode(n, rbac.Deny)
}
func unmarshalRBACNode(n *yaml.Node, acc rbac.Access) (out envoyx.NodeSet, err error) {
if y7s.IsMapping(n.Content[1]) {
out, err = unmarshalFlatRBACNode(n, acc)
if err != nil {
return
}
} else {
out, err = unmarshalNestedRBACNode(n, acc)
if err != nil {
return
}
}
for _, o := range out {
for _, r := range o.References {
if r.Scope.IsEmpty() {
continue
}
o.Scope = r.Scope
break
}
}
return
}
// unmarshalNestedRBACNode handles RBAC rules when they are nested inside a resource
//
// The edge-case exists since the node doesn't explicitly specify the resource
// it belongs to.
//
// Example:
//
// modules:
// module1:
// name: "module 1"
// fields: ...
// allow:
// role1:
// - read
// - delete
func unmarshalNestedRBACNode(n *yaml.Node, acc rbac.Access) (out envoyx.NodeSet, err error) {
return out, y7s.EachMap(n, func(role, op *yaml.Node) error {
out = append(out, &envoyx.Node{
Resource: &rbac.Rule{
Operation: op.Value,
Access: acc,
},
ResourceType: rbac.RuleResourceType,
References: map[string]envoyx.Ref{
"RoleID": {
// Providing resource type as plain text to reduce cross component references
ResourceType: "corteza::system:role",
Identifiers: envoyx.MakeIdentifiers(role.Value),
},
},
})
return nil
})
}
// unmarshalFlatRBACNode handles RBAC rules when they are provided on the root level
//
// Example:
//
// allow:
// role1:
// corteza::system/:
// - users.search
// - users.create
func unmarshalFlatRBACNode(n *yaml.Node, acc rbac.Access) (out envoyx.NodeSet, err error) {
// Handles role
return out, y7s.EachMap(n, func(role, perm *yaml.Node) error {
// Handles operations
return y7s.EachMap(perm, func(res, ops *yaml.Node) error {
// Handle operation (one RBAC rule per op)
return y7s.EachSeq(ops, func(op *yaml.Node) error {
out = append(out, &envoyx.Node{
Resource: &rbac.Rule{
Resource: res.Value,
Operation: op.Value,
Access: acc,
},
ResourceType: rbac.RuleResourceType,
References: envoyx.MergeRefs(
map[string]envoyx.Ref{"RoleID": {
// Providing resource type as plain text to reduce cross component references
ResourceType: "corteza::system:role",
Identifiers: envoyx.MakeIdentifiers(role.Value),
}},
envoyx.SplitResourceIdentifier(res.Value),
),
})
return nil
})
})
})
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// i18n unmarshal logic
// // // // // // // // // // // // // // // // // // // // // // // // //
func unmarshalLocaleNode(n *yaml.Node) (out envoyx.NodeSet, err error) {
err = y7s.EachMap(n, func(lang, loc *yaml.Node) error {
langTag := systemTypes.Lang{Tag: language.Make(lang.Value)}
return y7s.EachMap(loc, func(res, kv *yaml.Node) error {
return y7s.EachMap(kv, func(k, msg *yaml.Node) error {
out = append(out, &envoyx.Node{
Resource: &systemTypes.ResourceTranslation{
Resource: res.Value,
Lang: langTag,
K: k.Value,
Message: msg.Value,
},
// Providing resource type as plain text to reduce cross component references
ResourceType: "corteza::system:resource-translation",
References: envoyx.SplitResourceIdentifier(res.Value),
})
return nil
})
})
})
if err != nil {
return
}
for _, o := range out {
for _, r := range o.References {
if r.Scope.IsEmpty() {
continue
}
o.Scope = r.Scope
break
}
}
return
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Envoy config unmarshal logic
// // // // // // // // // // // // // // // // // // // // // // // // //
func (d *auxYamlDoc) decodeEnvoyConfig(n *yaml.Node) (out envoyx.EnvoyConfig) {
y7s.EachMap(n, func(k, v *yaml.Node) (err error) {
switch strings.ToLower(k.Value) {
case "skipif", "skip":
return y7s.DecodeScalar(v, "decode skip if", &out.SkipIf)
case "onexisting", "mergealg":
out.MergeAlg = envoyx.CastMergeAlg(v.Value)
}
return nil
})
return
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Utilities
// // // // // // // // // // // // // // // // // // // // // // // // //
func (d YamlDecoder) getReader(ctx context.Context, p envoyx.DecodeParams) (r io.Reader, err error) {
aux, ok := p.Params[paramsKeyStream]
if ok {
r, ok = aux.(io.Reader)
if ok {
return
}
}
// @todo consider adding support for managing files from a location
err = errors.Errorf("YAML decoder expects a stream conforming to io.Reader interface")
return
}
func (d *auxYamlDoc) getDocumentContext(n *yaml.Node) (dctx documentContext, err error) {
dctx = documentContext{
references: make(map[string]string),
}
err = y7s.EachMap(n, func(k, v *yaml.Node) error {
// @todo expand when needed. The previous implementation only supported
// namespaces on the root of the document.
if y7s.IsKind(v, yaml.ScalarNode) {
dctx.references[k.Value] = v.Value
}
return nil
})
return
}

View File

@@ -0,0 +1,305 @@
package {{ .package }}
{{ template "gocode/header-gentext.tpl" }}
import (
"context"
"io"
"time"
"github.com/cortezaproject/corteza/server/pkg/envoyx"
"github.com/cortezaproject/corteza/server/pkg/y7s"
"gopkg.in/yaml.v3"
"github.com/pkg/errors"
{{- range .imports }}
"{{ . }}"
{{- end }}
)
{{ $rootRes := .resources }}
type (
// YamlEncoder is responsible for encoding Corteza resources into
// a YAML supported format
YamlEncoder struct{}
)
const (
paramsKeyWriter = "writer"
)
// Encode encodes the given Corteza resources into some YAML supported format
//
// Encoding should not do any additional processing apart from matching with
// dependencies and runtime validation
//
// Preparation runs validation, default value initialization, matching with
// already existing instances, ...
//
// The prepare function receives a set of nodes grouped by the resource type.
// This enables some batching optimization and simplifications when it comes to
// matching with existing resources.
//
// Prepare does not receive any placeholder nodes which are used solely
// for dependency resolution.
func (e YamlEncoder) Encode(ctx context.Context, p envoyx.EncodeParams, rt string, nodes envoyx.NodeSet, tt envoyx.Traverser) (err error) {
var (
out *yaml.Node
aux *yaml.Node
)
_ = aux
w, err := e.getWriter(p)
if err != nil {
return
}
switch rt {
{{- range .resources }}
{{- if or .envoy.omit .envoy.yaml.omitEncoder}}
{{continue}}
{{ end -}}
case types.{{.expIdent}}ResourceType:
aux, err = e.encode{{.expIdent}}s(ctx, p, nodes, tt)
if err != nil {
return
}
// Root level resources are always encoded as a map
out, err = y7s.AddMap(out, "{{.ident}}", aux)
if err != nil {
return
}
{{ end -}}
default:
out, err = e.encode(ctx, out, p, rt, nodes, tt)
if err != nil {
return
}
}
// Don't output nil values since that will produce broken yaml docs
if out == nil {
return
}
return yaml.NewEncoder(w).Encode(out)
}
{{- range .resources }}
{{- if .envoy.omit}}
{{continue}}
{{ end -}}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Functions for resource {{.ident}}
// // // // // // // // // // // // // // // // // // // // // // // // //
func (e YamlEncoder) encode{{.expIdent}}s(ctx context.Context, p envoyx.EncodeParams, nodes envoyx.NodeSet, tt envoyx.Traverser) (out *yaml.Node, err error) {
var aux *yaml.Node
for _, n := range nodes {
aux, err = e.encode{{.expIdent}}(ctx, p, n, tt)
if err != nil {
return
}
out, err = y7s.AddSeq(out, aux)
if err != nil {
return
}
}
return
}
// encode{{.expIdent}} focuses on the specific resource invoked by the Encode method
func (e YamlEncoder) encode{{.expIdent}}(ctx context.Context, p envoyx.EncodeParams, node *envoyx.Node, tt envoyx.Traverser) (out *yaml.Node, err error) {
res := node.Resource.(*types.{{.expIdent}})
{{ $res := .expIdent }}
// Pre-compute some map values so we can omit error checking when encoding yaml nodes
{{ range .model.attributes -}}
{{- if eq .dal.type "Timestamp" -}}
{{- if .dal.nullable -}}
aux{{.expIdent}}, err := e.encodeTimestampNil(p, res.{{.expIdent}})
if err != nil {
return
}
{{- else -}}
aux{{.expIdent}}, err := e.encodeTimestamp(p, res.{{.expIdent}})
if err != nil {
return
}
{{- end -}}
{{- else if eq .dal.type "Ref" -}}
aux{{.expIdent}}, err := e.encodeRef(p, res.{{.expIdent}}, "{{.expIdent}}", node, tt)
if err != nil {
return
}
{{- else if .envoy.yaml.customEncoder -}}
aux{{.expIdent}}, err := e.encode{{$res}}{{.expIdent}}C(ctx, p, tt, node, res, res.{{.expIdent}})
if err != nil {
return
}
{{- end }}
{{end}}
out, err = y7s.AddMap(out,
{{ range .model.attributes -}}
{{- if .envoy.yaml.omitEncoder }}{{continue}}{{ end -}}
{{- if .envoy.yaml.customEncoder -}}
"{{.envoy.yaml.identKeyEncode}}", aux{{.expIdent}},
{{- else if eq .dal.type "Timestamp" -}}
{{- if .dal.nullable -}}
"{{.envoy.yaml.identKeyEncode}}", aux{{.expIdent}},
{{- else -}}
"{{.envoy.yaml.identKeyEncode}}", aux{{.expIdent}},
{{- end -}}
{{- else if eq .dal.type "Ref" -}}
"{{.envoy.yaml.identKeyEncode}}", aux{{.expIdent}},
{{- else -}}
"{{.envoy.yaml.identKeyEncode}}", res.{{.expIdent}},
{{- end }}
{{end}}
)
if err != nil {
return
}
// Handle nested resources
var aux *yaml.Node
_ = aux
{{ $a := . }}
{{- range $cmp := $rootRes }}
{{ if $cmp.envoy.omit }}{{continue}}{{ end }}
{{if not $cmp.parents}}{{continue}}{{end}}
{{/*
Only handle resources where the current resource would appear last
in the list of parents.
Since parents are ordered by _importance_ this removes the danger of
multiple parents decoding the same resource.
*/}}
{{ $p := (index $cmp.parents (sub (len $cmp.parents) 1)) }}
{{ if (eq $p.handle $a.ident) }}
{{ if eq $cmp.ident "moduleField"}}
// When processing module fields, we need to filter out the ones that
// don't belong to this module
//
// @todo offload this to dependency resolution; this is a hack
children := tt.ChildrenForResourceType(node, types.ModuleFieldResourceType)
var proc envoyx.NodeSet
selfRef := node.ToRef()
for _, c := range children {
if c.References["ModuleID"].Equals(selfRef) {
proc = append(proc, c)
}
}
aux, err = e.encodeModuleFields(ctx, p, proc, tt)
{{- else }}
aux, err = e.encode{{$cmp.expIdent}}s(ctx, p, tt.ChildrenForResourceType(node, types.{{$cmp.expIdent}}ResourceType), tt)
{{- end }}
if err != nil {
return
}
out, err = y7s.AddMap(out,
"{{$cmp.ident}}", aux,
)
if err != nil {
return
}
{{ end }}
{{- end }}
{{- range .envoy.yaml.extendedResourceEncoders }}
aux, err = e.encode{{.expIdent}}s(ctx, p, tt.ChildrenForResourceType(node, types.{{.expIdent}}ResourceType), tt)
if err != nil {
return
}
out, err = y7s.AddMap(out,
"{{.identKey}}", aux,
)
if err != nil {
return
}
{{- end }}
return
}
{{ end -}}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Encoding utils
// // // // // // // // // // // // // // // // // // // // // // // // //
func (e YamlEncoder) encodeTimestamp(p envoyx.EncodeParams, t time.Time) (any, error) {
if t.IsZero() {
return nil, nil
}
tz := p.Encoder.PreferredTimezone
if tz != "" {
tzL, err := time.LoadLocation(tz)
if err != nil {
return nil, err
}
t = t.In(tzL)
}
ly := p.Encoder.PreferredTimeLayout
if ly == "" {
ly = time.RFC3339
}
return t.Format(ly), nil
}
func (e YamlEncoder) encodeTimestampNil(p envoyx.EncodeParams, t *time.Time) (any, error) {
if t == nil { return nil, nil }
return e.encodeTimestamp(p, *t)
}
func (e YamlEncoder) encodeRef(p envoyx.EncodeParams, id uint64, field string, node *envoyx.Node, tt envoyx.Traverser) (any, error) {
parent := tt.ParentForRef(node, node.References[field])
// @todo should we panic instead?
// for now gracefully fallback to the ID
if parent == nil {
return id, nil
}
return parent.Identifiers.FriendlyIdentifier(), nil
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Utility functions
// // // // // // // // // // // // // // // // // // // // // // // // //
func (e YamlEncoder) getWriter(p envoyx.EncodeParams) (out io.Writer, err error) {
aux, ok := p.Params[paramsKeyWriter]
if ok {
out, ok = aux.(io.Writer)
if ok {
return
}
}
// @todo consider adding support for managing files from a location
err = errors.Errorf("YAML encoder expects a writer conforming to io.Writer interface")
return
}
func safeParentIdentifier(tt envoyx.Traverser, n *envoyx.Node, ref envoyx.Ref) (out string) {
aux := tt.ParentForRef(n, ref)
if aux == nil {
return ref.Identifiers.FriendlyIdentifier()
}
return aux.Identifiers.FriendlyIdentifier()
}