snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/pkg/envoyx/csv/decoder.go

@@ -0,0 +1,308 @@
+package csv
+
+import (
+	"context"
+	"encoding/csv"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoyx/datasource"
+	"github.com/gabriel-vasile/mimetype"
+	"github.com/spf13/cast"
+)
+
+type (
+	decoder struct {
+		ident string
+
+		src *os.File
+
+		reader   *csv.Reader
+		skipHead bool
+
+		delimiter           string
+		multiValueDelimiter string
+		multiValueBrackets  bool
+
+		header []string
+		row    datasource.RawRecord
+		count  uint64
+	}
+)
+
+func CanDecodeFile(f io.Reader) bool {
+	m, err := mimetype.DetectReader(f)
+	if err != nil {
+		return false
+	}
+
+	return CanDecodeExt(m.Extension())
+}
+
+func CanDecodeMime(m string) bool {
+	return m == "text/csv"
+}
+
+func CanDecodeExt(ext string) bool {
+	pt := strings.Split(ext, ".")
+	return strings.TrimSpace(pt[len(pt)-1]) == "csv"
+}
+
+// Decoder inits a new csv decoder from the given reader
+//
+// @todo hold small files in mem to avoid needles disc access
+func Decoder(r io.Reader, ident string, config map[string]any) (out *decoder, err error) {
+	out = &decoder{
+		ident: ident,
+	}
+
+	err = out.parseConfig(config)
+	if err != nil {
+		return
+	}
+
+	out.src, err = ioutil.TempFile(os.TempDir(), "*.csv")
+	if err != nil {
+		return
+	}
+
+	r, err = out.flushTemp(r)
+	defer out.Reset(nil)
+	if err != nil {
+		return
+	}
+
+	out.reader = csv.NewReader(r)
+	out.reader.ReuseRecord = true
+	if out.delimiter != "" {
+		out.reader.Comma = []rune(out.delimiter)[0]
+	}
+
+	// Header
+	aux, err := out.reader.Read()
+	out.header = append(out.header, aux...)
+	if err != nil {
+		return
+	}
+
+	out.row = make(datasource.RawRecord, len(out.header))
+
+	for {
+		_, err = out.reader.Read()
+		if err == io.EOF {
+			return out, nil
+		} else if err != nil {
+			return
+		}
+
+		// Entry count
+		out.count++
+	}
+}
+
+func (d *decoder) SetConfigs(config map[string]any) (err error) {
+	return d.parseConfig(config)
+}
+
+// Cleanup should be called before we stop using the decoder
+func (d *decoder) Cleanup() error {
+	return os.Remove(d.src.Name())
+}
+
+// SetIdent overwrites the system defined identifier
+func (d *decoder) SetIdent(ident string) {
+	d.ident = ident
+}
+
+// Ident returns the assigned identifier
+func (d *decoder) Ident() string {
+	return d.ident
+}
+
+// Fields returns every available field in this dataset
+func (d *decoder) Fields() []string {
+	return d.header
+}
+
+// Reset resets the decoder to the start
+func (d *decoder) Reset(_ context.Context) error {
+	_, err := d.src.Seek(0, 0)
+	d.skipHead = true
+	return err
+}
+
+// Next returns the field: value mapping for the next row
+func (d *decoder) Next(_ context.Context, out datasource.RawRecord) (more bool, err error) {
+	if d.skipHead {
+		_, err = d.reader.Read()
+		if err != nil {
+			return
+		}
+		d.skipHead = false
+	}
+
+	aux, err := d.reader.Read()
+	if err == io.EOF {
+		return false, nil
+	} else if err != nil {
+		return false, err
+	}
+	for i, h := range d.header {
+		for j, v := range d.parseComplexCSVCell(aux[i]) {
+			err = out.SetValue(h, uint(j), v)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	return true, nil
+}
+
+// Count returns the total number of rows in the dataset
+func (d *decoder) Count() uint64 {
+	return d.count
+}
+
+func (d *decoder) flushTemp(r io.Reader) (_ io.Reader, err error) {
+	_, err = io.Copy(d.src, r)
+	if err != nil {
+		return
+	}
+
+	d.src.Seek(0, 0)
+	return d.src, nil
+}
+
+// parseComplexCSVCell helps us properly multi value and complex JSON definitions
+func (d *decoder) parseComplexCSVCell(cell string) []string {
+	// Cover multi value fields
+	//
+	// Multi value fields are "mini csvs" encoded in the csv so we need to
+	// clean up the value and then CSV decode again.
+	if d.multiValueBrackets {
+		if strings.HasPrefix(cell, "[") && strings.HasSuffix(cell, "]") {
+			cell = cell[1 : len(cell)-1]
+		}
+	}
+
+	// Cover complex json definitions (geometry field)
+	//
+	// Multi value complex fields are delimiter-joined JSON objects; the JSON
+	// itself may contain the delimiter so split on complete objects instead.
+	if strings.HasPrefix(cell, "{") && strings.HasSuffix(cell, "}") {
+		return splitConcatenatedJSON(cell)
+	}
+
+	if len(cell) == 0 {
+		return nil
+	}
+
+	r := csv.NewReader(strings.NewReader(cell))
+	if d.multiValueDelimiter != "" {
+		r.Comma = []rune(d.multiValueDelimiter)[0]
+	}
+
+	records, err := r.ReadAll()
+	if err != nil {
+		return nil
+	}
+
+	return records[0]
+}
+
+// splitConcatenatedJSON splits a cell of delimiter-joined JSON objects into
+// the individual objects by tracking brace depth and string literals, so
+// delimiters and braces inside the JSON never produce a false split.
+//
+// A cell holding a single (possibly nested) object comes back as one value;
+// a malformed cell is returned whole so the value fails validation upstream
+// instead of being silently corrupted.
+func splitConcatenatedJSON(cell string) (out []string) {
+	var (
+		depth int
+		inStr bool
+		esc   bool
+		start = -1
+	)
+
+	for i := 0; i < len(cell); i++ {
+		c := cell[i]
+
+		if inStr {
+			switch {
+			case esc:
+				esc = false
+			case c == '\\':
+				esc = true
+			case c == '"':
+				inStr = false
+			}
+			continue
+		}
+
+		switch c {
+		case '"':
+			inStr = true
+		case '{':
+			if depth == 0 {
+				start = i
+			}
+			depth++
+		case '}':
+			depth--
+			if depth == 0 && start >= 0 {
+				out = append(out, cell[start:i+1])
+				start = -1
+			}
+		}
+	}
+
+	if depth != 0 || inStr || len(out) == 0 {
+		return []string{cell}
+	}
+
+	return out
+}
+
+func (d *decoder) parseConfig(cfg map[string]any) error {
+	if cfg == nil {
+		return nil
+	}
+
+	for k := range cfg {
+		switch k {
+		case "delimiter":
+			d.delimiter = cast.ToString(cfg[k])
+		case "multiValueDelimiter":
+			switch cast.ToString(cast.ToString(cfg[k])) {
+			case ",":
+				d.multiValueDelimiter = ","
+				d.multiValueBrackets = false
+			case ";":
+				d.multiValueDelimiter = ";"
+				d.multiValueBrackets = false
+			case "|":
+				d.multiValueDelimiter = "|"
+				d.multiValueBrackets = false
+			case "[,]":
+				d.multiValueDelimiter = ","
+				d.multiValueBrackets = true
+			case "[;]":
+				d.multiValueDelimiter = ";"
+				d.multiValueBrackets = true
+			case "[|]":
+				d.multiValueDelimiter = "|"
+				d.multiValueBrackets = true
+			}
+
+		default:
+			return fmt.Errorf("unknown parameter %s", k)
+		}
+	}
+
+	return nil
+}

server/pkg/envoyx/csv/decoder_test.go

@@ -0,0 +1,215 @@
+package csv
+
+import (
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoyx/datasource"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDecoder(t *testing.T) {
+	req := require.New(t)
+
+	t.Run("init & meta", func(t *testing.T) {
+		dc, err := Decoder(testReader(), "test.csv", nil)
+		req.NoError(err)
+
+		req.Equal(uint64(3), dc.Count())
+	})
+
+	t.Run("fields", func(t *testing.T) {
+		dc, err := Decoder(testReader(), "test.csv", nil)
+		req.NoError(err)
+
+		hh := dc.Fields()
+		req.Contains(hh, "f1")
+		req.Contains(hh, "f2")
+		req.Contains(hh, "f3")
+	})
+
+	t.Run("iterate", func(t *testing.T) {
+		dc, err := Decoder(testReader(), "test.csv", nil)
+		req.NoError(err)
+
+		aux := make(datasource.RawRecord)
+		var more bool
+
+		more, err = dc.Next(nil, aux)
+		req.NoError(err)
+		req.True(more)
+		req.Equal("r1f1", aux["f1"].Values[0])
+		req.Equal("r1f2", aux["f2"].Values[0])
+		req.Equal("r1f3", aux["f3"].Values[0])
+
+		more, err = dc.Next(nil, aux)
+		req.NoError(err)
+		req.True(more)
+		req.Equal("r2f1", aux["f1"].Values[0])
+		req.Equal("r2f2", aux["f2"].Values[0])
+		req.Equal("r2f3", aux["f3"].Values[0])
+
+		more, err = dc.Next(nil, aux)
+		req.NoError(err)
+		req.True(more)
+		req.Equal("r3f1", aux["f1"].Values[0])
+		req.Equal("r3f2", aux["f2"].Values[0])
+		req.Equal("r3f3", aux["f3"].Values[0])
+
+		more, err = dc.Next(nil, aux)
+		req.NoError(err)
+		req.False(more)
+	})
+}
+
+func TestParseComplexCSVCell(t *testing.T) {
+	t.Run("plain value", func(t *testing.T) {
+		d := &decoder{}
+
+		out := d.parseComplexCSVCell("cell")
+		require.Equal(t, []string{"cell"}, out)
+	})
+
+	t.Run("multi value without brackets", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ";",
+		}
+
+		out := d.parseComplexCSVCell("v1;v2")
+		require.Equal(t, []string{"v1", "v2"}, out)
+	})
+
+	t.Run("multi value with brackets", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ";",
+			multiValueBrackets:  true,
+		}
+
+		out := d.parseComplexCSVCell("[v1;v2]")
+		require.Equal(t, []string{"v1", "v2"}, out)
+	})
+
+	t.Run("multi value wrong delimiter", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ",",
+			multiValueBrackets:  true,
+		}
+
+		out := d.parseComplexCSVCell("[v1;v2]")
+		require.Equal(t, []string{"v1;v2"}, out)
+	})
+
+	t.Run("json", func(t *testing.T) {
+		d := &decoder{}
+
+		out := d.parseComplexCSVCell("{}")
+		require.Equal(t, []string{"{}"}, out)
+	})
+
+	t.Run("multi value json", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ";",
+			multiValueBrackets:  true,
+		}
+
+		out := d.parseComplexCSVCell("[{};{}]")
+		require.Equal(t, []string{"{}", "{}"}, out)
+	})
+
+	t.Run("single value geometry json", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ";",
+		}
+
+		out := d.parseComplexCSVCell(`{"coordinates":[47.754098,1.9335938]}`)
+		require.Equal(t, []string{`{"coordinates":[47.754098,1.9335938]}`}, out)
+	})
+
+	t.Run("multi value geometry json without brackets", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ";",
+		}
+
+		out := d.parseComplexCSVCell(`{"coordinates":[47.754098,1.9335938]};{"coordinates":[9.7956776,23.3789063]}`)
+		require.Equal(t, []string{
+			`{"coordinates":[47.754098,1.9335938]}`,
+			`{"coordinates":[9.7956776,23.3789063]}`,
+		}, out)
+	})
+
+	t.Run("multi value geometry json with brackets", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ";",
+			multiValueBrackets:  true,
+		}
+
+		out := d.parseComplexCSVCell(`[{"coordinates":[47.754098,1.9335938]};{"coordinates":[9.7956776,23.3789063]}]`)
+		require.Equal(t, []string{
+			`{"coordinates":[47.754098,1.9335938]}`,
+			`{"coordinates":[9.7956776,23.3789063]}`,
+		}, out)
+	})
+
+	t.Run("single value geometry json with comma delimiter", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ",",
+		}
+
+		// commas inside the JSON must not split the value
+		out := d.parseComplexCSVCell(`{"coordinates":[47.754098,1.9335938]}`)
+		require.Equal(t, []string{`{"coordinates":[47.754098,1.9335938]}`}, out)
+	})
+
+	t.Run("single value json with nested objects", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ",",
+		}
+
+		// "},{" inside the JSON must not split the value
+		out := d.parseComplexCSVCell(`{"points":[{"x":1},{"y":2}]}`)
+		require.Equal(t, []string{`{"points":[{"x":1},{"y":2}]}`}, out)
+	})
+
+	t.Run("multi value json with nested objects and braces in strings", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ";",
+		}
+
+		out := d.parseComplexCSVCell(`{"a":[{"x":1},{"y":2}],"s":"};{"};{"b":2}`)
+		require.Equal(t, []string{
+			`{"a":[{"x":1},{"y":2}],"s":"};{"}`,
+			`{"b":2}`,
+		}, out)
+	})
+
+	t.Run("multi value json split is delimiter agnostic", func(t *testing.T) {
+		// exports default to ";" while an unconfigured decoder falls back to
+		// ","; complete-object splitting covers both
+		d := &decoder{}
+
+		out := d.parseComplexCSVCell(`{"a":1};{"b":2}`)
+		require.Equal(t, []string{`{"a":1}`, `{"b":2}`}, out)
+
+		out = d.parseComplexCSVCell(`{"a":1},{"b":2}`)
+		require.Equal(t, []string{`{"a":1}`, `{"b":2}`}, out)
+	})
+
+	t.Run("malformed json cell returned whole", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ";",
+		}
+
+		out := d.parseComplexCSVCell(`{"a":{1};{"b"}`)
+		require.Equal(t, []string{`{"a":{1};{"b"}`}, out)
+	})
+}
+
+func testReader() io.Reader {
+	src := `f1,f2,f3
+r1f1,r1f2,r1f3
+r2f1,r2f2,r2f3
+r3f1,r3f2,r3f3`
+
+	return strings.NewReader(src)
+}

server/pkg/envoyx/datasource.go

@@ -0,0 +1,124 @@
+package envoyx
+
+import (
+	"context"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoyx/datasource"
+	"github.com/cortezaproject/corteza/server/pkg/y7s"
+	"gopkg.in/yaml.v3"
+)
+
+type (
+	Provider interface {
+		Next(ctx context.Context, out datasource.RawRecord) (more bool, err error)
+		Reset(ctx context.Context) error
+		SetIdent(string)
+		SetConfigs(map[string]any) error
+		Ident() string
+	}
+
+	Datasource interface {
+		Next(ctx context.Context, out datasource.RawRecord) (ident []string, more bool, err error)
+		Reset(ctx context.Context) error
+		SetProvider(Provider) bool
+	}
+
+	MapEntry struct {
+		Column string
+		Field  string
+		Skip   bool
+	}
+
+	FieldMapping struct {
+		// @note This had to be like so to simplify decoding
+		Map map[string]MapEntry
+	}
+
+	DatasourceMapping struct {
+		SourceIdent string   `yaml:"source"`
+		KeyField    []string `yaml:"key"`
+		References  map[string]string
+		Scope       map[string]string
+
+		// Defaultable indicates wether the mapping should keep the values where
+		// the ident is not explicitly mapped.
+		//
+		// When true, the value is assigned to the given identifier.
+		Defaultable bool `yaml:"defaultable"`
+		Mapping     FieldMapping
+	}
+)
+
+func SetDecoderSources(nn NodeSet, dd ...Provider) {
+	for _, n := range nn {
+		if n.Datasource == nil {
+			continue
+		}
+
+		for _, d := range dd {
+			if n.Datasource.SetProvider(d) {
+				break
+			}
+		}
+	}
+}
+
+// UnmarshalYAML is used to get the yaml parsed into a series of nodes so
+// we can easily pass it down
+func (d *FieldMapping) UnmarshalYAML(n *yaml.Node) (err error) {
+	d.Map = make(map[string]MapEntry)
+	if y7s.IsSeq(n) {
+		err = y7s.EachSeq(n, func(n *yaml.Node) error {
+			a, err := d.unmarshalMappingNode(n)
+			d.Map[a.Column] = a
+			return err
+		})
+	} else {
+		err = y7s.EachMap(n, func(k, n *yaml.Node) error {
+			a, err := d.unmarshalMappingNode(n)
+			if a.Column == "" {
+				err = y7s.DecodeScalar(k, "fieldMapping column", &a.Column)
+				if err != nil {
+					return err
+				}
+			}
+
+			d.Map[a.Column] = a
+			return err
+		})
+	}
+
+	return
+}
+
+func (d *FieldMapping) unmarshalMappingNode(n *yaml.Node) (out MapEntry, err error) {
+	if y7s.IsKind(n, yaml.ScalarNode) {
+		err = y7s.DecodeScalar(n, "Column", &out.Column)
+		if err != nil {
+			return
+		}
+		err = y7s.DecodeScalar(n, "Field", &out.Field)
+		return
+	}
+
+	// @todo we're omitting errors because there will be a bunch due to invalid
+	//       resource field types. This might be a bit unstable as other errors may
+	//       also get ignored.
+	//
+	//       A potential fix would be to firstly unmarshal into an any, check errors
+	//       and then unmarshal into the resource while omitting errors.
+	n.Decode(&out)
+
+	err = y7s.EachMap(n, func(k, v *yaml.Node) error {
+		switch strings.ToLower(k.Value) {
+		case "skip":
+			if v.Value == "/" {
+				out.Skip = true
+			}
+		}
+		return nil
+	})
+
+	return
+}

server/pkg/envoyx/datasource/datasource.go

@@ -0,0 +1,65 @@
+package datasource
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/spf13/cast"
+)
+
+type (
+	RawRecord      map[string]RawRecordValue
+	RawRecordValue struct {
+		Name   string
+		Values []string
+	}
+)
+
+func (ar RawRecord) SetValue(name string, pos uint, value any) (err error) {
+	aux := ar[name]
+
+	aux.Name = name
+
+	if pos >= uint(len(aux.Values)) {
+		aux.Values = append(aux.Values, make([]string, pos-uint(len(aux.Values))+1)...)
+	}
+
+	aux.Values[pos] = cast.ToString(value)
+
+	ar[name] = aux
+
+	return
+}
+
+func (r *RawRecord) UnmarshalJSON(data []byte) error {
+	raw := make(map[string]json.RawMessage)
+	if err := json.Unmarshal(data, &raw); err != nil {
+		return err
+	}
+
+	result := make(RawRecord)
+	for k, v := range raw {
+		var single string
+		if err := json.Unmarshal(v, &single); err == nil {
+			result[k] = RawRecordValue{
+				Name:   k,
+				Values: []string{single},
+			}
+			continue
+		}
+
+		var multi []string
+		if err := json.Unmarshal(v, &multi); err == nil {
+			result[k] = RawRecordValue{
+				Name:   k,
+				Values: multi,
+			}
+			continue
+		}
+
+		return fmt.Errorf("unexpected format for key %q", k)
+	}
+
+	*r = result
+	return nil
+}

server/pkg/envoyx/dep.go

@@ -0,0 +1,498 @@
+package envoyx
+
+type (
+	// DepGraph provides a collection of optionally connected subgraphs
+	//
+	// Each subgraph is dedicated for a specific node scope.
+	// A subgraph with scoped nodes may be connected to the subgraph with no
+	// defined scope.
+	DepGraph struct {
+		graphs []*depSubgraph
+	}
+
+	// depSubgraph provides a bi-directional dependency graph
+	depSubgraph struct {
+		scope Scope
+
+		nodes   []*depNode
+		nodeMap map[*Node]*depNode
+
+		parentSubgraphs map[*depSubgraph]bool
+		childSubgraphs  map[*depSubgraph]bool
+	}
+
+	// depNode provides a wrapper around the resource for eas of use within the dep. graph
+	depNode struct {
+		Node *Node
+
+		parents  map[*depNode]bool
+		children map[*depNode]bool
+
+		// Index to keep track of missing references at build time
+		missingReferences map[string]Ref
+	}
+)
+
+// BuildDepGraph constructs a dependency graph from the provided nodes
+//
+// We firstly group the nodes by scope, then build a subgraph for each scope,
+// and lastly merge the subgraphs into a single graph.
+func BuildDepGraph(nn ...*Node) (out *DepGraph) {
+	scopes := scopeNodes(nn...)
+
+	aux := make([]*depSubgraph, 0, len(scopes))
+	for _, ss := range scopes {
+		aux = append(aux, buildDepSubgraph(ss))
+	}
+
+	return buildDepGraph(aux...)
+}
+
+// scopeNodes groups the nodes by the scope
+//
+// The function returns a slice of NodeSet where each NodeSet only contains
+// nodes of the same scope (or no scope if none defined).
+func scopeNodes(nn ...*Node) (out []NodeSet) {
+	type (
+		// Defining a little wrapper around the NodeSet so we can use pointers
+		scopeWrap struct {
+			nn NodeSet
+		}
+	)
+
+	var (
+		// resource type -> identifier -> wrap
+		scopes = make(map[string]map[string]*scopeWrap)
+
+		// Holds the nodes with no defined scope
+		empty NodeSet
+	)
+
+	// Bucket nodes into scopes
+	//
+	// Use maps to make the process as efficient as possible.
+	for _, n := range nn {
+		// Empty scopes are a special case
+		if n.Scope.IsEmpty() {
+			empty = append(empty, n)
+			continue
+		}
+
+		// New resource type
+		if _, ok := scopes[n.Scope.ResourceType]; !ok {
+			scopes[n.Scope.ResourceType] = make(map[string]*scopeWrap, 4)
+		}
+
+		// Check if the identifiers of the current node scope exist in the index.
+		// If they do, update the wrap struct by adding this node, then register
+		// the same wrap struct to all other identifiers -- this allows for some
+		// recovery in case some resource uses a subset of one but not the other.
+		hasIdent := false
+		firstIdent := ""
+		for _, i := range n.Scope.Identifiers.Slice {
+			hasIdent = hasIdent || scopes[n.Scope.ResourceType][i] != nil
+			if hasIdent {
+				firstIdent = i
+				break
+			}
+		}
+
+		if !hasIdent {
+			// Not registered yet, create a new wrap for all of the identifiers
+			w := &scopeWrap{nn: append(make(NodeSet, 0, 10), n)}
+			for _, i := range n.Scope.Identifiers.Slice {
+				scopes[n.Scope.ResourceType][i] = w
+			}
+		} else {
+			// Already registered; update it and add missing identifiers
+			w := scopes[n.Scope.ResourceType][firstIdent]
+			w.nn = append(w.nn, n)
+			for _, i := range n.Scope.Identifiers.Slice {
+				scopes[n.Scope.ResourceType][i] = w
+			}
+		}
+	}
+
+	// Unpack the map into a slice of NodeSet the return expects
+	//
+	// @note since all identifiers are registered, the wraps would appear
+	//       duplicated so we need to filter them a bit
+	out = make([]NodeSet, 0, 10)
+	if len(empty) > 0 {
+		out = append(out, empty)
+	}
+	seen := make(map[*scopeWrap]bool)
+	for _, s := range scopes {
+		for _, ss := range s {
+			if seen[ss] {
+				continue
+			}
+			seen[ss] = true
+			out = append(out, ss.nn)
+		}
+	}
+
+	return out
+}
+
+// buildDepSubgraph constructs a dep. subgraph from the provided nodes
+//
+// The function returns a bidirectional graph of the nodes where the parent
+// represents the dependency of the current resource (a namespace would be
+// a parent of a module).
+//
+// The build process indexes some data for optimal operations down the line.
+func buildDepSubgraph(nn NodeSet) (out *depSubgraph) {
+	out = &depSubgraph{
+		scope: nn[0].Scope,
+
+		nodes:   make([]*depNode, len(nn)),
+		nodeMap: make(map[*Node]*depNode, len(nn)),
+
+		parentSubgraphs: make(map[*depSubgraph]bool),
+		childSubgraphs:  make(map[*depSubgraph]bool),
+	}
+
+	byIdentifier := make(map[string]map[string]*depNode, 8)
+	byNode := make(map[*Node]*depNode, len(nn))
+
+	// 1. index all of the nodes in a map so we can trivially connect them later
+	for i, _n := range nn {
+		n := _n
+
+		// Function blindly trusts it will be called with the correct data.
+		// @todo consider implementing this check but make sure all of the decoders
+		//       correctly set the scopes.
+		// if !n.Scope.Equals(nn[0].Scope) {
+		// 	panic("invalid state: subgraphs can only be constructed with nodes from the same scope")
+		// }
+
+		aux := &depNode{
+			Node: n,
+
+			// Keep track of missing references so we can figure them out optimally
+			missingReferences: make(map[string]Ref),
+
+			parents:  make(map[*depNode]bool),
+			children: make(map[*depNode]bool),
+		}
+
+		for field, ref := range n.References {
+			aux.missingReferences[field] = ref
+		}
+
+		byNode[n] = aux
+		out.nodes[i] = aux
+		out.nodeMap[n] = aux
+
+		if _, ok := byIdentifier[n.ResourceType]; !ok {
+			byIdentifier[n.ResourceType] = make(map[string]*depNode, 8)
+		}
+
+		for _, i := range n.Identifiers.Slice {
+			byIdentifier[n.ResourceType][i] = byNode[n]
+		}
+	}
+
+	// 2. link up the node with it's dependencies
+	for _, _n := range out.nodes {
+		n := _n
+
+		for field, ref := range n.Node.References {
+			resource := ref.ResourceType
+			found := false
+
+			for _, i := range ref.Identifiers.Slice {
+				ch, ok := byIdentifier[resource][i]
+				found = found || ok
+
+				if ok {
+					delete(n.missingReferences, field)
+					n.parents[ch] = true
+					ch.children[n] = true
+					break
+				}
+			}
+		}
+	}
+
+	return
+}
+
+// buildDepGraph constructs a dependency graph from the given set of subgraphs
+//
+// The subgraphs can be connected in case a scoped node would reference a
+// unscoped node (unscoped nodes can not reference scoped nodes, nor can nodes
+// from different scopes -- unneeded and removes some complexity).
+func buildDepGraph(gg ...*depSubgraph) (out *DepGraph) {
+	out = &DepGraph{
+		graphs: make([]*depSubgraph, 0, len(gg)),
+	}
+
+	// Get the unscoped graph
+	//
+	// For now, we can only xref to unscoped graphs.
+	// This might need to be generalized.
+	unscopedG := unscopedSubgraph(gg...)
+
+	// Iterate all graphs and try to resolve unresolved deps
+	for _, g := range gg {
+		out.graphs = append(out.graphs, g)
+
+		if unscopedG == nil {
+			continue
+		}
+
+		// Get nodes with missing references
+		missingNodes := g.nodesWithMissingRefs()
+		if len(missingNodes) == 0 {
+			continue
+		}
+
+		// Try to resolve missing refs using the unscoped graph
+		for _, _mn := range missingNodes {
+			mn := _mn
+
+			for field, ref := range mn.missingReferences {
+				n := depNodeForRef(ref, unscopedG.nodes...)
+				if n == nil {
+					continue
+				}
+
+				mn.parents[n] = true
+				n.children[mn] = true
+
+				delete(mn.missingReferences, field)
+				if len(mn.missingReferences) == 0 {
+					mn.missingReferences = nil
+				}
+			}
+		}
+
+		// xref the subgraphs
+		g.parentSubgraphs[unscopedG] = true
+		unscopedG.childSubgraphs[g] = true
+	}
+
+	return
+}
+
+// Graph traversal functions
+
+// Roots returns all nodes which are considered as root resources based on the current state
+//
+// For the most part, these are all resources with no parent resources.
+// If all resources define parents, then some home brew logic is ran
+func (g DepGraph) Roots() (out NodeSet) {
+	for _, sg := range g.graphs {
+		out = append(out, sg.Roots()...)
+	}
+	return
+}
+
+// ParentForRef returns a parent node of n which matches ref (nil if none)
+func (g DepGraph) ParentForRef(n *Node, ref Ref) (out *Node) {
+	for _, sg := range g.graphs {
+		out = sg.ParentForRef(n, ref)
+		if out != nil {
+			return
+		}
+	}
+	return
+}
+
+// ParentForRT returns a set of parent nodes matching the resource type
+func (g DepGraph) ParentForRT(n *Node, rt string) (out NodeSet) {
+	for _, sg := range g.graphs {
+		out = sg.ParentForRT(n, rt)
+		if out != nil {
+			return
+		}
+	}
+	return
+}
+
+// ChildrenForResourceType returns child nodes of n which match the resource type
+func (g DepGraph) ChildrenForResourceType(n *Node, rt string) (out NodeSet) {
+	for _, sg := range g.graphs {
+		out = sg.ChildrenForResourceType(n, rt)
+		if out != nil {
+			return
+		}
+	}
+	return
+}
+
+func (g DepGraph) NodeForRef(ref Ref) (out *Node) {
+	aux := make([]*Node, 0, 10)
+	for _, sg := range g.graphs {
+		for _, n := range sg.nodes {
+			aux = append(aux, n.Node)
+		}
+	}
+
+	return NodeForRef(ref, aux...)
+}
+
+// Children returns all child nodes of n
+func (g DepGraph) Children(n *Node) (out NodeSet) {
+	for _, sg := range g.graphs {
+		out = sg.Children(n)
+		if out != nil {
+			return
+		}
+	}
+	return
+}
+
+// MissingRegs returns a slice of all refs that are requested but not found in the graph
+func (g DepGraph) MissingRefs() (out []map[string]Ref) {
+	for _, sg := range g.graphs {
+		for _, n := range sg.nodes {
+			if len(n.missingReferences) == 0 {
+				continue
+			}
+			out = append(out, n.missingReferences)
+		}
+	}
+
+	return
+}
+
+func (g DepGraph) allNodes() (out []*depNode) {
+	for _, sg := range g.graphs {
+		out = append(out, sg.nodes...)
+	}
+
+	return
+}
+
+// Roots returns all nodes which are considered as root resources based on the current state
+//
+// For the most part, these are all resources with no parent resources.
+// If all resources define parents, then some home brew logic is ran
+//
+// @todo when we add more resources, we might need to expand this; for now
+//       it should work just fine.
+func (g depSubgraph) Roots() (out NodeSet) {
+	for _, n := range g.nodes {
+		if needyResources[n.Node.ResourceType] {
+			continue
+		}
+		out = append(out, n.Node)
+	}
+
+	if len(out) != 0 {
+		return
+	}
+
+	for _, n := range g.nodes {
+		if superNeedyResources[n.Node.ResourceType] {
+			continue
+		}
+		out = append(out, n.Node)
+	}
+
+	return
+}
+
+// ParentForRef returns a parent node of n which matches ref (nil if none)
+func (g depSubgraph) ParentForRef(n *Node, ref Ref) *Node {
+	return NodeForRef(ref, g.parent(n)...)
+}
+
+// ParentForRT returns a set of parent nodes matching the resource type
+func (g depSubgraph) ParentForRT(n *Node, rt string) NodeSet {
+	return NodesForResourceType(rt, g.parent(n)...)
+}
+
+// ChildrenForResourceType returns child nodes of n which match the resource type
+func (g depSubgraph) ChildrenForResourceType(n *Node, rt string) (out NodeSet) {
+	out = g.Children(n)
+	out = NodesForResourceType(rt, out...)
+	return
+}
+
+// Children returns all child nodes of n
+func (g depSubgraph) Children(n *Node) (out NodeSet) {
+	aux, ok := g.nodeMap[n]
+	if !ok {
+		return
+	}
+
+	for n := range aux.children {
+		out = append(out, n.Node)
+	}
+	return
+}
+
+func (g depSubgraph) parent(n *Node) (out NodeSet) {
+	aux, ok := g.nodeMap[n]
+	if !ok {
+		return
+	}
+
+	for n := range aux.parents {
+		out = append(out, n.Node)
+	}
+	return
+}
+
+// Utility functions
+
+func (g *depSubgraph) nodesWithMissingRefs() (out []*depNode) {
+	out = make([]*depNode, 0, 3)
+	for _, n := range g.nodes {
+		if len(n.missingReferences) > 0 {
+			out = append(out, n)
+		}
+	}
+
+	return
+}
+
+func unscopedSubgraph(gg ...*depSubgraph) *depSubgraph {
+	for _, g := range gg {
+		if g.scope.IsEmpty() {
+			return g
+		}
+	}
+
+	return nil
+}
+
+// depNodeForRef returns a node which matches the ref (nil if none)
+func depNodeForRef(ref Ref, nn ...*depNode) (out *depNode) {
+	for _, n := range nn {
+		if n.Node.ResourceType != ref.ResourceType {
+			continue
+		}
+
+		if n.Node.Identifiers.HasIntersection(ref.Identifiers) {
+			return n
+		}
+	}
+
+	return
+}
+
+// depNodesByResourceType returns a map where key is resource type, value slice of nodes
+func depNodesByResourceType(nn ...*depNode) (out map[string][]*depNode) {
+	out = make(map[string][]*depNode, 4)
+	for _, n := range nn {
+		out[n.Node.ResourceType] = append(out[n.Node.ResourceType], n)
+	}
+
+	return
+}
+
+// unpackDepNodes extracts envoy Nodes from the dep. nodes
+//
+// The function does no validation nor filtering for nil values.
+func unpackDepNodes(nn ...*depNode) (out NodeSet) {
+	out = make(NodeSet, 0, len(nn))
+	for _, n := range nn {
+		out = append(out, n.Node)
+	}
+	return out
+}

server/pkg/envoyx/dep_test.go

@@ -0,0 +1,389 @@
+package envoyx
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/cortezaproject/corteza/server/compose/types"
+	systemTypes "github.com/cortezaproject/corteza/server/system/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestScopeNodes(t *testing.T) {
+	req := require.New(t)
+
+	t.Run("none scoped", func(t *testing.T) {
+		a := &Node{}
+		b := &Node{}
+		c := &Node{}
+
+		ss := scopeNodes(a, b, c)
+		req.Len(ss, 1)
+		req.Len(ss[0], 3)
+		req.Contains(ss[0], a)
+	})
+
+	t.Run("all same scope", func(t *testing.T) {
+		a := &Node{
+			Scope: Scope{
+				ResourceType: types.NamespaceResourceType,
+				Identifiers:  MakeIdentifiers("a"),
+			},
+		}
+		b := &Node{
+			Scope: Scope{
+				ResourceType: types.NamespaceResourceType,
+				Identifiers:  MakeIdentifiers("a"),
+			},
+		}
+		c := &Node{
+			Scope: Scope{
+				ResourceType: types.NamespaceResourceType,
+				Identifiers:  MakeIdentifiers("a"),
+			},
+		}
+
+		ss := scopeNodes(a, b, c)
+		req.Len(ss, 1)
+		req.Len(ss[0], 3)
+		req.Contains(ss[0], a)
+	})
+
+	t.Run("mixed scope", func(t *testing.T) {
+		none1 := &Node{}
+		none2 := &Node{}
+
+		a1 := &Node{
+			Scope: Scope{
+				ResourceType: types.NamespaceResourceType,
+				Identifiers:  MakeIdentifiers("a"),
+			},
+		}
+		a2 := &Node{
+			Scope: Scope{
+				ResourceType: types.NamespaceResourceType,
+				Identifiers:  MakeIdentifiers("a"),
+			},
+		}
+
+		b1 := &Node{
+			Scope: Scope{
+				ResourceType: types.NamespaceResourceType,
+				Identifiers:  MakeIdentifiers("b"),
+			},
+		}
+		b2 := &Node{
+			Scope: Scope{
+				ResourceType: types.NamespaceResourceType,
+				Identifiers:  MakeIdentifiers("b"),
+			},
+		}
+
+		ss := scopeNodes(none1, none2, a1, a2, b1, b2)
+		req.Len(ss, 3)
+
+		req.Len(ss[0], 2)
+		req.Len(ss[1], 2)
+		req.Len(ss[2], 2)
+
+		req.True(ss[0][0].Scope.Identifiers.HasIntersection(ss[0][1].Scope.Identifiers))
+		req.True(ss[1][0].Scope.Identifiers.HasIntersection(ss[1][1].Scope.Identifiers))
+		req.True(ss[2][0].Scope.Identifiers.HasIntersection(ss[2][1].Scope.Identifiers))
+	})
+}
+
+func TestDepRoots(t *testing.T) {
+	req := require.New(t)
+
+	t.Run("no nodes", func(t *testing.T) {
+		gg := BuildDepGraph()
+		req.Len(gg.Roots(), 0)
+	})
+
+	t.Run("single non-needy node", func(t *testing.T) {
+		a1 := &Node{
+			ResourceType: systemTypes.UserResourceType,
+			Identifiers:  MakeIdentifiers("A.1"),
+		}
+
+		gg := BuildDepGraph(a1)
+
+		req.Len(gg.Roots(), 1)
+		req.Contains(gg.Roots(), a1)
+	})
+
+	t.Run("simple compose ns-mod dep setup", func(t *testing.T) {
+		a1 := &Node{
+			ResourceType: types.NamespaceResourceType,
+			Identifiers:  MakeIdentifiers("A.1"),
+		}
+		b1 := &Node{
+			ResourceType: types.ModuleResourceType,
+			Identifiers:  MakeIdentifiers("B.1"),
+			References: map[string]Ref{
+				"NamespaceID": a1.ToRef(),
+			},
+		}
+
+		gg := BuildDepGraph(a1, b1)
+
+		req.Len(gg.Roots(), 1)
+		req.Contains(gg.Roots(), a1)
+	})
+
+	t.Run("simple compose mod-mod field dep setup", func(t *testing.T) {
+		a1 := &Node{
+			ResourceType: types.ModuleResourceType,
+			Identifiers:  MakeIdentifiers("A.1"),
+		}
+		b1 := &Node{
+			ResourceType: types.ModuleFieldResourceType,
+			Identifiers:  MakeIdentifiers("B.1"),
+			References: map[string]Ref{
+				"NamespaceID": a1.ToRef(),
+			},
+		}
+
+		gg := BuildDepGraph(a1, b1)
+
+		req.Len(gg.Roots(), 1)
+		req.Contains(gg.Roots(), a1)
+	})
+}
+
+func TestDepTraversal(t *testing.T) {
+	req := require.New(t)
+
+	ns1 := &Node{
+		ResourceType: types.NamespaceResourceType,
+		Identifiers:  MakeIdentifiers("ns1"),
+		Scope: Scope{
+			ResourceType: types.NamespaceResourceType,
+			Identifiers:  MakeIdentifiers("ns1"),
+		},
+	}
+	ns2 := &Node{
+		ResourceType: types.NamespaceResourceType,
+		Identifiers:  MakeIdentifiers("ns2"),
+		Scope: Scope{
+			ResourceType: types.NamespaceResourceType,
+			Identifiers:  MakeIdentifiers("ns2"),
+		},
+	}
+
+	ns1mod1 := &Node{
+		ResourceType: types.ModuleResourceType,
+		Identifiers:  MakeIdentifiers("mod"),
+		References: map[string]Ref{
+			"NamespaceID": ns1.ToRef(),
+		},
+		Scope: ns1.Scope,
+	}
+	ns1mod1f1 := &Node{
+		ResourceType: types.ModuleFieldResourceType,
+		Identifiers:  MakeIdentifiers("f1"),
+		References: map[string]Ref{
+			"NamespaceID": ns1.ToRef(),
+			"ModuleID":    ns1mod1.ToRef(),
+		},
+		Scope: ns1.Scope,
+	}
+
+	ns2mod1 := &Node{
+		ResourceType: types.ModuleResourceType,
+		Identifiers:  MakeIdentifiers("mod"),
+		References: map[string]Ref{
+			"NamespaceID": ns2.ToRef(),
+		},
+		Scope: ns2.Scope,
+	}
+	ns2mod1f1 := &Node{
+		ResourceType: types.ModuleFieldResourceType,
+		Identifiers:  MakeIdentifiers("f1"),
+		References: map[string]Ref{
+			"NamespaceID": ns2.ToRef(),
+			"ModuleID":    ns2mod1.ToRef(),
+		},
+		Scope: ns2.Scope,
+	}
+
+	gg := BuildDepGraph(ns1, ns2, ns1mod1, ns1mod1f1, ns2mod1, ns2mod1f1)
+
+	t.Run("children of namespace root", func(t *testing.T) {
+		cc := gg.Children(ns1)
+
+		req.Len(cc, 2)
+		req.Contains(cc, ns1mod1)
+		req.Contains(cc, ns1mod1f1)
+	})
+
+	t.Run("children module", func(t *testing.T) {
+		cc := gg.Children(ns1mod1)
+
+		req.Len(cc, 1)
+		req.Contains(cc, ns1mod1f1)
+	})
+
+	t.Run("parent by ref missing", func(t *testing.T) {
+		p := gg.ParentForRef(ns1mod1, Ref{ResourceType: types.NamespaceResourceType, Identifiers: MakeIdentifiers("asdf"), Scope: ns1.Scope})
+
+		req.Nil(p)
+	})
+
+	t.Run("parent by ref wrong scope", func(t *testing.T) {
+		p := gg.ParentForRef(ns1mod1, Ref{ResourceType: types.NamespaceResourceType, Identifiers: ns2.Identifiers, Scope: ns2.Scope})
+
+		req.Nil(p)
+	})
+
+	t.Run("parent by ref", func(t *testing.T) {
+		p := gg.ParentForRef(ns1mod1, Ref{ResourceType: types.NamespaceResourceType, Identifiers: ns1.Identifiers, Scope: ns1.Scope})
+
+		req.NotNil(p)
+		req.Equal(ns1, p)
+	})
+}
+
+func TestDepXLinking(t *testing.T) {
+	req := require.New(t)
+
+	s1 := &Node{
+		ResourceType: systemTypes.UserResourceType,
+		Identifiers:  MakeIdentifiers("u1"),
+	}
+	s2 := &Node{
+		ResourceType: systemTypes.UserResourceType,
+		Identifiers:  MakeIdentifiers("u2"),
+		Scope: Scope{
+			ResourceType: types.NamespaceResourceType,
+			Identifiers:  MakeIdentifiers("ns1"),
+		},
+	}
+
+	ns1 := &Node{
+		ResourceType: types.NamespaceResourceType,
+		Identifiers:  MakeIdentifiers("ns1"),
+		References: map[string]Ref{
+			"S1": s1.ToRef(),
+		},
+		Scope: Scope{
+			ResourceType: types.NamespaceResourceType,
+			Identifiers:  MakeIdentifiers("ns1"),
+		},
+	}
+	ns2 := &Node{
+		ResourceType: types.NamespaceResourceType,
+		Identifiers:  MakeIdentifiers("ns2"),
+		References: map[string]Ref{
+			"S2": s2.ToRef(),
+		},
+		Scope: Scope{
+			ResourceType: types.NamespaceResourceType,
+			Identifiers:  MakeIdentifiers("ns2"),
+		},
+	}
+	_ = ns2
+
+	t.Run("scoped node access unscoped ref", func(t *testing.T) {
+		gg := BuildDepGraph(s1, s2, ns1)
+		req.NotNil(gg.ParentForRef(ns1, s1.ToRef()))
+	})
+
+	t.Run("does not have missing refs", func(t *testing.T) {
+		gg := BuildDepGraph(s1, s2, ns1)
+		mm := gg.MissingRefs()
+		req.Len(mm, 0)
+	})
+
+	t.Run("scoped node prevented ref of different scope", func(t *testing.T) {
+		gg := BuildDepGraph(s1, s2, ns2)
+		req.Nil(gg.ParentForRef(ns2, s2.ToRef()))
+	})
+
+	t.Run("has missing refs", func(t *testing.T) {
+		gg := BuildDepGraph(s1, s2, ns2)
+		mm := gg.MissingRefs()
+		req.Len(mm, 1)
+	})
+}
+
+// goos: linux
+// goarch: amd64
+// pkg: github.com/cortezaproject/corteza/server/pkg/envoyx
+// cpu: Intel(R) Core(TM) i7-8750H CPU @ 2.20GHz
+// BenchmarkDepGraphConstruction-12    	     826	   1495619 ns/op	 1633445 B/op	    6168 allocs/op
+// PASS
+func BenchmarkDepGraphConstruction(b *testing.B) {
+	ns1 := &Node{
+		ResourceType: types.NamespaceResourceType,
+		Identifiers:  MakeIdentifiers("ns1"),
+		Scope: Scope{
+			ResourceType: types.NamespaceResourceType,
+			Identifiers:  MakeIdentifiers("ns1"),
+		},
+	}
+	ns2 := &Node{
+		ResourceType: types.NamespaceResourceType,
+		Identifiers:  MakeIdentifiers("ns2"),
+		Scope: Scope{
+			ResourceType: types.NamespaceResourceType,
+			Identifiers:  MakeIdentifiers("ns2"),
+		},
+	}
+
+	ns1mod1 := &Node{
+		ResourceType: types.ModuleResourceType,
+		Identifiers:  MakeIdentifiers("mod"),
+		References: map[string]Ref{
+			"NamespaceID": ns1.ToRef(),
+		},
+		Scope: ns1.Scope,
+	}
+	ns1mod1f1 := &Node{
+		ResourceType: types.ModuleFieldResourceType,
+		Identifiers:  MakeIdentifiers("f1"),
+		References: map[string]Ref{
+			"NamespaceID": ns1.ToRef(),
+			"ModuleID":    ns1mod1.ToRef(),
+		},
+		Scope: ns1.Scope,
+	}
+
+	ns2mod1 := &Node{
+		ResourceType: types.ModuleResourceType,
+		Identifiers:  MakeIdentifiers("mod"),
+		References: map[string]Ref{
+			"NamespaceID": ns2.ToRef(),
+		},
+		Scope: ns2.Scope,
+	}
+	ns2mod1f1 := &Node{
+		ResourceType: types.ModuleFieldResourceType,
+		Identifiers:  MakeIdentifiers("f1"),
+		References: map[string]Ref{
+			"NamespaceID": ns2.ToRef(),
+			"ModuleID":    ns2mod1.ToRef(),
+		},
+		Scope: ns2.Scope,
+	}
+
+	qwerty := make(NodeSet, 0, 1000-6)
+	for i := 0; i < 1000-6; i++ {
+		qwerty = append(qwerty, &Node{
+			ResourceType: types.ModuleFieldResourceType,
+			Identifiers:  MakeIdentifiers(fmt.Sprintf("gg_f_%d", i)),
+			References: map[string]Ref{
+				"NamespaceID": ns1.ToRef(),
+				"ModuleID":    ns1mod1.ToRef(),
+			},
+			Scope: ns1.Scope,
+		})
+	}
+
+	qwerty = append(qwerty, NodeSet{ns1, ns2, ns1mod1, ns1mod1f1, ns2mod1, ns2mod1f1}...)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		BuildDepGraph(qwerty...)
+	}
+}

server/pkg/envoyx/envoy.go

@@ -0,0 +1,338 @@
+package envoyx
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/expr"
+)
+
+type (
+	Service struct {
+		decoders map[decodeType][]Decoder
+
+		encoders  map[encodeType][]Encoder
+		preparers map[encodeType][]Preparer
+	}
+
+	// Traverser provides a structure which can be used to traverse the node's deps
+	Traverser interface {
+		// ParentForRef returns the parent of the provided node which matches the ref
+		//
+		// If no parent is found, nil is returned.
+		ParentForRef(*Node, Ref) *Node
+
+		// ParentForRT returns a set of parent nodes matching the resource type
+		ParentForRT(*Node, string) NodeSet
+
+		// ChildrenForResourceType returns the children of the provided node which
+		// match the provided resource type
+		ChildrenForResourceType(*Node, string) NodeSet
+
+		// Children returns all of the children of the provided node
+		Children(*Node) NodeSet
+
+		// NodeForRef returns the node which matches the provided ref
+		NodeForRef(Ref) *Node
+	}
+
+	Preparer interface {
+		// Prepare performs generic preprocessing on the provided nodes
+		//
+		// The function is called for every resource type where all of the nodes of
+		// that resource type are passed as the argument.
+		Prepare(context.Context, EncodeParams, string, NodeSet) error
+	}
+
+	Encoder interface {
+		// Encode encodes the data
+		//
+		// The function receives a set of root-level nodes (with no parent dependencies)
+		// and a Traverser it can use to handle all of the child nodes.
+		Encode(context.Context, EncodeParams, string, NodeSet, Traverser) (err error)
+	}
+
+	PrepareEncoder interface {
+		Preparer
+		Encoder
+	}
+
+	Decoder interface {
+		// Decode returns a set of Nodes extracted based on the provided definition
+		Decode(ctx context.Context, p DecodeParams) (out NodeSet, err error)
+	}
+
+	canCheckFile interface {
+		CanFile(f *os.File) bool
+	}
+
+	canCheckExt interface {
+		CanExt(name string) bool
+	}
+
+	DecodeParams struct {
+		Type   decodeType
+		Params map[string]any
+		Config map[string]any
+		Filter map[string]ResourceFilter
+	}
+
+	EncodeParams struct {
+		Type    encodeType
+		Params  map[string]any
+		Envoy   EnvoyConfig
+		Encoder EncoderConfig
+
+		FieldMapping []MapEntry
+
+		// @note these are only used by records since v1 did just that
+		// @todo make these more expanded
+		DeferOk  func()
+		DeferNok func(error) error
+		Defer    func()
+	}
+	EncoderConfig struct {
+		DefaultUserID       uint64
+		PreferredTimeLayout string
+		PreferredTimezone   string
+	}
+
+	ResourceFilter struct {
+		Identifiers Identifiers
+		Refs        map[string]Ref
+
+		Limit uint
+		Query string
+		Scope Scope
+	}
+
+	decodeType string
+	encodeType string
+	mergeAlg   int
+)
+
+var (
+	global *Service
+
+	ex = expr.NewParser()
+)
+
+const (
+	OnConflictDefault mergeAlg = iota
+	OnConflictReplace
+	OnConflictSkip
+	OnConflictPanic
+	// OnConflictMergeLeft  mergeAlg = "mergeLeft"
+	// OnConflictMergeRight mergeAlg = "mergeRight"
+
+	DecodeTypeURI   decodeType = "uri"
+	DecodeTypeIO    decodeType = "io"
+	DecodeTypeStore decodeType = "store"
+
+	EncodeTypeURI   encodeType = "uri"
+	EncodeTypeStore encodeType = "store"
+	EncodeTypeIo    encodeType = "io"
+)
+
+// New initializes a new Envoy service
+func New() *Service {
+	return &Service{}
+}
+
+// SetGlobal sets the global envoy service
+func SetGlobal(n *Service) {
+	global = n
+}
+
+// Global gets the global envoy service
+func Global() *Service {
+	if global == nil {
+		panic("global service not defined")
+	}
+
+	return global
+}
+
+func Initialized() bool {
+	return global != nil
+}
+
+// Decode returns a set of envoy Nodes based on the given decode params
+func (svc *Service) Decode(ctx context.Context, p DecodeParams) (nodes NodeSet, providers []Provider, err error) {
+	err = p.validate()
+	if err != nil {
+		return
+	}
+
+	switch p.Type {
+	case DecodeTypeURI:
+		return svc.decodeUri(ctx, p)
+	case DecodeTypeIO:
+		return svc.decodeIo(ctx, p)
+	case DecodeTypeStore:
+		nodes, err = svc.decodeStore(ctx, p)
+		return
+	default:
+		err = fmt.Errorf("unsupported decoder type %s", p.Type)
+	}
+
+	return
+}
+
+func (svc *Service) Bake(ctx context.Context, p EncodeParams, providers []Provider, nodes ...*Node) (gg *DepGraph, err error) {
+	err = svc.bakeEnvoyConfig(p.Envoy, nodes...)
+	if err != nil {
+		return
+	}
+
+	err = svc.bakeExpressions(nodes...)
+	if err != nil {
+		return
+	}
+
+	svc.bakeProviders(providers, nodes...)
+
+	gg = BuildDepGraph(nodes...)
+	return
+}
+
+// Encode encodes Corteza resources bases on the provided encode params
+//
+// use the BuildDepGraph function to build the default dependency graph.
+func (svc *Service) Encode(ctx context.Context, p EncodeParams, dg *DepGraph) (err error) {
+	err = p.validate()
+	if err != nil {
+		return
+	}
+
+	switch p.Type {
+	case EncodeTypeStore:
+		return svc.encodeStore(ctx, dg, p)
+	case EncodeTypeIo:
+		return svc.encodeIo(ctx, dg, p)
+	default:
+		err = fmt.Errorf("unsupported encoder type %s", p.Type)
+	}
+	return
+}
+
+func (svc *Service) AddDecoder(t decodeType, dd ...Decoder) {
+	if svc.decoders == nil {
+		svc.decoders = make(map[decodeType][]Decoder)
+	}
+	svc.decoders[t] = append(svc.decoders[t], dd...)
+}
+
+func (svc *Service) AddEncoder(t encodeType, ee ...Encoder) {
+	if svc.encoders == nil {
+		svc.encoders = make(map[encodeType][]Encoder)
+	}
+	if svc.preparers == nil {
+		svc.preparers = make(map[encodeType][]Preparer)
+	}
+	svc.encoders[t] = append(svc.encoders[t], ee...)
+
+	for _, e := range ee {
+		p, ok := e.(Preparer)
+		if !ok {
+			continue
+		}
+
+		svc.preparers[t] = append(svc.preparers[t], p)
+	}
+}
+
+func (svc *Service) AddPreparer(t encodeType, pp ...Preparer) {
+	if svc.preparers == nil {
+		svc.preparers = make(map[encodeType][]Preparer)
+	}
+	svc.preparers[t] = append(svc.preparers[t], pp...)
+}
+
+// Utility functions
+
+func (p DecodeParams) validate() (err error) {
+	switch p.Type {
+	case DecodeTypeURI:
+		_, ok := p.Params["uri"]
+		if !ok {
+			return fmt.Errorf("URI decoder expects a uri location parameter")
+		}
+
+	case DecodeTypeStore:
+
+	}
+
+	// @todo...
+
+	return
+}
+
+func (p EncodeParams) validate() (err error) {
+	switch p.Type {
+	// @todo...
+	}
+
+	// @todo...
+
+	return
+}
+
+func CastMergeAlg(v string) (mergeAlg mergeAlg) {
+	switch strings.ToLower(v) {
+	case "replace", "mergeleft":
+		mergeAlg = OnConflictReplace
+	case "skip", "mergeright":
+		mergeAlg = OnConflictSkip
+	case "panic", "error":
+		mergeAlg = OnConflictPanic
+	}
+
+	return
+}
+
+func (svc *Service) bakeEnvoyConfig(dft EnvoyConfig, nodes ...*Node) (err error) {
+	for _, n := range nodes {
+		n.Config = svc.mergeEnvoyConfigs(n.Config, dft)
+	}
+
+	return
+}
+
+func (svc *Service) bakeExpressions(nodes ...*Node) (err error) {
+	for _, n := range nodes {
+		if n.Config.SkipIf == "" {
+			continue
+		}
+
+		n.Config.SkipIfEval, err = ex.Parse(n.Config.SkipIf)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (svc *Service) bakeProviders(providers []Provider, nodes ...*Node) {
+	SetDecoderSources(nodes, providers...)
+}
+
+func (svc *Service) mergeEnvoyConfigs(a, b EnvoyConfig) (c EnvoyConfig) {
+	c = a
+	if c.MergeAlg == OnConflictDefault {
+		c.MergeAlg = b.MergeAlg
+	}
+	if c.MergeAlg == OnConflictDefault {
+		c.MergeAlg = OnConflictReplace
+	}
+
+	// @todo pre eval this?
+	if c.SkipIf == "" {
+		c.SkipIf = b.SkipIf
+	}
+
+	return
+}

server/pkg/envoyx/envoy_test.go

@@ -0,0 +1,66 @@
+package envoyx
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestBake(t *testing.T) {
+	ctx := context.Background()
+	req := require.New(t)
+
+	t.Run("set default envoy config", func(t *testing.T) {
+		a := &Node{
+			Config: EnvoyConfig{
+				MergeAlg: OnConflictDefault,
+				SkipIf:   "",
+			},
+		}
+		b := &Node{
+			Config: EnvoyConfig{
+				MergeAlg: OnConflictReplace,
+				SkipIf:   "",
+			},
+		}
+		c := &Node{
+			Config: EnvoyConfig{
+				MergeAlg: OnConflictReplace,
+				SkipIf:   "true",
+			},
+		}
+		_, err := (&Service{}).Bake(ctx, EncodeParams{Envoy: EnvoyConfig{MergeAlg: OnConflictPanic, SkipIf: "false"}}, nil, a, b, c)
+		req.NoError(err)
+
+		req.Equal(OnConflictPanic, a.Config.MergeAlg)
+		req.Equal("false", a.Config.SkipIf)
+
+		req.Equal(OnConflictReplace, b.Config.MergeAlg)
+		req.Equal("false", b.Config.SkipIf)
+
+		req.Equal(OnConflictReplace, c.Config.MergeAlg)
+		req.Equal("true", c.Config.SkipIf)
+	})
+
+	t.Run("precompute expressions", func(t *testing.T) {
+		a := &Node{
+			Config: EnvoyConfig{
+				MergeAlg: OnConflictDefault,
+				SkipIf:   "",
+			},
+		}
+		b := &Node{
+			Config: EnvoyConfig{
+				MergeAlg: OnConflictReplace,
+				SkipIf:   "a == b",
+			},
+		}
+
+		_, err := (&Service{}).Bake(ctx, EncodeParams{}, nil, a, b)
+		req.NoError(err)
+
+		req.Nil(a.Config.SkipIfEval)
+		req.NotNil(b.Config.SkipIfEval)
+	})
+}

server/pkg/envoyx/io.go

@@ -0,0 +1,187 @@
+package envoyx
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoyx/csv"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx/json"
+)
+
+func (svc *Service) decodeIo(ctx context.Context, p DecodeParams) (nodes NodeSet, providers []Provider, err error) {
+	// Get and validate the reader param
+	r, ok := p.Params["reader"]
+	if !ok {
+		err = fmt.Errorf("cannot decode IO: no reader parameter provided")
+		return
+	}
+	reader, ok := r.(io.Reader)
+	if !ok {
+		err = fmt.Errorf("cannot decode IO: reader should be io.Reader encoded")
+		return
+	}
+
+	// Get and validate the mimetype param
+	_, ok = p.Params["mime"].(string)
+	if !ok {
+		// @todo guess mime type
+		p.Params["mime"] = "text/yaml"
+	}
+
+	// @todo different ext based on mimetype -- currently yaml is the only supported one
+	tmpFileName := "*.yaml"
+	f, err := os.CreateTemp(os.TempDir(), tmpFileName)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	defer os.Remove(f.Name())
+
+	_, err = io.Copy(f, reader)
+	if err != nil {
+		return
+	}
+
+	return svc.decodeFile(ctx, p, os.TempDir(), f.Name())
+}
+
+func (svc *Service) decodeUri(ctx context.Context, p DecodeParams) (nodes NodeSet, providers []Provider, err error) {
+	aUri, ok := p.Params["uri"]
+	if !ok {
+		err = fmt.Errorf("cannot decode URI: no uri parameter provided")
+		return
+	}
+
+	uri, ok := aUri.(string)
+	if !ok {
+		err = fmt.Errorf("cannot decode URI: uri should be string encoded: got %v", aUri)
+		return
+	}
+
+	pp := strings.Split(uri, "://")
+
+	proto := pp[0]
+	rest := pp[1]
+
+	switch proto {
+	case "file":
+		fileInfo, err := os.Stat(rest)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if fileInfo.IsDir() {
+			// decode whole directory
+			return svc.decodeDirectory(ctx, p, rest)
+		}
+
+		// decode specific file
+		var base string
+		bits := strings.Split(rest, string(os.PathSeparator))
+		base = strings.Join(bits[0:len(bits)-1], string(os.PathSeparator))
+		return svc.decodeFile(ctx, p, base, rest)
+
+	default:
+		err = fmt.Errorf("unsupported URI protocol %s", proto)
+		return
+	}
+}
+
+func (nvyx *Service) encodeIo(ctx context.Context, dg *DepGraph, p EncodeParams) (err error) {
+	for rt, nn := range NodesByResourceType(dg.Roots()...) {
+		for _, se := range nvyx.encoders[EncodeTypeIo] {
+			nn = OmitPlaceholderNodes(nn...)
+			if len(nn) == 0 {
+				continue
+			}
+
+			err = se.Encode(ctx, p, rt, OmitPlaceholderNodes(nn...), dg)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	return
+}
+
+func (nvyx *Service) decodeDirectory(ctx context.Context, p DecodeParams, path string) (nodes NodeSet, providers []Provider, err error) {
+	basePath := path
+	return nodes, providers, filepath.Walk(path, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// @todo consider supporting nested directories
+		if info.IsDir() {
+			return nil
+		}
+
+		auxNodes, auxProviders, err := nvyx.decodeFile(ctx, p, basePath, path)
+		if err != nil {
+			return err
+		}
+
+		nodes = append(nodes, auxNodes...)
+		providers = append(providers, auxProviders...)
+		return nil
+	})
+}
+
+func (nvyx *Service) decodeFile(ctx context.Context, p DecodeParams, basePath, path string) (nodes NodeSet, providers []Provider, err error) {
+	var aux NodeSet
+	f, err := os.Open(path)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+
+	p.Params["stream"] = f
+
+	for _, d := range nvyx.decoders[DecodeTypeURI] {
+		if cc, ok := d.(canCheckFile); ok {
+			if !cc.CanFile(f) {
+				continue
+			}
+		}
+
+		aux, err = d.Decode(ctx, p)
+		if err != nil {
+			return
+		}
+		nodes = append(nodes, aux...)
+
+		_, err = f.Seek(0, 0)
+		if err != nil {
+			return
+		}
+	}
+
+	providerIdent := strings.Replace(f.Name(), basePath, "", -1)
+	providerIdent = strings.TrimLeft(providerIdent, "/")
+
+	if csv.CanDecodeFile(f) || csv.CanDecodeExt(f.Name()) {
+		f.Seek(0, 0)
+		aux, err := csv.Decoder(f, providerIdent, p.Config)
+		if err != nil {
+			return nil, nil, err
+		}
+		providers = append(providers, aux)
+	}
+
+	if json.CanDecodeFile(f) || json.CanDecodeExt(f.Name()) {
+		f.Seek(0, 0)
+		aux, err := json.Decoder(f, providerIdent)
+		if err != nil {
+			return nil, nil, err
+		}
+		providers = append(providers, aux)
+	}
+
+	return
+}

server/pkg/envoyx/json/decoder.go

@@ -0,0 +1,170 @@
+package json
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoyx/datasource"
+	"github.com/gabriel-vasile/mimetype"
+)
+
+type (
+	decoder struct {
+		ident string
+
+		src    *os.File
+		reader *json.Decoder
+
+		header []string
+		count  uint64
+	}
+)
+
+func CanDecodeFile(f io.Reader) bool {
+	m, err := mimetype.DetectReader(f)
+	if err != nil {
+		return false
+	}
+
+	return CanDecodeExt(m.Extension())
+}
+
+func CanDecodeMime(m string) bool {
+	return m == "application/json" || m == "application/jsonlines"
+}
+
+func CanDecodeExt(ext string) bool {
+	pt := strings.Split(ext, ".")
+	ext = strings.TrimSpace(pt[len(pt)-1])
+	return ext == "jsonl" || ext == "json" || ext == "ndjson"
+}
+
+// Decoder inits a new csv decoder from the given reader
+//
+// @todo hold small files in mem to avoid needles disc access
+func Decoder(r io.Reader, ident string) (out *decoder, err error) {
+	out = &decoder{
+		ident: ident,
+	}
+
+	out.src, err = ioutil.TempFile(os.TempDir(), "*.ndjson")
+	if err != nil {
+		return
+	}
+
+	err = out.flushTemp(r)
+	defer out.src.Seek(0, 0)
+	if err != nil {
+		return
+	}
+
+	out.reader = json.NewDecoder(out.src)
+
+	seenHeader := make(map[string]bool)
+
+	var aux datasource.RawRecord
+	for out.reader.More() {
+		err = out.reader.Decode(&aux)
+		if err == io.EOF {
+			return out, nil
+		} else if err != nil {
+			return
+		}
+
+		for f := range aux {
+			if seenHeader[f] {
+				continue
+			}
+			seenHeader[f] = true
+
+			out.header = append(out.header, f)
+		}
+
+		// Entry count
+		out.count++
+	}
+
+	return
+}
+
+func (d *decoder) SetConfigs(config map[string]any) (err error) {
+	// @todo when needed
+	return
+}
+
+// Cleanup should be called before we stop using the decoder
+func (d *decoder) Cleanup() error {
+	return os.Remove(d.src.Name())
+}
+
+// SetIdent overwrites the system defined identifier
+func (d *decoder) SetIdent(ident string) {
+	d.ident = ident
+}
+
+// Ident returns the assigned identifier
+func (d *decoder) Ident() string {
+	return d.ident
+}
+
+// Fields returns every available field in this dataset
+func (d *decoder) Fields() []string {
+	return d.header
+}
+
+// Reset resets the decoder to the start
+func (d *decoder) Reset(_ context.Context) error {
+	_, err := d.src.Seek(0, 0)
+	if err != nil {
+		return err
+	}
+
+	d.reader = json.NewDecoder(d.src)
+	return nil
+}
+
+// Next returns the field: value mapping for the next row
+func (d *decoder) Next(_ context.Context, out datasource.RawRecord) (more bool, err error) {
+	tmp := make(datasource.RawRecord)
+	err = d.reader.Decode(&tmp)
+
+	if err == io.EOF {
+		return false, nil
+	} else if err != nil {
+		return false, err
+	}
+
+	// Empty out missing fields to keep consistent with CSV
+	for _, h := range d.header {
+		if _, ok := tmp[h]; !ok {
+			vv := tmp[h]
+			vv.Values = make([]string, 0)
+			tmp[h] = vv
+		}
+	}
+
+	for k, v := range tmp {
+		out[k] = v
+	}
+
+	return true, nil
+}
+
+// Count returns the total number of rows in the dataset
+func (d *decoder) Count() uint64 {
+	return d.count
+}
+
+func (d *decoder) flushTemp(r io.Reader) (err error) {
+	_, err = io.Copy(d.src, r)
+	if err != nil {
+		return
+	}
+
+	d.src.Seek(0, 0)
+	return nil
+}

server/pkg/envoyx/json/decoder_test.go

@@ -0,0 +1,72 @@
+package json
+
+import (
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoyx/datasource"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDecoder(t *testing.T) {
+	req := require.New(t)
+
+	t.Run("init & meta", func(t *testing.T) {
+		dc, err := Decoder(testReader(), "test.csv")
+		req.NoError(err)
+
+		req.Equal(uint64(3), dc.Count())
+	})
+
+	t.Run("fields", func(t *testing.T) {
+		dc, err := Decoder(testReader(), "test.csv")
+		req.NoError(err)
+
+		hh := dc.Fields()
+		req.Contains(hh, "f1")
+		req.Contains(hh, "f2")
+		req.Contains(hh, "f3")
+	})
+
+	t.Run("iterate", func(t *testing.T) {
+		dc, err := Decoder(testReader(), "test.csv")
+		req.NoError(err)
+
+		aux := make(datasource.RawRecord)
+		var more bool
+
+		more, err = dc.Next(nil, aux)
+		req.NoError(err)
+		req.True(more)
+		req.Equal("r1f1", aux["f1"].Values[0])
+		req.Equal("r1f2", aux["f2"].Values[0])
+		req.Equal("r1f3", aux["f3"].Values[0])
+
+		more, err = dc.Next(nil, aux)
+		req.NoError(err)
+		req.True(more)
+		req.Equal("r2f1", aux["f1"].Values[0])
+		req.Equal("r2f2", aux["f2"].Values[0])
+		req.Equal("r2f3", aux["f3"].Values[0])
+
+		more, err = dc.Next(nil, aux)
+		req.NoError(err)
+		req.True(more)
+		req.Equal("r3f1", aux["f1"].Values[0])
+		req.Equal("r3f2", aux["f2"].Values[0])
+		req.Equal("r3f3", aux["f3"].Values[0])
+
+		more, err = dc.Next(nil, aux)
+		req.NoError(err)
+		req.False(more)
+	})
+}
+
+func testReader() io.Reader {
+	src := `{"f1": "r1f1", "f2": "r1f2", "f3": "r1f3"}
+{"f1": "r2f1", "f2": "r2f2", "f3": "r2f3"}
+{"f1": "r3f1", "f2": "r3f2", "f3": "r3f3"}`
+
+	return strings.NewReader(src)
+}

server/pkg/envoyx/localeutils.go

@@ -0,0 +1,99 @@
+package envoyx
+
+import (
+	"fmt"
+
+	"github.com/cortezaproject/corteza/server/system/types"
+)
+
+type (
+	localer interface {
+		ResourceTranslation() string
+	}
+)
+
+func ResourceTranslationsForNodes(tt types.ResourceTranslationSet, nn ...*Node) (translations NodeSet, err error) {
+	translations = make(NodeSet, 0, len(tt)/2)
+	dups := make(map[types.Lang]map[string]map[string]bool)
+
+	for _, n := range nn {
+		if n.Placeholder {
+			continue
+		}
+
+		c, ok := n.Resource.(localer)
+		if !ok {
+			continue
+		}
+
+		// Split up the path of this resource
+		//
+		// @todo move over to those generated functions
+		resPath := splitResourcePath(c.ResourceTranslation())
+
+		// Find all of the translations that fall under this resource
+		for _, r := range tt {
+
+			// Split up the path of the rule
+			//
+			// @todo move over to that generated function
+			rulePath := splitResourcePath(r.Resource)
+			// @note resource translations don't support wildcards
+			if !isPathSubset(rulePath, resPath, false) {
+				// Mismatch; skip
+				continue
+			}
+
+			// Check if this translation has already been seen
+			if dups[r.Lang] != nil && dups[r.Lang][r.Resource][r.K] {
+				continue
+			}
+
+			// Parse the path so we can process it further
+			// @todo make a generic function for RBAC rules and res. tr.
+			localeRt, res, path, err := ParseRule(r.Resource)
+			if err != nil {
+				return nil, err
+			}
+
+			if localeRt != n.ResourceType {
+				// Resource type missmatch; skip
+				continue
+			}
+
+			// Get the refs
+			rf := make(map[string]Ref, 2)
+			for i, ref := range append(path, res) {
+				// Whenever you'd use a wildcard, it will produce a nil so it
+				// needs to be skipped
+				if ref == nil {
+					continue
+				}
+
+				ref.Scope = n.Scope
+
+				// @todo make the thing not a pointer
+				rf[fmt.Sprintf("Path.%d", i)] = *ref
+			}
+
+			translations = append(translations, &Node{
+				Resource: r,
+
+				ResourceType: types.ResourceTranslationResourceType,
+				References:   rf,
+				Scope:        n.Scope,
+			})
+
+			// Update the dup checking index
+			if dups[r.Lang] == nil {
+				dups[r.Lang] = make(map[string]map[string]bool)
+			}
+			if dups[r.Lang][r.Resource] == nil {
+				dups[r.Lang][r.Resource] = make(map[string]bool)
+			}
+			dups[r.Lang][r.Resource][r.K] = true
+		}
+	}
+
+	return
+}

server/pkg/envoyx/node.go

@@ -0,0 +1,350 @@
+package envoyx
+
+import (
+	"strconv"
+
+	"github.com/cortezaproject/corteza/server/pkg/expr"
+	"github.com/spf13/cast"
+)
+
+type (
+	// Node is a wrapper around a Corteza resource for use within Envoy
+	Node struct {
+		Resource   resource
+		Datasource Datasource
+
+		ResourceType string
+		Identifiers  Identifiers
+		References   map[string]Ref
+		Scope        Scope
+
+		// Placeholders are resources which were added to help resolve missing deps
+		Placeholder bool
+		Config      EnvoyConfig
+		Evaluated   Evaluated
+	}
+
+	Evaluated struct {
+		Skip bool
+	}
+
+	EnvoyConfig struct {
+		MergeAlg   mergeAlg
+		SkipIf     string
+		SkipIfEval expr.Evaluable
+	}
+
+	NodeSet []*Node
+
+	resource interface {
+		SetValue(name string, pos uint, value any) error
+		GetValue(name string, pos uint) (any, error)
+		GetID() uint64
+	}
+
+	Identifiers struct {
+		Slice []string
+		Index map[string]bool
+	}
+
+	// Scope lets us group nodes based on some common context
+	//
+	// Scope is primarily used to scope low-code applications to denote to what
+	// namespace a specific module reference belongs to.
+	// In the previous version this was referred to as reference constraints;
+	// This is the same but different.
+	//
+	// When constructing dependency graphs, nodes with the same scope are grouped together.
+	// Nodes from the same scope can reference each other.
+	// Nodes from a defined scope can reference nodes from an undefined scope,
+	// but not the other way around.
+	Scope struct {
+		ResourceType string
+		Identifiers  Identifiers
+	}
+
+	// Ref defines a reference to a different resource
+	//
+	// The reference only holds if all three parts match -- the resource type,
+	// there is an intersection between the identifiers, and the scope matches.
+	Ref struct {
+		ResourceType string
+		Identifiers  Identifiers
+		Scope        Scope
+		// @todo consider replacing with something that indicates
+		//       it can't be fetched from the DB
+		Optional bool
+	}
+
+	prunner interface {
+		Prune(rt string)
+	}
+)
+
+// MakeIdentifiers initializes an Identifiers instance from the given slice
+func MakeIdentifiers(ii ...any) (out Identifiers) {
+	return Identifiers{}.Add(ii...)
+}
+
+// NodesByResourceType returns Nodes grouped by their resource type
+func NodesByResourceType(nn ...*Node) (out map[string]NodeSet) {
+	out = make(map[string]NodeSet, len(nn)/2)
+	for _, n := range nn {
+		out[n.ResourceType] = append(out[n.ResourceType], n)
+	}
+
+	return
+}
+
+// NodesForResourceType returns which belong to the given resource type
+func NodesForResourceType(rt string, nn ...*Node) (out NodeSet) {
+	return NodesByResourceType(nn...)[rt]
+}
+
+// NodeForRef returns the Node that matches the given ref
+func NodeForRef(ref Ref, nn ...*Node) (out *Node) {
+	for _, n := range nn {
+		if !n.Scope.Equals(ref.Scope) {
+			continue
+		}
+
+		if n.ResourceType != ref.ResourceType {
+			continue
+		}
+
+		if n.Identifiers.HasIntersection(ref.Identifiers) {
+			return n
+		}
+	}
+
+	return
+}
+
+func OmitPlaceholderNodes(nn ...*Node) (out NodeSet) {
+	out = make(NodeSet, 0, len(nn))
+	for _, n := range nn {
+		if n.Placeholder {
+			continue
+		}
+		out = append(out, n)
+	}
+	return
+}
+
+func MergeRefs(a map[string]Ref, bb ...map[string]Ref) (c map[string]Ref) {
+	c = make(map[string]Ref)
+
+	for k, v := range a {
+		c[k] = v
+	}
+
+	for _, b := range bb {
+		for k, v := range b {
+			c[k] = v
+		}
+	}
+
+	return
+}
+
+// MergeIdents merges the two identifiers and returns a new one
+func MergeIdents(a, b Identifiers) (cc Identifiers) {
+	cc = Identifiers{
+		Index: make(map[string]bool, 2),
+	}
+
+	for a := range a.Index {
+		cc.Index[a] = true
+	}
+	for b := range b.Index {
+		cc.Index[b] = true
+	}
+
+	for c := range cc.Index {
+		cc.Slice = append(cc.Slice, c)
+	}
+
+	return
+}
+
+func (n Node) ToRef() Ref {
+	return Ref{
+		ResourceType: n.ResourceType,
+		Identifiers:  n.Identifiers,
+		Scope:        n.Scope,
+	}
+}
+
+func (n *Node) Prune(ref Ref) {
+	for k, nodeRef := range n.References {
+		if nodeRef.Equals(ref) {
+			delete(n.References, k)
+		}
+	}
+
+	if pp, ok := n.Resource.(prunner); ok {
+		// @todo improve when needed; for now it's ok
+		pp.Prune(ref.ResourceType)
+	}
+}
+
+func (r Ref) Idents() (ints []uint64, rest []string) {
+	return r.Identifiers.Idents()
+}
+
+// ResourceFilter returns a filter which would match the referenced resource
+func (r Ref) ResourceFilter() (out map[string]ResourceFilter) {
+	out = make(map[string]ResourceFilter)
+	out[r.ResourceType] = ResourceFilter{
+		Identifiers: r.Identifiers,
+		Scope:       r.Scope,
+
+		// A ref would point to a single resource.
+		// Don't set the limit so we can error out on ambiguity.
+		// Limit: 1,
+	}
+
+	return
+}
+
+func (a Ref) Equals(b Ref) bool {
+	if a.ResourceType != b.ResourceType {
+		return false
+	}
+	if !a.Scope.Equals(b.Scope) {
+		return false
+	}
+	if !a.Identifiers.HasIntersection(b.Identifiers) {
+		return false
+	}
+
+	return true
+}
+
+func (a Scope) Equals(b Scope) bool {
+	if a.IsEmpty() && b.IsEmpty() {
+		return true
+	}
+
+	if a.ResourceType != b.ResourceType {
+		return false
+	}
+
+	return a.Identifiers.HasIntersection(b.Identifiers)
+}
+
+func (s Scope) IsEmpty() bool {
+	return s.ResourceType == "" && len(s.Identifiers.Slice) == 0
+}
+
+// Add adds the given values to the identifier
+func (ii Identifiers) Add(vv ...any) (out Identifiers) {
+	if ii.Index == nil {
+		ii.Index = make(map[string]bool, len(vv))
+		ii.Slice = make([]string, 0, len(vv))
+	}
+
+	for _, v := range vv {
+		switch casted := v.(type) {
+		case string:
+			if casted == "" {
+				continue
+			}
+		case uint64, uint, int, int64:
+			if casted == 0 {
+				continue
+			}
+		}
+
+		if c, ok := v.(Identifiers); ok {
+			ii = ii.Merge(c)
+			continue
+		}
+
+		aux := cast.ToString(v)
+		if aux == "" || aux == "0" {
+			continue
+		}
+
+		ii.Slice = append(ii.Slice, aux)
+		ii.Index[aux] = true
+	}
+
+	return ii
+}
+
+func (ii Identifiers) IdentsAsStrings() (ids, rest []string) {
+	aux, rest := ii.Idents()
+
+	for _, a := range aux {
+		ids = append(ids, strconv.FormatUint(a, 10))
+	}
+
+	return
+}
+
+// Idents returns a slice of numeric and text identifiers
+func (ii Identifiers) Idents() (ints []uint64, rest []string) {
+	var aux uint64
+	var err error
+
+	for _, i := range ii.Slice {
+		aux, err = cast.ToUint64E(i)
+		if err != nil {
+			rest = append(rest, i)
+		} else {
+			ints = append(ints, aux)
+		}
+	}
+
+	return
+}
+
+// Intersection returns a slice of identifiers which are in an intersection
+func (aa Identifiers) Intersection(bb Identifiers) (out []string) {
+	for _, b := range bb.Slice {
+		if aa.Index[b] {
+			out = append(out, b)
+		}
+	}
+
+	return
+}
+
+// HasIntersection returns true if the two identifiers define an intersection
+func (aa Identifiers) HasIntersection(bb Identifiers) bool {
+	if len(aa.Slice) == 0 && len(bb.Slice) == 0 {
+		return true
+	}
+
+	// If the second identifies are empty we can assume that anything has an intersection with them.
+	// This is useful when we want to remove any resource of some type.
+	if len(bb.Slice) == 0 {
+		return true
+	}
+
+	return len(aa.Intersection(bb)) > 0
+}
+
+// Merge merges the two identifiers and returns a new one
+// @todo deprecate this; use MergeIdents instead
+func (aa Identifiers) Merge(bb Identifiers) (cc Identifiers) {
+	return MergeIdents(aa, bb)
+}
+
+// FriendlyIdentifier returns the best available identifier
+//
+// If any non-ID identifiers are available, it uses the first one.
+// If no non-ID identifiers are available, it returns the first ID.
+func (ii Identifiers) FriendlyIdentifier() (out string) {
+	a, b := ii.Idents()
+	if len(b) > 0 {
+		return b[0]
+	}
+
+	if len(a) > 0 {
+		return strconv.FormatUint(a[0], 10)
+	}
+
+	return
+}

server/pkg/envoyx/node_test.go

@@ -0,0 +1,118 @@
+package envoyx
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNodeForRef(t *testing.T) {
+	req := require.New(t)
+
+	ref := Ref{
+		ResourceType: "a",
+		Identifiers:  MakeIdentifiers("a1", "a2"),
+		Scope: Scope{
+			ResourceType: "a",
+			Identifiers:  MakeIdentifiers("a1", "a2"),
+		},
+	}
+
+	t.Run("no nodes defined", func(t *testing.T) {
+		req.Nil(NodeForRef(ref))
+	})
+
+	t.Run("not found", func(t *testing.T) {
+		nn := NodeForRef(ref, NodeSet{{
+			// The type misses
+			ResourceType: "b",
+			Identifiers:  MakeIdentifiers("a1", "a2"),
+			Scope: Scope{
+				ResourceType: "a",
+				Identifiers:  MakeIdentifiers("a1", "a2"),
+			},
+		}}...)
+		req.Nil(nn)
+	})
+
+	t.Run("not found wrong scope", func(t *testing.T) {
+		nn := NodeForRef(ref, NodeSet{{
+			// The type misses
+			ResourceType: "a",
+			Identifiers:  MakeIdentifiers("a1", "a2"),
+			Scope: Scope{
+				ResourceType: "b",
+				Identifiers:  MakeIdentifiers("a1", "a2"),
+			},
+		}}...)
+		req.Nil(nn)
+	})
+
+	t.Run("found", func(t *testing.T) {
+		nn := NodeForRef(ref, NodeSet{{
+			// The type misses
+			ResourceType: "a",
+			Identifiers:  MakeIdentifiers("a1", "a2"),
+			Scope: Scope{
+				ResourceType: "a",
+				Identifiers:  MakeIdentifiers("a1", "a2"),
+			},
+		}}...)
+		req.NotNil(nn)
+	})
+}
+
+func TestIdents(t *testing.T) {
+	req := require.New(t)
+
+	ii := MakeIdentifiers("asdf", "asd123", "321das", "12367831412")
+
+	ints, ss := ii.Idents()
+	req.Len(ss, 3)
+	req.Contains(ss, "asdf")
+	req.Contains(ss, "asd123")
+	req.Contains(ss, "321das")
+
+	req.Len(ints, 1)
+	req.Contains(ints, uint64(12367831412))
+}
+
+func TestIntersection(t *testing.T) {
+	req := require.New(t)
+
+	aa := MakeIdentifiers("a", "b")
+	bb := MakeIdentifiers("a", "b")
+	cc := MakeIdentifiers("b", "c")
+	dd := MakeIdentifiers("c", "d")
+
+	t.Run("complete overlap", func(t *testing.T) {
+		req.True(aa.HasIntersection(bb))
+	})
+
+	t.Run("partial overlap", func(t *testing.T) {
+		req.True(aa.HasIntersection(cc))
+	})
+
+	t.Run("completely off", func(t *testing.T) {
+		req.False(aa.HasIntersection(dd))
+	})
+}
+
+func TestFriendlyIdentifier(t *testing.T) {
+	req := require.New(t)
+
+	t.Run("empty", func(t *testing.T) {
+		ii := MakeIdentifiers()
+		req.Equal(ii.FriendlyIdentifier(), "")
+	})
+
+	t.Run("regular", func(t *testing.T) {
+		ii := MakeIdentifiers("h1", "h2", "123123123")
+		req.Equal(ii.FriendlyIdentifier(), "h1")
+	})
+
+	t.Run("fallback to ID", func(t *testing.T) {
+		ii := MakeIdentifiers("123123123")
+		req.Equal(ii.FriendlyIdentifier(), "123123123")
+	})
+}

server/pkg/envoyx/rbac_references_automation.gen.go

@@ -0,0 +1,24 @@
+package envoyx
+
+// This file is auto-generated.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+//
+
+import (
+	"github.com/cortezaproject/corteza/server/automation/types"
+)
+
+// AutomationWorkflowRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func AutomationWorkflowRbacReferences(workflow string) (res *Ref, pp []*Ref, err error) {
+	if workflow != "*" {
+		res = &Ref{ResourceType: types.WorkflowResourceType, Identifiers: MakeIdentifiers(workflow)}
+	}
+
+	return
+}

server/pkg/envoyx/rbac_references_compose.gen.go

@@ -0,0 +1,129 @@
+package envoyx
+
+// This file is auto-generated.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+//
+
+import (
+	"github.com/cortezaproject/corteza/server/compose/types"
+)
+
+// ComposeChartRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func ComposeChartRbacReferences(namespaceID string, chart string) (res *Ref, pp []*Ref, err error) {
+	if namespaceID != "*" {
+		pp = append(pp, &Ref{ResourceType: types.NamespaceResourceType, Identifiers: MakeIdentifiers(namespaceID)})
+	}
+	if chart != "*" {
+		res = &Ref{ResourceType: types.ChartResourceType, Identifiers: MakeIdentifiers(chart)}
+	}
+
+	return
+}
+
+// ComposeModuleRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func ComposeModuleRbacReferences(namespaceID string, module string) (res *Ref, pp []*Ref, err error) {
+	if namespaceID != "*" {
+		pp = append(pp, &Ref{ResourceType: types.NamespaceResourceType, Identifiers: MakeIdentifiers(namespaceID)})
+	}
+	if module != "*" {
+		res = &Ref{ResourceType: types.ModuleResourceType, Identifiers: MakeIdentifiers(module)}
+	}
+
+	return
+}
+
+// ComposeModuleFieldRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func ComposeModuleFieldRbacReferences(namespaceID string, moduleID string, moduleField string) (res *Ref, pp []*Ref, err error) {
+	if namespaceID != "*" {
+		pp = append(pp, &Ref{ResourceType: types.NamespaceResourceType, Identifiers: MakeIdentifiers(namespaceID)})
+	}
+	if moduleID != "*" {
+		pp = append(pp, &Ref{ResourceType: types.ModuleResourceType, Identifiers: MakeIdentifiers(moduleID)})
+	}
+	if moduleField != "*" {
+		res = &Ref{ResourceType: types.ModuleFieldResourceType, Identifiers: MakeIdentifiers(moduleField)}
+	}
+
+	return
+}
+
+// ComposeNamespaceRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func ComposeNamespaceRbacReferences(namespace string) (res *Ref, pp []*Ref, err error) {
+	if namespace != "*" {
+		res = &Ref{ResourceType: types.NamespaceResourceType, Identifiers: MakeIdentifiers(namespace)}
+	}
+
+	return
+}
+
+// ComposePageRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func ComposePageRbacReferences(namespaceID string, page string) (res *Ref, pp []*Ref, err error) {
+	if namespaceID != "*" {
+		pp = append(pp, &Ref{ResourceType: types.NamespaceResourceType, Identifiers: MakeIdentifiers(namespaceID)})
+	}
+	if page != "*" {
+		res = &Ref{ResourceType: types.PageResourceType, Identifiers: MakeIdentifiers(page)}
+	}
+
+	return
+}
+
+// ComposePageLayoutRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func ComposePageLayoutRbacReferences(namespaceID string, pageID string, pageLayout string) (res *Ref, pp []*Ref, err error) {
+	if namespaceID != "*" {
+		pp = append(pp, &Ref{ResourceType: types.NamespaceResourceType, Identifiers: MakeIdentifiers(namespaceID)})
+	}
+	if pageID != "*" {
+		pp = append(pp, &Ref{ResourceType: types.PageResourceType, Identifiers: MakeIdentifiers(pageID)})
+	}
+	if pageLayout != "*" {
+		res = &Ref{ResourceType: types.PageLayoutResourceType, Identifiers: MakeIdentifiers(pageLayout)}
+	}
+
+	return
+}
+
+// ComposeRecordRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func ComposeRecordRbacReferences(namespaceID string, moduleID string, record string) (res *Ref, pp []*Ref, err error) {
+	if namespaceID != "*" {
+		pp = append(pp, &Ref{ResourceType: types.NamespaceResourceType, Identifiers: MakeIdentifiers(namespaceID)})
+	}
+	if moduleID != "*" {
+		pp = append(pp, &Ref{ResourceType: types.ModuleResourceType, Identifiers: MakeIdentifiers(moduleID)})
+	}
+	if record != "*" {
+		res = &Ref{ResourceType: types.RecordResourceType, Identifiers: MakeIdentifiers(record)}
+	}
+
+	return
+}

server/pkg/envoyx/rbac_references_federation.gen.go

@@ -0,0 +1,56 @@
+package envoyx
+
+// This file is auto-generated.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+//
+
+import (
+	"github.com/cortezaproject/corteza/server/federation/types"
+)
+
+// FederationNodeRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func FederationNodeRbacReferences(node string) (res *Ref, pp []*Ref, err error) {
+	if node != "*" {
+		res = &Ref{ResourceType: types.NodeResourceType, Identifiers: MakeIdentifiers(node)}
+	}
+
+	return
+}
+
+// FederationExposedModuleRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func FederationExposedModuleRbacReferences(nodeID string, exposedModule string) (res *Ref, pp []*Ref, err error) {
+	if nodeID != "*" {
+		pp = append(pp, &Ref{ResourceType: types.NodeResourceType, Identifiers: MakeIdentifiers(nodeID)})
+	}
+	if exposedModule != "*" {
+		res = &Ref{ResourceType: types.ExposedModuleResourceType, Identifiers: MakeIdentifiers(exposedModule)}
+	}
+
+	return
+}
+
+// FederationSharedModuleRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func FederationSharedModuleRbacReferences(nodeID string, sharedModule string) (res *Ref, pp []*Ref, err error) {
+	if nodeID != "*" {
+		pp = append(pp, &Ref{ResourceType: types.NodeResourceType, Identifiers: MakeIdentifiers(nodeID)})
+	}
+	if sharedModule != "*" {
+		res = &Ref{ResourceType: types.SharedModuleResourceType, Identifiers: MakeIdentifiers(sharedModule)}
+	}
+
+	return
+}

server/pkg/envoyx/rbac_references_system.gen.go

@@ -0,0 +1,154 @@
+package envoyx
+
+// This file is auto-generated.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+//
+
+import (
+	"github.com/cortezaproject/corteza/server/system/types"
+)
+
+// SystemApplicationRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func SystemApplicationRbacReferences(application string) (res *Ref, pp []*Ref, err error) {
+	if application != "*" {
+		res = &Ref{ResourceType: types.ApplicationResourceType, Identifiers: MakeIdentifiers(application)}
+	}
+
+	return
+}
+
+// SystemApigwRouteRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func SystemApigwRouteRbacReferences(apigwRoute string) (res *Ref, pp []*Ref, err error) {
+	if apigwRoute != "*" {
+		res = &Ref{ResourceType: types.ApigwRouteResourceType, Identifiers: MakeIdentifiers(apigwRoute)}
+	}
+
+	return
+}
+
+// SystemAuthClientRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func SystemAuthClientRbacReferences(authClient string) (res *Ref, pp []*Ref, err error) {
+	if authClient != "*" {
+		res = &Ref{ResourceType: types.AuthClientResourceType, Identifiers: MakeIdentifiers(authClient)}
+	}
+
+	return
+}
+
+// SystemDataPrivacyRequestRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func SystemDataPrivacyRequestRbacReferences(dataPrivacyRequest string) (res *Ref, pp []*Ref, err error) {
+	if dataPrivacyRequest != "*" {
+		res = &Ref{ResourceType: types.DataPrivacyRequestResourceType, Identifiers: MakeIdentifiers(dataPrivacyRequest)}
+	}
+
+	return
+}
+
+// SystemQueueRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func SystemQueueRbacReferences(queue string) (res *Ref, pp []*Ref, err error) {
+	if queue != "*" {
+		res = &Ref{ResourceType: types.QueueResourceType, Identifiers: MakeIdentifiers(queue)}
+	}
+
+	return
+}
+
+// SystemReportRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func SystemReportRbacReferences(report string) (res *Ref, pp []*Ref, err error) {
+	if report != "*" {
+		res = &Ref{ResourceType: types.ReportResourceType, Identifiers: MakeIdentifiers(report)}
+	}
+
+	return
+}
+
+// SystemRoleRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func SystemRoleRbacReferences(role string) (res *Ref, pp []*Ref, err error) {
+	if role != "*" {
+		res = &Ref{ResourceType: types.RoleResourceType, Identifiers: MakeIdentifiers(role)}
+	}
+
+	return
+}
+
+// SystemUserGroupRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func SystemUserGroupRbacReferences(userGroup string) (res *Ref, pp []*Ref, err error) {
+	if userGroup != "*" {
+		res = &Ref{ResourceType: types.UserGroupResourceType, Identifiers: MakeIdentifiers(userGroup)}
+	}
+
+	return
+}
+
+// SystemTemplateRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func SystemTemplateRbacReferences(template string) (res *Ref, pp []*Ref, err error) {
+	if template != "*" {
+		res = &Ref{ResourceType: types.TemplateResourceType, Identifiers: MakeIdentifiers(template)}
+	}
+
+	return
+}
+
+// SystemUserRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func SystemUserRbacReferences(user string) (res *Ref, pp []*Ref, err error) {
+	if user != "*" {
+		res = &Ref{ResourceType: types.UserResourceType, Identifiers: MakeIdentifiers(user)}
+	}
+
+	return
+}
+
+// SystemDalConnectionRbacReferences generates RBAC references
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func SystemDalConnectionRbacReferences(dalConnection string) (res *Ref, pp []*Ref, err error) {
+	if dalConnection != "*" {
+		res = &Ref{ResourceType: types.DalConnectionResourceType, Identifiers: MakeIdentifiers(dalConnection)}
+	}
+
+	return
+}

server/pkg/envoyx/rbac_rules_parse.gen.go

@@ -0,0 +1,297 @@
+package envoyx
+
+// This file is auto-generated.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+//
+
+import (
+	"fmt"
+	automationTypes "github.com/cortezaproject/corteza/server/automation/types"
+	composeTypes "github.com/cortezaproject/corteza/server/compose/types"
+	federationTypes "github.com/cortezaproject/corteza/server/federation/types"
+	systemTypes "github.com/cortezaproject/corteza/server/system/types"
+	"strings"
+)
+
+// Parse generates resource setting logic for each resource
+//
+// Resources with "envoy: false" are skipped
+//
+// This function is auto-generated
+func ParseRule(res string) (string, *Ref, []*Ref, error) {
+	if res == "" {
+		return "", nil, nil, fmt.Errorf("empty resource")
+	}
+
+	sp := "/"
+
+	res = strings.TrimSpace(res)
+	res = strings.TrimRight(res, sp)
+	rr := strings.Split(res, sp)
+
+	// only service defined (corteza::system, corteza::compose, ...)
+	if len(rr) == 1 {
+		return res, nil, nil, nil
+	}
+
+	// full thing
+	resourceType, path := rr[0], rr[1:]
+
+	for p := 1; p < len(path); p++ {
+		if path[p] != "*" && path[p-1] == "*" {
+			return "", nil, nil, fmt.Errorf("invalid path wildcard combination for '%s'", res)
+		}
+	}
+
+	// @todo this is to support res. tr. resources also.
+	//       Split it into a separate function and remove this.
+	if !strings.HasPrefix(resourceType, "corteza::") {
+		resourceType = "corteza::" + resourceType
+	}
+
+	// make the resource provide the slice of parent resources we should nest under
+	switch resourceType {
+	case systemTypes.ApplicationResourceType:
+		if len(path) != 1 {
+			return "", nil, nil, fmt.Errorf("expecting 1 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := SystemApplicationRbacReferences(
+			path[0],
+		)
+		return resourceType, ref, pp, err
+
+	case systemTypes.ApigwRouteResourceType:
+		if len(path) != 1 {
+			return "", nil, nil, fmt.Errorf("expecting 1 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := SystemApigwRouteRbacReferences(
+			path[0],
+		)
+		return resourceType, ref, pp, err
+
+	case systemTypes.AuthClientResourceType:
+		if len(path) != 1 {
+			return "", nil, nil, fmt.Errorf("expecting 1 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := SystemAuthClientRbacReferences(
+			path[0],
+		)
+		return resourceType, ref, pp, err
+
+	case systemTypes.DataPrivacyRequestResourceType:
+		if len(path) != 1 {
+			return "", nil, nil, fmt.Errorf("expecting 1 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := SystemDataPrivacyRequestRbacReferences(
+			path[0],
+		)
+		return resourceType, ref, pp, err
+
+	case systemTypes.QueueResourceType:
+		if len(path) != 1 {
+			return "", nil, nil, fmt.Errorf("expecting 1 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := SystemQueueRbacReferences(
+			path[0],
+		)
+		return resourceType, ref, pp, err
+
+	case systemTypes.ReportResourceType:
+		if len(path) != 1 {
+			return "", nil, nil, fmt.Errorf("expecting 1 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := SystemReportRbacReferences(
+			path[0],
+		)
+		return resourceType, ref, pp, err
+
+	case systemTypes.RoleResourceType:
+		if len(path) != 1 {
+			return "", nil, nil, fmt.Errorf("expecting 1 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := SystemRoleRbacReferences(
+			path[0],
+		)
+		return resourceType, ref, pp, err
+
+	case systemTypes.UserGroupResourceType:
+		if len(path) != 1 {
+			return "", nil, nil, fmt.Errorf("expecting 1 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := SystemUserGroupRbacReferences(
+			path[0],
+		)
+		return resourceType, ref, pp, err
+
+	case systemTypes.TemplateResourceType:
+		if len(path) != 1 {
+			return "", nil, nil, fmt.Errorf("expecting 1 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := SystemTemplateRbacReferences(
+			path[0],
+		)
+		return resourceType, ref, pp, err
+
+	case systemTypes.UserResourceType:
+		if len(path) != 1 {
+			return "", nil, nil, fmt.Errorf("expecting 1 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := SystemUserRbacReferences(
+			path[0],
+		)
+		return resourceType, ref, pp, err
+
+	case systemTypes.DalConnectionResourceType:
+		if len(path) != 1 {
+			return "", nil, nil, fmt.Errorf("expecting 1 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := SystemDalConnectionRbacReferences(
+			path[0],
+		)
+		return resourceType, ref, pp, err
+
+	case composeTypes.ChartResourceType:
+		if len(path) != 2 {
+			return "", nil, nil, fmt.Errorf("expecting 2 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := ComposeChartRbacReferences(
+			path[0],
+			path[1],
+		)
+		return resourceType, ref, pp, err
+
+	case composeTypes.ModuleResourceType:
+		if len(path) != 2 {
+			return "", nil, nil, fmt.Errorf("expecting 2 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := ComposeModuleRbacReferences(
+			path[0],
+			path[1],
+		)
+		return resourceType, ref, pp, err
+
+	case composeTypes.ModuleFieldResourceType:
+		if len(path) != 3 {
+			return "", nil, nil, fmt.Errorf("expecting 3 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := ComposeModuleFieldRbacReferences(
+			path[0],
+			path[1],
+			path[2],
+		)
+		return resourceType, ref, pp, err
+
+	case composeTypes.NamespaceResourceType:
+		if len(path) != 1 {
+			return "", nil, nil, fmt.Errorf("expecting 1 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := ComposeNamespaceRbacReferences(
+			path[0],
+		)
+		return resourceType, ref, pp, err
+
+	case composeTypes.PageResourceType:
+		if len(path) != 2 {
+			return "", nil, nil, fmt.Errorf("expecting 2 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := ComposePageRbacReferences(
+			path[0],
+			path[1],
+		)
+		return resourceType, ref, pp, err
+
+	case composeTypes.PageLayoutResourceType:
+		if len(path) != 3 {
+			return "", nil, nil, fmt.Errorf("expecting 3 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := ComposePageLayoutRbacReferences(
+			path[0],
+			path[1],
+			path[2],
+		)
+		return resourceType, ref, pp, err
+
+	case composeTypes.RecordResourceType:
+		if len(path) != 3 {
+			return "", nil, nil, fmt.Errorf("expecting 3 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := ComposeRecordRbacReferences(
+			path[0],
+			path[1],
+			path[2],
+		)
+		return resourceType, ref, pp, err
+
+	case automationTypes.WorkflowResourceType:
+		if len(path) != 1 {
+			return "", nil, nil, fmt.Errorf("expecting 1 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := AutomationWorkflowRbacReferences(
+			path[0],
+		)
+		return resourceType, ref, pp, err
+
+	case federationTypes.NodeResourceType:
+		if len(path) != 1 {
+			return "", nil, nil, fmt.Errorf("expecting 1 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := FederationNodeRbacReferences(
+			path[0],
+		)
+		return resourceType, ref, pp, err
+
+	case federationTypes.ExposedModuleResourceType:
+		if len(path) != 2 {
+			return "", nil, nil, fmt.Errorf("expecting 2 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := FederationExposedModuleRbacReferences(
+			path[0],
+			path[1],
+		)
+		return resourceType, ref, pp, err
+
+	case federationTypes.SharedModuleResourceType:
+		if len(path) != 2 {
+			return "", nil, nil, fmt.Errorf("expecting 2 reference components in path, got %d", len(path))
+		}
+		ref, pp, err := FederationSharedModuleRbacReferences(
+			path[0],
+			path[1],
+		)
+		return resourceType, ref, pp, err
+
+	case systemTypes.ComponentResourceType:
+		if len(path) != 0 {
+			return "", nil, nil, fmt.Errorf("expecting 0 reference components in path, got %d", len(path))
+		}
+
+		// Component resource, no path
+		return resourceType, nil, nil, nil
+	case composeTypes.ComponentResourceType:
+		if len(path) != 0 {
+			return "", nil, nil, fmt.Errorf("expecting 0 reference components in path, got %d", len(path))
+		}
+
+		// Component resource, no path
+		return resourceType, nil, nil, nil
+	case automationTypes.ComponentResourceType:
+		if len(path) != 0 {
+			return "", nil, nil, fmt.Errorf("expecting 0 reference components in path, got %d", len(path))
+		}
+
+		// Component resource, no path
+		return resourceType, nil, nil, nil
+	case federationTypes.ComponentResourceType:
+		if len(path) != 0 {
+			return "", nil, nil, fmt.Errorf("expecting 0 reference components in path, got %d", len(path))
+		}
+
+		// Component resource, no path
+		return resourceType, nil, nil, nil
+	}
+
+	// return unhandled resource as-is
+	return resourceType, nil, nil, nil
+}

server/pkg/envoyx/rbacutils.go

@@ -0,0 +1,144 @@
+package envoyx
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/rbac"
+	"github.com/cortezaproject/corteza/server/system/types"
+)
+
+type (
+	rbacer interface {
+		RbacResource() string
+	}
+)
+
+func RBACRulesForNodes(rr rbac.RuleSet, nn ...*Node) (rules NodeSet, err error) {
+	rules = make(NodeSet, 0, len(rr)/2)
+	dups := make(map[uint64]map[string]map[string]bool)
+
+	for _, n := range nn {
+		if n.Placeholder {
+			continue
+		}
+
+		c, ok := n.Resource.(rbacer)
+		if !ok {
+			continue
+		}
+
+		// Split up the path of this resource
+		//
+		// @todo move over to those generated functions
+		resPath := splitResourcePath(c.RbacResource())
+
+		// Find all of the rules that fall under this resource
+		for _, r := range rr {
+			if r.RoleID == 0 {
+				// Can't exist; skip to avoid edge cases
+				continue
+			}
+
+			// Split up the path of the rule
+			//
+			// @todo move over to that generated function
+			rulePath := splitResourcePath(r.Resource)
+
+			// Don't handle rules that are not a subset of the resource
+			// @todo this should be handled by the parent probably
+			if len(rulePath) > 0 && rulePath[0] == "*" || rulePath[0] == "" {
+				continue
+			}
+
+			if !isPathSubset(rulePath, resPath, true) {
+				// Mismatch; skip
+				continue
+			}
+
+			// Check if this rule has already been seen
+			if dups[r.RoleID] != nil && dups[r.RoleID][r.Resource] != nil && dups[r.RoleID][r.Resource][r.Operation] {
+				continue
+			}
+
+			// Parse the path so we can process it further
+			ruleRt, res, path, err := ParseRule(r.Resource)
+			if err != nil {
+				return nil, err
+			}
+
+			if ruleRt != n.ResourceType {
+				// Type missmatch; skip
+				continue
+			}
+
+			// Get the refs
+			rf := make(map[string]Ref, 2)
+
+			for i, ref := range append(path, res) {
+				// Whenever you'd use a wildcard, it will produce a nil so it
+				// needs to be skipped
+				if ref == nil {
+					continue
+				}
+
+				ref.Scope = n.Scope
+
+				// @todo make the thing not a pointer
+				rf[fmt.Sprintf("Path.%d", i)] = *ref
+			}
+
+			// Ref to the rule
+			rf["RoleID"] = Ref{
+				ResourceType: types.RoleResourceType,
+				Identifiers:  MakeIdentifiers(r.RoleID),
+			}
+
+			rules = append(rules, &Node{
+				Resource: r,
+
+				ResourceType: rbac.RuleResourceType,
+				References:   rf,
+				Scope:        n.Scope,
+			})
+
+			// Update the dup checking index
+			if dups[r.RoleID] == nil {
+				dups[r.RoleID] = make(map[string]map[string]bool)
+			}
+			if dups[r.RoleID][r.Resource] == nil {
+				dups[r.RoleID][r.Resource] = make(map[string]bool)
+			}
+			dups[r.RoleID][r.Resource][r.Operation] = true
+		}
+	}
+
+	return
+}
+
+func splitResourcePath(p string) []string {
+	return strings.Split(p, "/")[1:]
+}
+
+func isPathSubset(rulePath, resPath []string, wildcards bool) bool {
+	if len(rulePath) == 0 && len(resPath) == 0 {
+		return true
+	}
+
+	// The lengths must match since missing bits are replaced with wildcards
+	if len(rulePath) != len(resPath) {
+		return false
+	}
+
+	for i := 0; i < len(resPath); i++ {
+		if wildcards && rulePath[i] == "*" {
+			// Rule matches everything from now on; if we got this far, we're good
+			return true
+		}
+		if rulePath[i] != resPath[i] {
+			return false
+		}
+	}
+
+	return true
+}

server/pkg/envoyx/store.go

@@ -0,0 +1,62 @@
+package envoyx
+
+import (
+	"context"
+	"sort"
+)
+
+func (svc *Service) decodeStore(ctx context.Context, p DecodeParams) (nn NodeSet, err error) {
+	var aux NodeSet
+	for _, d := range svc.decoders[DecodeTypeStore] {
+		aux, err = d.Decode(ctx, p)
+		if err != nil {
+			return
+		}
+		nn = append(nn, aux...)
+	}
+
+	return
+}
+
+func (svc *Service) encodeStore(ctx context.Context, dg *DepGraph, p EncodeParams) (err error) {
+	// Prepping
+	//
+	// @note this is ok for now but if we add things like importing into
+	//       multiple Cortezas at the same time, this won't be ok and each
+	//       encoder should get it's own thing
+	for rt, nn := range depNodesByResourceType(dg.allNodes()...) {
+		for _, e := range svc.preparers[EncodeTypeStore] {
+			err = e.Prepare(ctx, p, rt, OmitPlaceholderNodes(unpackDepNodes(nn...)...))
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	// Encoding
+	nodes := NodesByResourceType(dg.Roots()...)
+	for _, rt := range svc.sortResourceTypes(nodes) {
+		nn := nodes[rt]
+		for _, se := range svc.encoders[EncodeTypeStore] {
+			err = se.Encode(ctx, p, rt, OmitPlaceholderNodes(nn...), dg)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	return
+}
+
+func (svc *Service) sortResourceTypes(nn map[string]NodeSet) (out []string) {
+	for rt := range nn {
+		out = append(out, rt)
+	}
+	sort.Strings(out)
+
+	for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 {
+		out[i], out[j] = out[j], out[i]
+	}
+
+	return
+}

server/pkg/envoyx/util_rbac.gen.go

@@ -0,0 +1,664 @@
+package envoyx
+
+// This file is auto-generated.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+//
+
+import (
+	"strings"
+)
+
+// SplitResourceIdentifier takes an identifier string and splices it into path
+// identifiers as defined by the resource
+func SplitResourceIdentifier(ref string) (out map[string]Ref) {
+	out = make(map[string]Ref, 3)
+
+	ref = strings.TrimRight(ref, "/")
+	pp := strings.Split(ref, "/")
+	rt := pp[0]
+	pp = pp[1:]
+
+	// @todo this is to support res. tr. resources also.
+	//       Split it into a separate function and remove this.
+	if !strings.HasPrefix(rt, "corteza::") {
+		rt = "corteza::" + rt
+	}
+
+	gRef := func(pp []string, i int) string {
+		if pp[i] == "*" {
+			return ""
+		}
+		return pp[i]
+	}
+
+	switch rt {
+
+	case "corteza::system:apigw-filter":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:apigw-filter",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:apigw-route":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:apigw-route",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:application":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:application",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:attachment":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:attachment",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:auth-client":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:auth-client",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:auth-confirmed-client":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:auth-confirmed-client",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:auth-oa2token":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:auth-oa2token",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:auth-session":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:auth-session",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:credential":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:credential",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:dal-connection":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:dal-connection",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:dal-schema-alteration":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:dal-schema-alteration",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:dal-sensitivity-level":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:dal-sensitivity-level",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:data-privacy-request":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:data-privacy-request",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:data-privacy-request-comment":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:data-privacy-request-comment",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:notification":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:notification",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:queue":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:queue",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:queue-message":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:queue-message",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:reminder":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:reminder",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:report":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:report",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:resource-translation":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:resource-translation",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:role":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:role",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:role-member":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:role-member",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:settings":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:settings",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:template":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:template",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:user":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:user",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::system:user-group":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::system:user-group",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::compose:attachment":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::compose:attachment",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::compose:chart":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		aux := gRef(pp, 0)
+		if aux != "" {
+			scope.ResourceType = "corteza::compose:namespace"
+			scope.Identifiers = MakeIdentifiers(aux)
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::compose:namespace",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+		if gRef(pp, 1) == "" {
+			return
+		}
+
+		out["Path.1"] = Ref{
+			ResourceType: "corteza::compose:chart",
+			Identifiers:  MakeIdentifiers(gRef(pp, 1)),
+			Scope:        scope,
+		}
+
+	case "corteza::compose:module":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		aux := gRef(pp, 0)
+		if aux != "" {
+			scope.ResourceType = "corteza::compose:namespace"
+			scope.Identifiers = MakeIdentifiers(aux)
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::compose:namespace",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+		if gRef(pp, 1) == "" {
+			return
+		}
+
+		out["Path.1"] = Ref{
+			ResourceType: "corteza::compose:module",
+			Identifiers:  MakeIdentifiers(gRef(pp, 1)),
+			Scope:        scope,
+		}
+
+	case "corteza::compose:module-field":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		aux := gRef(pp, 0)
+		if aux != "" {
+			scope.ResourceType = "corteza::compose:namespace"
+			scope.Identifiers = MakeIdentifiers(aux)
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::compose:namespace",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+		if gRef(pp, 1) == "" {
+			return
+		}
+
+		out["Path.1"] = Ref{
+			ResourceType: "corteza::compose:module",
+			Identifiers:  MakeIdentifiers(gRef(pp, 1)),
+			Scope:        scope,
+		}
+
+		if gRef(pp, 2) == "" {
+			return
+		}
+
+		out["Path.2"] = Ref{
+			ResourceType: "corteza::compose:module-field",
+			Identifiers:  MakeIdentifiers(gRef(pp, 2)),
+			Scope:        scope,
+		}
+
+	case "corteza::compose:namespace":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		scope.ResourceType = "corteza::compose:namespace"
+		scope.Identifiers = MakeIdentifiers(gRef(pp, 0))
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::compose:namespace",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::compose:page":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		aux := gRef(pp, 0)
+		if aux != "" {
+			scope.ResourceType = "corteza::compose:namespace"
+			scope.Identifiers = MakeIdentifiers(aux)
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::compose:namespace",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+		if gRef(pp, 1) == "" {
+			return
+		}
+
+		out["Path.1"] = Ref{
+			ResourceType: "corteza::compose:page",
+			Identifiers:  MakeIdentifiers(gRef(pp, 1)),
+			Scope:        scope,
+		}
+
+	case "corteza::compose:page-layout":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		aux := gRef(pp, 0)
+		if aux != "" {
+			scope.ResourceType = "corteza::compose:namespace"
+			scope.Identifiers = MakeIdentifiers(aux)
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::compose:namespace",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+		if gRef(pp, 1) == "" {
+			return
+		}
+
+		out["Path.1"] = Ref{
+			ResourceType: "corteza::compose:page",
+			Identifiers:  MakeIdentifiers(gRef(pp, 1)),
+			Scope:        scope,
+		}
+
+		if gRef(pp, 2) == "" {
+			return
+		}
+
+		out["Path.2"] = Ref{
+			ResourceType: "corteza::compose:page-layout",
+			Identifiers:  MakeIdentifiers(gRef(pp, 2)),
+			Scope:        scope,
+		}
+
+	case "corteza::compose:record":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		aux := gRef(pp, 0)
+		if aux != "" {
+			scope.ResourceType = "corteza::compose:namespace"
+			scope.Identifiers = MakeIdentifiers(aux)
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::compose:namespace",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+		if gRef(pp, 1) == "" {
+			return
+		}
+
+		out["Path.1"] = Ref{
+			ResourceType: "corteza::compose:module",
+			Identifiers:  MakeIdentifiers(gRef(pp, 1)),
+			Scope:        scope,
+		}
+
+		if gRef(pp, 2) == "" {
+			return
+		}
+
+		out["Path.2"] = Ref{
+			ResourceType: "corteza::compose:record",
+			Identifiers:  MakeIdentifiers(gRef(pp, 2)),
+			Scope:        scope,
+		}
+
+	case "corteza::compose:record-revision":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::compose:record-revision",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::automation:session":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::automation:session",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::automation:trigger":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::automation:trigger",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	case "corteza::automation:workflow":
+		scope := Scope{}
+
+		if gRef(pp, 0) == "" {
+			return
+		}
+
+		out["Path.0"] = Ref{
+			ResourceType: "corteza::automation:workflow",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+			Scope:        scope,
+		}
+
+	}
+
+	return
+}

server/pkg/envoyx/utils.gen.go

@@ -0,0 +1,34 @@
+package envoyx
+
+// This file is auto-generated.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+//
+
+import ()
+
+var (
+	// needyResources is a list of resources that require a parent resource
+	//
+	// This list is primarily used when figuring out what nodes the dep. graph
+	// should return when traversing.
+	needyResources = map[string]bool{
+
+		"corteza::compose:chart":        true,
+		"corteza::compose:module":       true,
+		"corteza::compose:module-field": true,
+
+		"corteza::compose:page":        true,
+		"corteza::compose:page-layout": true,
+		"corteza::compose:record":      true,
+
+		"corteza::compose:record-datasource": true,
+	}
+
+	// superNeedyResources is the second level of filtering in case the first
+	// pass removes everything
+	superNeedyResources = map[string]bool{
+		"corteza::compose:module-field": true,
+	}
+)