snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/compose/envoy/csv_encode.go

@@ -0,0 +1,208 @@
+package envoy
+
+import (
+	"context"
+	"encoding/csv"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx/datasource"
+	"github.com/spf13/cast"
+)
+
+type (
+	CsvEncoder struct{}
+)
+
+func (e CsvEncoder) Encode(ctx context.Context, p envoyx.EncodeParams, rt string, nodes envoyx.NodeSet, tt envoyx.Traverser) (err error) {
+	w, err := e.getWriter(p)
+	if err != nil {
+		return
+	}
+
+	cw := csv.NewWriter(w)
+
+	switch rt {
+	case ComposeRecordDatasourceAuxType:
+		_, err = e.encodeRecordDatasources(ctx, cw, p, nodes, tt)
+		if err != nil {
+			return
+		}
+	}
+
+	cw.Flush()
+	return
+}
+
+func (e CsvEncoder) encodeRecordDatasources(ctx context.Context, writer *csv.Writer, p envoyx.EncodeParams, nodes envoyx.NodeSet, tt envoyx.Traverser) (out any, err error) {
+	for _, n := range nodes {
+		_, err = e.encodeRecordDatasource(ctx, writer, p, n, tt)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (e CsvEncoder) encodeRecordDatasource(ctx context.Context, writer *csv.Writer, p envoyx.EncodeParams, node *envoyx.Node, tt envoyx.Traverser) (_ any, err error) {
+	rds := node.Datasource.(*RecordDatasource)
+	resolved := map[string]string{}
+
+	var loc *time.Location
+	loc, err = resolveTimezone(p)
+	if err != nil {
+		return
+	}
+
+	header := make([]string, 0, 4)
+
+	for _, m := range p.FieldMapping {
+		if strings.ToLower(m.Field) != "id" {
+			header = append(header, m.Field)
+		}
+	}
+	header = append([]string{"ID"}, header...)
+
+	mvDelimiter := ";"
+	wrapBrackets := false
+
+	if p.Params["multiValueDelimiter"] == nil {
+		wrapBrackets = false
+		mvDelimiter = ";"
+	}
+
+	switch cast.ToString(p.Params["multiValueDelimiter"]) {
+	case ",":
+		mvDelimiter = ","
+		wrapBrackets = false
+	case ";":
+		mvDelimiter = ";"
+		wrapBrackets = false
+	case "|":
+		mvDelimiter = "|"
+		wrapBrackets = false
+	case "[,]":
+		mvDelimiter = ","
+		wrapBrackets = true
+	case "[;]":
+		mvDelimiter = ";"
+		wrapBrackets = true
+	case "[|]":
+		mvDelimiter = "|"
+		wrapBrackets = true
+	}
+
+	getValue := func(cache datasource.RawRecord, h string) string {
+		v := cache[h]
+		if len(v.Values) == 0 {
+			return ""
+		}
+
+		// Encode as mf if the OG field is multi value OR we're on the resolved record column
+		src, ok := resolved[h]
+		isDt := rds.datetimeFields[h] || (ok && rds.datetimeFields[src])
+		if !rds.multivalues[h] && !(ok && rds.multivalues[src]) {
+			vv := v.Values[0]
+			if isDt {
+				vv = formatInTimezone(vv, loc)
+			}
+
+			return vv
+		}
+
+		auxv := make([]string, 0, len(v.Values))
+		for _, vv := range v.Values {
+			if isDt {
+				vv = formatInTimezone(vv, loc)
+			}
+
+			if strings.Contains(vv, mvDelimiter) {
+				auxv = append(auxv, fmt.Sprintf("\"%s\"", vv))
+			} else {
+				auxv = append(auxv, vv)
+			}
+		}
+
+		out := strings.Join(auxv, mvDelimiter)
+		if wrapBrackets {
+			out = fmt.Sprintf("[%s]", out)
+		}
+
+		return out
+	}
+
+	// Pre-build resolved field set from datasource so header is correct even
+	// when the first row has no value for a resolved field.
+	resolvedFieldSet := make(map[string]bool)
+	for _, f := range rds.ResolvedFields() {
+		resolvedFieldSet[f] = true
+	}
+
+	row := make([]string, 0, 4)
+	var (
+		more     bool
+		hWritten bool
+	)
+	for {
+		cache := make(datasource.RawRecord)
+		_, more, err = rds.Next(ctx, cache)
+		if err != nil || !more {
+			return
+		}
+
+		if len(header) == 0 {
+			for k := range cache {
+				header = append(header, k)
+			}
+		}
+
+		if !hWritten {
+			hWritten = true
+
+			// Splice in "X ID" columns after each resolved field
+			newHeader := make([]string, 0, len(header)+len(resolvedFieldSet))
+			for _, h := range header {
+				newHeader = append(newHeader, h)
+				if resolvedFieldSet[h] {
+					idCol := fmt.Sprintf("%s ID", h)
+					newHeader = append(newHeader, idCol)
+					resolved[idCol] = h
+				}
+			}
+			header = newHeader
+
+			err = writer.Write(header)
+			if err != nil {
+				return
+			}
+		}
+
+		for _, h := range header {
+			row = append(row, getValue(cache, h))
+		}
+
+		err = writer.Write(row)
+		if err != nil {
+			return
+		}
+
+		row = nil
+	}
+}
+
+func (e CsvEncoder) getWriter(p envoyx.EncodeParams) (out io.Writer, err error) {
+	aux, ok := p.Params[paramsKeyWriter]
+	if ok {
+		out, ok = aux.(io.Writer)
+		if ok {
+			return
+		}
+	}
+
+	err = fmt.Errorf("csv encoder expects a writer conforming to io.Writer interface")
+	return
+}

server/compose/envoy/jsonl_encode.go

@@ -0,0 +1,153 @@
+package envoy
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx/datasource"
+	"github.com/cortezaproject/corteza/server/pkg/j7s"
+)
+
+type (
+	JsonlEncoder struct{}
+)
+
+func (e JsonlEncoder) Encode(ctx context.Context, p envoyx.EncodeParams, rt string, nodes envoyx.NodeSet, tt envoyx.Traverser) (err error) {
+	w, err := e.getWriter(p)
+	if err != nil {
+		return
+	}
+
+	jw := json.NewEncoder(w)
+
+	switch rt {
+	case ComposeRecordDatasourceAuxType:
+		_, err = e.encodeRecordDatasources(ctx, jw, p, nodes, tt)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (e JsonlEncoder) encodeRecordDatasources(ctx context.Context, writer *json.Encoder, p envoyx.EncodeParams, nodes envoyx.NodeSet, tt envoyx.Traverser) (out any, err error) {
+	for _, n := range nodes {
+		_, err = e.encodeRecordDatasource(ctx, writer, p, n, tt)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (e JsonlEncoder) encodeRecordDatasource(ctx context.Context, writer *json.Encoder, p envoyx.EncodeParams, node *envoyx.Node, tt envoyx.Traverser) (_ any, err error) {
+	rds := node.Datasource.(*RecordDatasource)
+
+	var loc *time.Location
+	loc, err = resolveTimezone(p)
+	if err != nil {
+		return
+	}
+
+	header := make([]string, 0, 4)
+
+	hasID := false
+	for _, m := range p.FieldMapping {
+		header = append(header, m.Field)
+
+		hasID = hasID || strings.ToLower(m.Field) == "id"
+	}
+
+	if !hasID {
+		header = append([]string{"ID"}, header...)
+	}
+
+	var more bool
+	for {
+		out := make(datasource.RawRecord)
+		_, more, err = rds.Next(ctx, out)
+		if err != nil || !more {
+			break
+		}
+
+		if len(header) == 0 {
+			for k := range out {
+				header = append(header, k)
+			}
+		}
+
+		row, _ := j7s.MakeMap()
+		for _, h := range header {
+			proc := func(h string) {
+				v := out[h]
+				if len(v.Values) == 0 {
+					return
+				}
+
+				srcKey := strings.TrimSuffix(h, " value")
+				isDt := rds.datetimeFields[h] || rds.datetimeFields[srcKey]
+
+				if !rds.multivalues[h] {
+					vv := v.Values[0]
+					if isDt {
+						vv = formatInTimezone(vv, loc)
+					}
+
+					row, err = j7s.AddMap(row, h, vv)
+					if err != nil {
+						return
+					}
+				} else {
+					auxv, _ := j7s.MakeSeq()
+					for _, vv := range v.Values {
+						if isDt {
+							vv = formatInTimezone(vv, loc)
+						}
+
+						auxv, err = j7s.AddSeq(auxv, vv)
+						if err != nil {
+							return
+						}
+					}
+
+					row, err = j7s.AddMap(row, h, auxv)
+					if err != nil {
+						return
+					}
+				}
+			}
+
+			proc(h)
+			if _, ok := out[fmt.Sprintf("%s value", h)]; ok {
+				proc(fmt.Sprintf("%s value", h))
+			}
+		}
+
+		err = writer.Encode(row)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (e JsonlEncoder) getWriter(p envoyx.EncodeParams) (out io.Writer, err error) {
+	aux, ok := p.Params[paramsKeyWriter]
+	if ok {
+		out, ok = aux.(io.Writer)
+		if ok {
+			return
+		}
+	}
+
+	err = fmt.Errorf("jsonl encoder expects a writer conforming to io.Writer interface")
+	return
+}

server/compose/envoy/record_datasource.go

@@ -0,0 +1,653 @@
+package envoy
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/compose/dalutils"
+	"github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx/datasource"
+	"github.com/cortezaproject/corteza/server/pkg/errors"
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/cortezaproject/corteza/server/pkg/logger"
+	"github.com/cortezaproject/corteza/server/store"
+	systemTypes "github.com/cortezaproject/corteza/server/system/types"
+	"github.com/modern-go/reflect2"
+	"github.com/spf13/cast"
+	"go.uber.org/zap"
+)
+
+type (
+	// RecordDatasource provides a mechanism for you to access large
+	// record datasets optimally
+	RecordDatasource struct {
+		Mapping  envoyx.DatasourceMapping
+		Provider envoyx.Provider
+
+		multivalues    map[string]bool
+		datetimeFields map[string]bool
+
+		CheckExisting func(ctx context.Context, ref ...[]string) ([]uint64, error)
+
+		currentIndex int
+
+		// Index to map from ref to ID
+		// @todo we might need to flush these to the disc in case a huge dataset is passed in
+		refToID map[string]uint64
+		// @todo might be worth putting both into one map; not sure how much space we'd save up
+		existingIDs map[uint64]bool
+	}
+
+	// iteratorProvider is a wrapper around the dal.Iterator to conform to the
+	// envoy.Provider interface
+	iteratorProvider struct {
+		iter dal.Iterator
+
+		// Items related to ref resolution
+		// @todo can be removed when reworked
+		resolveRefs  bool
+		includeRefID bool
+		relMods      map[string]refModWrap
+		dal          dal.FullService
+
+		// Access control for field-level permissions
+		ac             recordValueAccessController
+		mod            *types.Module
+		readableFields map[string]bool
+
+		rows      []datasource.RawRecord
+		buffIndex int
+		done      bool
+
+		// User ref resolution
+		store      store.Storer
+		userFields map[string]bool
+	}
+
+	refModWrap struct {
+		modLvl1   *types.Module
+		labelLvl1 string
+
+		modLvl2   *types.Module
+		labelLvl2 string
+	}
+
+	// recordValueAccessController interface for checking field-level read permissions
+	recordValueAccessController interface {
+		CanReadRecordValueOnModuleField(context.Context, *types.ModuleField) bool
+	}
+)
+
+const (
+	bufferPullChunkSize = int(100)
+)
+
+func mkIteratorProvider(ctx context.Context, ac recordValueAccessController, s store.Storer, dl dal.FullService, iter dal.Iterator, mod *types.Module, resolveRefs bool, includeRefID bool) (out *iteratorProvider, err error) {
+	out = &iteratorProvider{
+		iter:         iter,
+		dal:          dl,
+		resolveRefs:  resolveRefs,
+		includeRefID: includeRefID,
+
+		ac:  ac,
+		mod: mod,
+	}
+
+	// Pre-compute readable fields based on access control permissions
+	// This avoids checking permissions on every record iteration
+	if !reflect2.IsNil(ac) {
+		out.readableFields = make(map[string]bool)
+		for _, f := range mod.Fields {
+			// Ensure NamespaceID is set on the field (it may not be loaded from store)
+			// RBAC checks require the full resource path including namespace
+			if f.NamespaceID == 0 {
+				f.NamespaceID = mod.NamespaceID
+			}
+			out.readableFields[f.Name] = ac.CanReadRecordValueOnModuleField(ctx, f)
+		}
+	}
+
+	refMods := make(map[string]refModWrap)
+	userFields := make(map[string]bool)
+
+	for _, name := range []string{"createdBy", "updatedBy", "ownedBy", "deletedBy"} {
+		userFields[name] = true
+	}
+
+	for _, f := range mod.Fields {
+		switch f.Kind {
+		case "Record":
+			refMods[f.Name], err = mkRecordRefWrap(ctx, s, f)
+			if err != nil {
+				return
+			}
+		case "User":
+			userFields[f.Name] = true
+		}
+	}
+
+	out.relMods = refMods
+	out.store = s
+	out.userFields = userFields
+
+	return
+}
+
+// ResolvedFields returns field names that will have a companion "X ID" column when includeRefID is enabled.
+func (ip *iteratorProvider) ResolvedFields() []string {
+	if !ip.resolveRefs || !ip.includeRefID {
+		return nil
+	}
+
+	out := make([]string, 0, len(ip.userFields)+len(ip.relMods))
+	for f := range ip.userFields {
+		out = append(out, f)
+	}
+
+	for f := range ip.relMods {
+		out = append(out, f)
+	}
+
+	return out
+}
+
+// ResolvedFields returns field names that will have a companion "X value" column.
+func (rd *RecordDatasource) ResolvedFields() []string {
+	if ip, ok := rd.Provider.(*iteratorProvider); ok {
+		return ip.ResolvedFields()
+	}
+	return nil
+}
+
+func (rd *RecordDatasource) SetProvider(s envoyx.Provider) bool {
+	if rd.Mapping.SourceIdent != s.Ident() {
+		return false
+	}
+
+	rd.Provider = s
+	return true
+}
+
+func (rd *RecordDatasource) Next(ctx context.Context, out datasource.RawRecord) (ident []string, more bool, err error) {
+	rowCache := make(datasource.RawRecord)
+
+	more, err = rd.Provider.Next(ctx, rowCache)
+	if err != nil || !more {
+		return
+	}
+
+	rd.applyMapping(rowCache, out)
+
+	if len(rd.Mapping.KeyField) == 0 {
+		ident = append(ident, strconv.FormatInt(int64(rd.currentIndex), 10))
+	} else {
+		for _, k := range rd.Mapping.KeyField {
+			ident = append(ident, strings.Join(rowCache[k].Values, ","))
+		}
+	}
+
+	rd.currentIndex++
+
+	return
+}
+
+func (rd *iteratorProvider) SetConfigs(map[string]any) error {
+	return nil
+}
+
+func (rd *RecordDatasource) Reset(ctx context.Context) (err error) {
+	rd.currentIndex = 0
+	return rd.Provider.Reset(ctx)
+}
+
+func (rd *RecordDatasource) applyMapping(in, out datasource.RawRecord) {
+	if len(rd.Mapping.Mapping.Map) == 0 {
+		if !rd.Mapping.Defaultable {
+			return
+		}
+
+		for k, v := range in {
+			out[k] = v
+		}
+		return
+	}
+
+	if rd.Mapping.Defaultable {
+		rd.applyMappingWithDefaults(in, out)
+	} else {
+		rd.applyMappingWoDefaults(in, out)
+	}
+}
+
+func (rd *RecordDatasource) applyMappingWithDefaults(in, out datasource.RawRecord) {
+	maps := make(map[string]envoyx.MapEntry)
+	for k, v := range rd.Mapping.Mapping.Map {
+		maps[k] = v
+	}
+
+	for k, v := range in {
+		if m, ok := maps[k]; ok {
+			if m.Skip {
+				continue
+			}
+			out[m.Field] = v
+		} else {
+			out[k] = v
+		}
+	}
+}
+
+func (rd *RecordDatasource) applyMappingWoDefaults(in, out datasource.RawRecord) {
+	for _, m := range rd.Mapping.Mapping.Map {
+		if m.Skip {
+			continue
+		}
+
+		out[m.Field] = in[m.Column]
+	}
+}
+
+func (rd *RecordDatasource) ResolveRef(ref ...any) (out uint64, err error) {
+	idents, err := cast.ToStringSliceE(ref)
+	if err != nil {
+		return
+	}
+
+	for i, ident := range idents {
+		idents[i] = strings.Replace(ident, "-", "_", -1)
+	}
+
+	out = rd.refToID[strings.Join(idents, "-")]
+	return
+}
+
+func (rd *RecordDatasource) ResolveRefS(ref ...string) (out uint64, err error) {
+	aux := make([]any, len(ref))
+	for i, r := range ref {
+		aux[i] = r
+	}
+
+	return rd.ResolveRef(aux...)
+}
+
+// @todo this should be replaced by some smarter structure
+func (rd *RecordDatasource) AddRef(id uint64, idents ...string) {
+	for i, ident := range idents {
+		idents[i] = strings.Replace(ident, "-", "_", -1)
+	}
+
+	rd.refToID[strings.Join(idents, "-")] = id
+}
+
+func (ip *iteratorProvider) Next(ctx context.Context, out datasource.RawRecord) (more bool, err error) {
+	if ip.resolveRefs {
+		return ip.nextResolved(ctx, out)
+	}
+
+	return ip.next(ctx, out)
+}
+
+func (ip *iteratorProvider) next(ctx context.Context, out datasource.RawRecord) (more bool, err error) {
+	rowCache := make(datasource.RawRecord)
+
+	if !ip.iter.Next(ctx) {
+		return false, ip.iter.Err()
+	}
+
+	err = ip.iter.Scan(rowCache)
+	if err != nil {
+		return
+	}
+
+	// Apply field-level access control filtering
+	ip.filterUnreadableFields(rowCache)
+
+	for k, v := range rowCache {
+		out[k] = v
+	}
+
+	return true, nil
+}
+
+func (ip *iteratorProvider) nextResolved(ctx context.Context, out datasource.RawRecord) (more bool, err error) {
+	if ip.done && ip.buffIndex >= len(ip.rows) {
+		return false, nil
+	}
+
+	if ip.buffIndex == len(ip.rows) {
+
+		// pull chunk
+		ip.rows = make([]datasource.RawRecord, 0)
+		ip.buffIndex = 0
+
+		for i := 0; i < bufferPullChunkSize; i++ {
+			rowCache := make(datasource.RawRecord)
+			if !ip.iter.Next(ctx) {
+				ip.done = true
+
+				err := ip.iter.Err()
+				if err != nil {
+					return false, err
+				}
+
+				break
+			}
+
+			err = ip.iter.Scan(rowCache)
+			if err != nil {
+				return
+			}
+
+			ip.rows = append(ip.rows, rowCache)
+		}
+
+		if len(ip.rows) == 0 {
+			return false, nil
+		}
+
+		// resolve stuff
+		err = ip.resolveReferences(ctx, ip.dal)
+		if err != nil {
+			return
+		}
+
+		err = ip.resolveUsers(ctx, ip.store)
+		if err != nil {
+			return
+		}
+	}
+
+	rowCache := ip.rows[ip.buffIndex]
+	ip.buffIndex++
+
+	// Apply field-level access control filtering
+	ip.filterUnreadableFields(rowCache)
+
+	for k, v := range rowCache {
+		out[k] = v
+	}
+
+	return true, nil
+}
+
+func (ip *iteratorProvider) resolveReferences(ctx context.Context, ds dal.FullService) (err error) {
+	// @todo I'll need to chunk these up for to reduce DB query count...
+
+	for i, cacheRecord := range ip.rows {
+
+		for refField, refWrap := range ip.relMods {
+			value := cacheRecord[refField]
+			if len(value.Values) == 0 {
+				continue
+			}
+
+			aux := []string{}
+			for _, v := range value.Values {
+				aux = append(aux, fmt.Sprintf("recordID=%s", v))
+			}
+
+			var relRecords types.RecordSet
+
+			resLab := refWrap.labelLvl1
+			qq := fmt.Sprintf("(%s)", strings.Join(aux, " OR "))
+			relRecords, _, err = dalutils.ComposeRecordsList(ctx, ds, refWrap.modLvl1, types.RecordFilter{
+				Query: qq,
+				Paging: filter.Paging{
+					Limit: uint(len(aux)),
+				},
+			})
+
+			if err != nil {
+				return err
+			}
+
+			// lvl 2 nesting; current max lvl
+			if refWrap.modLvl2 != nil {
+				resLab = refWrap.labelLvl2
+
+				// Iterate related records and collect lvl 2 identifiers
+				aux := []string{}
+				for _, rec := range relRecords {
+					for _, v := range rec.Values.FilterByName(refWrap.labelLvl1) {
+						aux = append(aux, fmt.Sprintf("recordID=%s", v.Value))
+					}
+				}
+
+				qq := fmt.Sprintf("(%s)", strings.Join(aux, " OR "))
+				relRecords, _, err = dalutils.ComposeRecordsList(ctx, ds, refWrap.modLvl2, types.RecordFilter{
+					Query: qq,
+					Paging: filter.Paging{
+						Limit: uint(len(aux)),
+					},
+				})
+
+				if err != nil {
+					return err
+				}
+			}
+
+			// Build a map from recordID to resolved label for ordered assignment
+			labelByID := make(map[string]string, len(relRecords))
+			for _, rec := range relRecords {
+				if rec.Values == nil {
+					continue
+				}
+				v := rec.Values.Get(resLab, 0)
+				if v == nil {
+					continue
+				}
+				labelByID[strconv.FormatUint(rec.ID, 10)] = v.Value
+			}
+
+			for j, origID := range value.Values {
+				label, ok := labelByID[origID]
+				if !ok {
+					continue
+				}
+				if ip.includeRefID {
+					cacheRecord.SetValue(fmt.Sprintf("%s ID", refField), uint(j), origID)
+				}
+				cacheRecord.SetValue(refField, uint(j), label)
+			}
+
+			ip.rows[i] = cacheRecord
+		}
+	}
+
+	return
+}
+
+// @todo consider omitting these from the interface since they're not always needed
+func (ip *iteratorProvider) Reset(ctx context.Context) (err error) {
+	return
+}
+
+// @todo consider omitting these from the interface since they're not always needed
+func (ip *iteratorProvider) Ident() (out string) {
+	return
+}
+
+// @todo consider omitting these from the interface since they're not always needed
+func (ip *iteratorProvider) SetIdent(string) {
+}
+
+func mkRecordRefWrap(ctx context.Context, s store.Storer, f *types.ModuleField) (wrap refModWrap, err error) {
+	relModID := f.Options.UInt64("moduleID")
+	if relModID == 0 {
+		return
+	}
+
+	var relMod *types.Module
+	relMod, err = s.LookupComposeModuleByID(ctx, relModID)
+	if err != nil {
+		// Referenced module may have been deleted or made inaccessible since the
+		// field was configured. Skip ref resolution for this field instead of
+		// failing the entire export — the column is still emitted, just unresolved.
+		if errors.IsNotFound(err) {
+			logger.Default().Warn(
+				"skipping record ref resolution: referenced module not found",
+				zap.String("field", f.Name),
+				zap.Uint64("moduleID", relModID),
+			)
+			err = nil
+		}
+		return
+	}
+
+	relMod.Fields, _, err = s.SearchComposeModuleFields(ctx, types.ModuleFieldFilter{
+		ModuleID: []uint64{relMod.ID},
+	})
+	if err != nil {
+		return
+	}
+
+	wrap = refModWrap{
+		modLvl1:   relMod,
+		labelLvl1: f.Options.String("labelField"),
+	}
+
+	if f.Options.String("recordLabelField") != "" {
+		// Two-level label resolution: the labelField on the related module must
+		// itself be a Record field pointing to another module. If that chain is
+		// broken (labelField renamed/removed, target module missing), skip the
+		// nested resolution instead of aborting the export.
+		labelFieldName := f.Options.String("labelField")
+		nestedRef := wrap.modLvl1.Fields.FindByName(labelFieldName)
+		if nestedRef == nil {
+			logger.Default().Warn(
+				"skipping nested record ref resolution: labelField not found on related module",
+				zap.String("field", f.Name),
+				zap.String("labelField", labelFieldName),
+				zap.Uint64("relatedModuleID", relMod.ID),
+			)
+			return
+		}
+		nestedModID := nestedRef.Options.UInt64("moduleID")
+		if nestedModID == 0 {
+			logger.Default().Warn(
+				"skipping nested record ref resolution: labelField has no moduleID option",
+				zap.String("field", f.Name),
+				zap.String("labelField", labelFieldName),
+			)
+			return
+		}
+
+		wrap.modLvl2, err = s.LookupComposeModuleByID(ctx, nestedModID)
+		if err != nil {
+			if errors.IsNotFound(err) {
+				logger.Default().Warn(
+					"skipping nested record ref resolution: nested module not found",
+					zap.String("field", f.Name),
+					zap.Uint64("nestedModuleID", nestedModID),
+				)
+				err = nil
+			}
+			return
+		}
+		wrap.labelLvl2 = f.Options.String("recordLabelField")
+	}
+
+	return
+}
+
+func (ip *iteratorProvider) resolveUsers(ctx context.Context, s store.Storer) (err error) {
+	if len(ip.userFields) == 0 || s == nil {
+		return
+	}
+
+	// Collect unique user IDs across the current chunk
+	seen := make(map[string]bool)
+	for _, row := range ip.rows {
+		for fieldName := range ip.userFields {
+			for _, val := range row[fieldName].Values {
+				if val != "" && val != "0" {
+					seen[val] = true
+				}
+			}
+		}
+	}
+	if len(seen) == 0 {
+		return
+	}
+
+	ids := make([]string, 0, len(seen))
+	for id := range seen {
+		ids = append(ids, id)
+	}
+
+	uu, _, err := store.SearchUsers(ctx, s, systemTypes.UserFilter{
+		UserID:    ids,
+		Paging:    filter.Paging{Limit: 0},
+		Deleted:   filter.StateInclusive,
+		Suspended: filter.StateInclusive,
+	})
+	if err != nil {
+		return
+	}
+
+	labels := make(map[string]string, len(uu))
+	for _, u := range uu {
+		labels[strconv.FormatUint(u.ID, 10)] = userLabel(u)
+	}
+
+	for i, row := range ip.rows {
+		for fieldName := range ip.userFields {
+			v := row[fieldName]
+			if len(v.Values) == 0 {
+				continue
+			}
+			for j, val := range v.Values {
+				if val == "0" || val == "" {
+					row.SetValue(fieldName, uint(j), "")
+					continue
+				}
+				if label, ok := labels[val]; ok {
+					if ip.includeRefID {
+						row.SetValue(fmt.Sprintf("%s ID", fieldName), uint(j), val)
+					}
+					row.SetValue(fieldName, uint(j), label)
+				}
+			}
+		}
+		ip.rows[i] = row
+	}
+	return
+}
+
+func userLabel(u *systemTypes.User) string {
+	if u.Name != "" {
+		return u.Name
+	}
+	if u.Email != "" {
+		return u.Email
+	}
+	if u.Handle != "" {
+		return u.Handle
+	}
+	return strconv.FormatUint(u.ID, 10)
+}
+
+// filterUnreadableFields removes values the given user does not have access to
+func (ip *iteratorProvider) filterUnreadableFields(rowCache datasource.RawRecord) {
+	if reflect2.IsNil(ip.ac) {
+		return
+	}
+
+	for fieldName := range rowCache {
+		// Skip system fields (ID, createdAt, etc.) - these are always readable
+		// System fields are lowercase and don't have associated module field definitions
+		if _, hasField := ip.readableFields[fieldName]; !hasField {
+			// Field not in module definition, likely a system field - keep it
+			continue
+		}
+
+		// Check if user can read this field
+		if !ip.readableFields[fieldName] {
+			delete(rowCache, fieldName)
+		}
+	}
+}

server/compose/envoy/record_getter.go

@@ -0,0 +1,117 @@
+package envoy
+
+import (
+	"context"
+
+	"github.com/cortezaproject/corteza/server/compose/dalutils"
+	"github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/spf13/cast"
+)
+
+type (
+	// recordGetter is a utility struct to resolve record references from
+	// different parts of the system such as the dep graph and the database
+	recordGetter struct {
+		// Bits to get data from the dep graph
+		relDatasource *RecordDatasource
+
+		// Bits to get data from the database
+		dalSvc     dal.FullService
+		relModule  types.Module
+		baseFilter types.RecordFilter
+
+		// resolved refs, including misses; imports resolve the same handful
+		// of refs over and over so this saves a query per value
+		cache map[string]uint64
+	}
+)
+
+func makeRecordGetter(dalSvc dal.FullService, tt envoyx.Traverser, n *envoyx.Node, modRef, dsRef envoyx.Ref) (g *recordGetter) {
+	g = &recordGetter{
+		dalSvc: dalSvc,
+		cache:  make(map[string]uint64),
+	}
+
+	// Resolve from dep graph
+	auxDs := tt.ParentForRef(n, dsRef)
+	if auxDs != nil {
+		g.relDatasource = auxDs.Datasource.(*RecordDatasource)
+	}
+
+	// Resolve from the database
+	mod := g.getRefMod(tt, n, modRef)
+	if mod != nil {
+		g.baseFilter = types.RecordFilter{
+			ModuleID:    mod.ID,
+			NamespaceID: mod.NamespaceID,
+		}
+	}
+
+	return
+}
+
+// resolve resolves the provided reference into a record ID; 0 if can't be resolved
+func (g *recordGetter) resolve(ctx context.Context, ref any) (out uint64, err error) {
+	key := cast.ToString(ref)
+	if out, ok := g.cache[key]; ok {
+		return out, nil
+	}
+	defer func() {
+		if err == nil {
+			g.cache[key] = out
+		}
+	}()
+
+	// Try to get from datasource
+	if g.relDatasource != nil {
+		out, err = g.getDS(ref)
+		if err != nil {
+			return
+		}
+	}
+	if out > 0 {
+		return
+	}
+
+	// Fallback to the store
+	out, err = g.getDB(ctx, ref)
+	if err != nil {
+		return
+	}
+	if out > 0 {
+		return
+	}
+	return
+}
+
+func (g *recordGetter) getDS(ref any) (out uint64, err error) {
+	return g.relDatasource.ResolveRef(ref)
+}
+
+func (g *recordGetter) getDB(ctx context.Context, ref any) (out uint64, err error) {
+	// @note the old version only resolved IDs so that's what we're doing here also
+	// @todo consider expanding this
+	id, err := cast.ToUint64E(ref)
+	if err != nil {
+		return 0, nil
+	}
+
+	aux, err := dalutils.ComposeRecordsFind(ctx, g.dalSvc, &g.relModule, id)
+	if err != nil {
+		return
+	}
+
+	out = aux.ID
+	return
+}
+
+func (g *recordGetter) getRefMod(tt envoyx.Traverser, n *envoyx.Node, ref envoyx.Ref) (refMod *types.Module) {
+	aux := tt.ParentForRef(n, ref)
+	if aux == nil {
+		return nil
+	}
+
+	return aux.Resource.(*types.Module)
+}

server/compose/envoy/store_decode.gen.go

@@ -0,0 +1,907 @@
+package envoy
+
+// This file is auto-generated.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+//
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/pkg/id"
+	"github.com/cortezaproject/corteza/server/store"
+	"github.com/pkg/errors"
+)
+
+type (
+	// StoreDecoder is responsible for generating Envoy nodes from already stored
+	// resources which can then be managed by Envoy and imported via an encoder.
+	StoreDecoder struct{}
+
+	filterWrap struct {
+		rt string
+		f  envoyx.ResourceFilter
+	}
+)
+
+const (
+	paramsKeyStorer = "storer"
+	paramsKeyDAL    = "dal"
+)
+
+var (
+	// @todo temporary fix to make unused pkg/id not throw errors
+	_ = id.Next
+)
+
+// Decode returns a set of envoy nodes based on the provided params
+//
+// StoreDecoder expects the DecodeParam of `storer` and `dal` which conform
+// to the store.Storer and dal.FullService interfaces.
+func (d StoreDecoder) Decode(ctx context.Context, p envoyx.DecodeParams) (out envoyx.NodeSet, err error) {
+	// @todo we can optionally not require them based on what we're doing
+	s, err := d.getStorer(p)
+	if err != nil {
+		return
+	}
+	dl, err := d.getDal(p)
+	if err != nil {
+		return
+	}
+
+	return d.decode(ctx, s, dl, p)
+}
+
+func (d StoreDecoder) decode(ctx context.Context, s store.Storer, dl dal.FullService, p envoyx.DecodeParams) (out envoyx.NodeSet, err error) {
+	// Preprocessing and basic filtering (to omit what this decoder can't handle)
+	wrappedFilters := d.prepFilters(p.Filter)
+
+	// Get all scoped nodes
+	scopedNodes, err := d.getScopeNodes(ctx, s, dl, wrappedFilters)
+	if err != nil {
+		return
+	}
+
+	// Get all reference nodes
+	refNodes, refRefs, err := d.getReferenceNodes(ctx, s, dl, wrappedFilters)
+	if err != nil {
+		return
+	}
+
+	// Process filters to get the envoy nodes
+	err = func() (err error) {
+		var aux envoyx.NodeSet
+		for i, wf := range wrappedFilters {
+			switch wf.rt {
+			case types.ChartResourceType:
+				aux, err = d.decodeChart(ctx, s, dl, d.makeChartFilter(scopedNodes[i], refNodes[i], wf.f))
+				if err != nil {
+					return
+				}
+				for _, a := range aux {
+					a.Identifiers = a.Identifiers.Merge(wf.f.Identifiers)
+					a.References = envoyx.MergeRefs(a.References, refRefs[i])
+				}
+				out = append(out, aux...)
+
+			case types.ModuleResourceType:
+				aux, err = d.decodeModule(ctx, s, dl, d.makeModuleFilter(scopedNodes[i], refNodes[i], wf.f))
+				if err != nil {
+					return
+				}
+				for _, a := range aux {
+					a.Identifiers = a.Identifiers.Merge(wf.f.Identifiers)
+					a.References = envoyx.MergeRefs(a.References, refRefs[i])
+				}
+				out = append(out, aux...)
+
+			case types.ModuleFieldResourceType:
+				aux, err = d.decodeModuleField(ctx, s, dl, d.makeModuleFieldFilter(scopedNodes[i], refNodes[i], wf.f))
+				if err != nil {
+					return
+				}
+				for _, a := range aux {
+					a.Identifiers = a.Identifiers.Merge(wf.f.Identifiers)
+					a.References = envoyx.MergeRefs(a.References, refRefs[i])
+				}
+				out = append(out, aux...)
+
+			case types.NamespaceResourceType:
+				aux, err = d.decodeNamespace(ctx, s, dl, d.makeNamespaceFilter(scopedNodes[i], refNodes[i], wf.f))
+				if err != nil {
+					return
+				}
+				for _, a := range aux {
+					a.Identifiers = a.Identifiers.Merge(wf.f.Identifiers)
+					a.References = envoyx.MergeRefs(a.References, refRefs[i])
+				}
+				out = append(out, aux...)
+
+			case types.PageResourceType:
+				aux, err = d.decodePage(ctx, s, dl, d.makePageFilter(scopedNodes[i], refNodes[i], wf.f))
+				if err != nil {
+					return
+				}
+				for _, a := range aux {
+					a.Identifiers = a.Identifiers.Merge(wf.f.Identifiers)
+					a.References = envoyx.MergeRefs(a.References, refRefs[i])
+				}
+				out = append(out, aux...)
+
+			case types.PageLayoutResourceType:
+				aux, err = d.decodePageLayout(ctx, s, dl, d.makePageLayoutFilter(scopedNodes[i], refNodes[i], wf.f))
+				if err != nil {
+					return
+				}
+				for _, a := range aux {
+					a.Identifiers = a.Identifiers.Merge(wf.f.Identifiers)
+					a.References = envoyx.MergeRefs(a.References, refRefs[i])
+				}
+				out = append(out, aux...)
+
+			default:
+				aux, err = d.extendDecoder(ctx, s, dl, p, wf.rt, refNodes[i], wf.f)
+				if err != nil {
+					return
+				}
+				for _, a := range aux {
+					a.Identifiers = a.Identifiers.Merge(wf.f.Identifiers)
+					a.References = envoyx.MergeRefs(a.References, refRefs[i])
+				}
+				out = append(out, aux...)
+			}
+		}
+		return
+	}()
+	if err != nil {
+		err = errors.Wrap(err, "failed to decode filters")
+		return
+	}
+
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource chart
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (d StoreDecoder) decodeChart(ctx context.Context, s store.Storer, dl dal.FullService, f types.ChartFilter) (out envoyx.NodeSet, err error) {
+	// @todo this might need to be improved.
+	//       Currently, no resource is vast enough to pose a problem.
+	rr, _, err := store.SearchComposeCharts(ctx, s, f)
+	if err != nil {
+		return
+	}
+
+	for _, r := range rr {
+		var n *envoyx.Node
+		n, err = ChartToEnvoyNode(r)
+		if err != nil {
+			return
+		}
+		out = append(out, n)
+	}
+
+	return
+}
+
+func ChartToEnvoyNode(r *types.Chart) (node *envoyx.Node, err error) {
+	// Identifiers
+	ii := envoyx.MakeIdentifiers(
+		r.Handle,
+		r.ID,
+	)
+
+	// Handle references
+	// Omit any non-defined values
+	refs := map[string]envoyx.Ref{}
+	if r.NamespaceID > 0 {
+		refs["NamespaceID"] = envoyx.Ref{
+			ResourceType: "corteza::compose:namespace",
+			Identifiers:  envoyx.MakeIdentifiers(r.NamespaceID),
+		}
+	}
+
+	refs = envoyx.MergeRefs(refs, decodeChartRefs(r))
+
+	var scope envoyx.Scope
+
+	scope = envoyx.Scope{
+		ResourceType: refs["NamespaceID"].ResourceType,
+		Identifiers:  refs["NamespaceID"].Identifiers,
+	}
+	for k, ref := range refs {
+		// @todo temporary solution to not needlessly scope resources.
+		//       Optimally, this would be selectively handled by codegen.
+		if !strings.HasPrefix(ref.ResourceType, "corteza::compose") {
+			continue
+		}
+
+		ref.Scope = scope
+		refs[k] = ref
+	}
+
+	node = &envoyx.Node{
+		Resource: r,
+
+		ResourceType: types.ChartResourceType,
+		Identifiers:  ii,
+		References:   refs,
+		Scope:        scope,
+	}
+	return
+}
+
+func (d StoreDecoder) makeChartFilter(scope *envoyx.Node, refs map[string]*envoyx.Node, auxf envoyx.ResourceFilter) (out types.ChartFilter) {
+	out.Limit = auxf.Limit
+
+	ids, hh := auxf.Identifiers.Idents()
+	_ = ids
+	_ = hh
+
+	out.ChartID = id.Strings(ids...)
+
+	if len(hh) > 0 {
+		out.Handle = hh[0]
+	}
+
+	// Refs
+	var (
+		ar *envoyx.Node
+		ok bool
+	)
+	_ = ar
+	_ = ok
+
+	ar, ok = refs["NamespaceID"]
+	if ok {
+		out.NamespaceID = ar.Resource.GetID()
+	}
+
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource module
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (d StoreDecoder) decodeModule(ctx context.Context, s store.Storer, dl dal.FullService, f types.ModuleFilter) (out envoyx.NodeSet, err error) {
+	// @todo this might need to be improved.
+	//       Currently, no resource is vast enough to pose a problem.
+	rr, _, err := store.SearchComposeModules(ctx, s, f)
+	if err != nil {
+		return
+	}
+
+	for _, r := range rr {
+		var n *envoyx.Node
+		n, err = ModuleToEnvoyNode(r)
+		if err != nil {
+			return
+		}
+		out = append(out, n)
+	}
+
+	aux, err := d.extendedModuleDecoder(ctx, s, dl, f, out)
+	if err != nil {
+		return
+	}
+	out = append(out, aux...)
+
+	return
+}
+
+func ModuleToEnvoyNode(r *types.Module) (node *envoyx.Node, err error) {
+	// Identifiers
+	ii := envoyx.MakeIdentifiers(
+		r.Handle,
+		r.ID,
+	)
+
+	// Handle references
+	// Omit any non-defined values
+	refs := map[string]envoyx.Ref{}
+	if r.NamespaceID > 0 {
+		refs["NamespaceID"] = envoyx.Ref{
+			ResourceType: "corteza::compose:namespace",
+			Identifiers:  envoyx.MakeIdentifiers(r.NamespaceID),
+		}
+	}
+
+	var scope envoyx.Scope
+
+	scope = envoyx.Scope{
+		ResourceType: refs["NamespaceID"].ResourceType,
+		Identifiers:  refs["NamespaceID"].Identifiers,
+	}
+	for k, ref := range refs {
+		// @todo temporary solution to not needlessly scope resources.
+		//       Optimally, this would be selectively handled by codegen.
+		if !strings.HasPrefix(ref.ResourceType, "corteza::compose") {
+			continue
+		}
+
+		ref.Scope = scope
+		refs[k] = ref
+	}
+
+	node = &envoyx.Node{
+		Resource: r,
+
+		ResourceType: types.ModuleResourceType,
+		Identifiers:  ii,
+		References:   refs,
+		Scope:        scope,
+	}
+	return
+}
+
+func (d StoreDecoder) makeModuleFilter(scope *envoyx.Node, refs map[string]*envoyx.Node, auxf envoyx.ResourceFilter) (out types.ModuleFilter) {
+	out.Limit = auxf.Limit
+
+	ids, hh := auxf.Identifiers.Idents()
+	_ = ids
+	_ = hh
+
+	out.ModuleID = id.Strings(ids...)
+
+	if len(hh) > 0 {
+		out.Handle = hh[0]
+	}
+
+	// Refs
+	var (
+		ar *envoyx.Node
+		ok bool
+	)
+	_ = ar
+	_ = ok
+
+	ar, ok = refs["NamespaceID"]
+	if ok {
+		out.NamespaceID = ar.Resource.GetID()
+	}
+
+	out = d.extendModuleFilter(scope, refs, auxf, out)
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource moduleField
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (d StoreDecoder) decodeModuleField(ctx context.Context, s store.Storer, dl dal.FullService, f types.ModuleFieldFilter) (out envoyx.NodeSet, err error) {
+	// @todo this might need to be improved.
+	//       Currently, no resource is vast enough to pose a problem.
+	rr, _, err := store.SearchComposeModuleFields(ctx, s, f)
+	if err != nil {
+		return
+	}
+
+	for _, r := range rr {
+		var n *envoyx.Node
+		n, err = ModuleFieldToEnvoyNode(r)
+		if err != nil {
+			return
+		}
+		out = append(out, n)
+	}
+
+	return
+}
+
+func ModuleFieldToEnvoyNode(r *types.ModuleField) (node *envoyx.Node, err error) {
+	// Identifiers
+	ii := envoyx.MakeIdentifiers(
+		r.ID,
+		r.Name,
+	)
+
+	// Handle references
+	// Omit any non-defined values
+	refs := map[string]envoyx.Ref{}
+	if r.ModuleID > 0 {
+		refs["ModuleID"] = envoyx.Ref{
+			ResourceType: "corteza::compose:module",
+			Identifiers:  envoyx.MakeIdentifiers(r.ModuleID),
+		}
+	}
+
+	refs = envoyx.MergeRefs(refs, decodeModuleFieldRefs(r))
+
+	var scope envoyx.Scope
+
+	scope = envoyx.Scope{
+		ResourceType: refs["NamespaceID"].ResourceType,
+		Identifiers:  refs["NamespaceID"].Identifiers,
+	}
+	for k, ref := range refs {
+		// @todo temporary solution to not needlessly scope resources.
+		//       Optimally, this would be selectively handled by codegen.
+		if !strings.HasPrefix(ref.ResourceType, "corteza::compose") {
+			continue
+		}
+
+		ref.Scope = scope
+		refs[k] = ref
+	}
+
+	node = &envoyx.Node{
+		Resource: r,
+
+		ResourceType: types.ModuleFieldResourceType,
+		Identifiers:  ii,
+		References:   refs,
+		Scope:        scope,
+	}
+	return
+}
+
+// Resource should define a custom filter builder
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource namespace
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (d StoreDecoder) decodeNamespace(ctx context.Context, s store.Storer, dl dal.FullService, f types.NamespaceFilter) (out envoyx.NodeSet, err error) {
+	// @todo this might need to be improved.
+	//       Currently, no resource is vast enough to pose a problem.
+	rr, _, err := store.SearchComposeNamespaces(ctx, s, f)
+	if err != nil {
+		return
+	}
+
+	for _, r := range rr {
+		var n *envoyx.Node
+		n, err = NamespaceToEnvoyNode(r)
+		if err != nil {
+			return
+		}
+		out = append(out, n)
+	}
+
+	return
+}
+
+func NamespaceToEnvoyNode(r *types.Namespace) (node *envoyx.Node, err error) {
+	// Identifiers
+	ii := envoyx.MakeIdentifiers(
+		r.ID,
+		r.Slug,
+	)
+
+	// Handle references
+	// Omit any non-defined values
+	refs := map[string]envoyx.Ref{}
+
+	var scope envoyx.Scope
+
+	scope = envoyx.Scope{
+		ResourceType: types.NamespaceResourceType,
+		Identifiers:  ii,
+	}
+
+	node = &envoyx.Node{
+		Resource: r,
+
+		ResourceType: types.NamespaceResourceType,
+		Identifiers:  ii,
+		References:   refs,
+		Scope:        scope,
+	}
+	return
+}
+
+func (d StoreDecoder) makeNamespaceFilter(scope *envoyx.Node, refs map[string]*envoyx.Node, auxf envoyx.ResourceFilter) (out types.NamespaceFilter) {
+	out.Limit = auxf.Limit
+
+	ids, hh := auxf.Identifiers.Idents()
+	_ = ids
+	_ = hh
+
+	out.NamespaceID = id.Strings(ids...)
+
+	if len(hh) > 0 {
+		out.Slug = hh[0]
+	}
+
+	// Refs
+	var (
+		ar *envoyx.Node
+		ok bool
+	)
+	_ = ar
+	_ = ok
+
+	out = d.extendNamespaceFilter(scope, refs, auxf, out)
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource page
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (d StoreDecoder) decodePage(ctx context.Context, s store.Storer, dl dal.FullService, f types.PageFilter) (out envoyx.NodeSet, err error) {
+	// @todo this might need to be improved.
+	//       Currently, no resource is vast enough to pose a problem.
+	rr, _, err := store.SearchComposePages(ctx, s, f)
+	if err != nil {
+		return
+	}
+
+	for _, r := range rr {
+		var n *envoyx.Node
+		n, err = PageToEnvoyNode(r)
+		if err != nil {
+			return
+		}
+		out = append(out, n)
+	}
+
+	return
+}
+
+func PageToEnvoyNode(r *types.Page) (node *envoyx.Node, err error) {
+	// Identifiers
+	ii := envoyx.MakeIdentifiers(
+		r.Handle,
+		r.ID,
+	)
+
+	// Handle references
+	// Omit any non-defined values
+	refs := map[string]envoyx.Ref{}
+	if r.ModuleID > 0 {
+		refs["ModuleID"] = envoyx.Ref{
+			ResourceType: "corteza::compose:module",
+			Identifiers:  envoyx.MakeIdentifiers(r.ModuleID),
+		}
+	}
+	if r.NamespaceID > 0 {
+		refs["NamespaceID"] = envoyx.Ref{
+			ResourceType: "corteza::compose:namespace",
+			Identifiers:  envoyx.MakeIdentifiers(r.NamespaceID),
+		}
+	}
+	if r.SelfID > 0 {
+		refs["SelfID"] = envoyx.Ref{
+			ResourceType: "corteza::compose:page",
+			Identifiers:  envoyx.MakeIdentifiers(r.SelfID),
+		}
+	}
+
+	refs = envoyx.MergeRefs(refs, decodePageRefs(r))
+
+	var scope envoyx.Scope
+
+	scope = envoyx.Scope{
+		ResourceType: refs["NamespaceID"].ResourceType,
+		Identifiers:  refs["NamespaceID"].Identifiers,
+	}
+	for k, ref := range refs {
+		// @todo temporary solution to not needlessly scope resources.
+		//       Optimally, this would be selectively handled by codegen.
+		if !strings.HasPrefix(ref.ResourceType, "corteza::compose") {
+			continue
+		}
+
+		ref.Scope = scope
+		refs[k] = ref
+	}
+
+	node = &envoyx.Node{
+		Resource: r,
+
+		ResourceType: types.PageResourceType,
+		Identifiers:  ii,
+		References:   refs,
+		Scope:        scope,
+	}
+	return
+}
+
+func (d StoreDecoder) makePageFilter(scope *envoyx.Node, refs map[string]*envoyx.Node, auxf envoyx.ResourceFilter) (out types.PageFilter) {
+	out.Limit = auxf.Limit
+
+	ids, hh := auxf.Identifiers.Idents()
+	_ = ids
+	_ = hh
+
+	out.PageID = id.Strings(ids...)
+
+	if len(hh) > 0 {
+		out.Handle = hh[0]
+	}
+
+	// Refs
+	var (
+		ar *envoyx.Node
+		ok bool
+	)
+	_ = ar
+	_ = ok
+
+	ar, ok = refs["ModuleID"]
+	if ok {
+		out.ModuleID = ar.Resource.GetID()
+	}
+
+	ar, ok = refs["NamespaceID"]
+	if ok {
+		out.NamespaceID = ar.Resource.GetID()
+	}
+
+	ar, ok = refs["SelfID"]
+	if ok {
+		out.ParentID = ar.Resource.GetID()
+	}
+
+	out = d.extendPageFilter(scope, refs, auxf, out)
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource pageLayout
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (d StoreDecoder) decodePageLayout(ctx context.Context, s store.Storer, dl dal.FullService, f types.PageLayoutFilter) (out envoyx.NodeSet, err error) {
+	// @todo this might need to be improved.
+	//       Currently, no resource is vast enough to pose a problem.
+	rr, _, err := store.SearchComposePageLayouts(ctx, s, f)
+	if err != nil {
+		return
+	}
+
+	for _, r := range rr {
+		var n *envoyx.Node
+		n, err = PageLayoutToEnvoyNode(r)
+		if err != nil {
+			return
+		}
+		out = append(out, n)
+	}
+
+	return
+}
+
+func PageLayoutToEnvoyNode(r *types.PageLayout) (node *envoyx.Node, err error) {
+	// Identifiers
+	ii := envoyx.MakeIdentifiers(
+		r.Handle,
+		r.ID,
+	)
+
+	// Handle references
+	// Omit any non-defined values
+	refs := map[string]envoyx.Ref{}
+	if r.NamespaceID > 0 {
+		refs["NamespaceID"] = envoyx.Ref{
+			ResourceType: "corteza::compose:namespace",
+			Identifiers:  envoyx.MakeIdentifiers(r.NamespaceID),
+		}
+	}
+	if r.OwnedBy > 0 {
+		refs["OwnedBy"] = envoyx.Ref{
+			ResourceType: "corteza::system:user",
+			Identifiers:  envoyx.MakeIdentifiers(r.OwnedBy),
+		}
+	}
+	if r.PageID > 0 {
+		refs["PageID"] = envoyx.Ref{
+			ResourceType: "corteza::compose:page",
+			Identifiers:  envoyx.MakeIdentifiers(r.PageID),
+		}
+	}
+	if r.ParentID > 0 {
+		refs["ParentID"] = envoyx.Ref{
+			ResourceType: "corteza::compose:page-layout",
+			Identifiers:  envoyx.MakeIdentifiers(r.ParentID),
+		}
+	}
+
+	var scope envoyx.Scope
+
+	scope = envoyx.Scope{
+		ResourceType: refs["NamespaceID"].ResourceType,
+		Identifiers:  refs["NamespaceID"].Identifiers,
+	}
+	for k, ref := range refs {
+		// @todo temporary solution to not needlessly scope resources.
+		//       Optimally, this would be selectively handled by codegen.
+		if !strings.HasPrefix(ref.ResourceType, "corteza::compose") {
+			continue
+		}
+
+		ref.Scope = scope
+		refs[k] = ref
+	}
+
+	node = &envoyx.Node{
+		Resource: r,
+
+		ResourceType: types.PageLayoutResourceType,
+		Identifiers:  ii,
+		References:   refs,
+		Scope:        scope,
+	}
+	return
+}
+
+func (d StoreDecoder) makePageLayoutFilter(scope *envoyx.Node, refs map[string]*envoyx.Node, auxf envoyx.ResourceFilter) (out types.PageLayoutFilter) {
+	out.Limit = auxf.Limit
+
+	ids, hh := auxf.Identifiers.Idents()
+	_ = ids
+	_ = hh
+
+	out.PageLayoutID = id.Strings(ids...)
+
+	if len(hh) > 0 {
+		out.Handle = hh[0]
+	}
+
+	// Refs
+	var (
+		ar *envoyx.Node
+		ok bool
+	)
+	_ = ar
+	_ = ok
+
+	ar, ok = refs["NamespaceID"]
+	if ok {
+		out.NamespaceID = ar.Resource.GetID()
+	}
+
+	ar, ok = refs["PageID"]
+	if ok {
+		out.PageID = ar.Resource.GetID()
+	}
+
+	ar, ok = refs["ParentID"]
+	if ok {
+		out.ParentID = ar.Resource.GetID()
+	}
+
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Utilities
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (d StoreDecoder) getStorer(p envoyx.DecodeParams) (s store.Storer, err error) {
+	aux, ok := p.Params[paramsKeyStorer]
+	if ok {
+		s, ok = aux.(store.Storer)
+		if ok {
+			return
+		}
+	}
+
+	err = errors.Errorf("store decoder expects a storer conforming to store.Storer interface")
+	return
+}
+
+func (d StoreDecoder) getDal(p envoyx.DecodeParams) (dl dal.FullService, err error) {
+	aux, ok := p.Params[paramsKeyDAL]
+	if ok {
+		dl, ok = aux.(dal.FullService)
+		if ok {
+			return
+		}
+	}
+
+	err = errors.Errorf("store decoder expects a DAL conforming to dal.FullService interface")
+	return
+}
+
+func (d StoreDecoder) prepFilters(ff map[string]envoyx.ResourceFilter) (out []filterWrap) {
+	out = make([]filterWrap, 0, len(ff))
+	for rt, f := range ff {
+		// Handle resources that don't belong to this decoder
+		if !strings.HasPrefix(rt, "corteza::compose") {
+			continue
+		}
+
+		out = append(out, filterWrap{rt: rt, f: f})
+	}
+
+	return
+}
+
+func (d StoreDecoder) getScopeNodes(ctx context.Context, s store.Storer, dl dal.FullService, ff []filterWrap) (scopes envoyx.NodeSet, err error) {
+	// Get all requested scopes
+	scopes = make(envoyx.NodeSet, len(ff))
+
+	err = func() (err error) {
+		for i, fw := range ff {
+			if fw.f.Scope.ResourceType == "" {
+				continue
+			}
+
+			// For now the scope can only point to namespace so this will do
+			var nn envoyx.NodeSet
+			nn, err = d.decodeNamespace(ctx, s, dl, d.makeNamespaceFilter(nil, nil, envoyx.ResourceFilter{Identifiers: fw.f.Scope.Identifiers}))
+			if err != nil {
+				return
+			}
+			if len(nn) > 1 {
+				err = fmt.Errorf("ambiguous scope %v: matches multiple resources", fw.f.Scope)
+				return
+			}
+			if len(nn) == 0 {
+				err = fmt.Errorf("invalid scope %v: resource not found", fw.f)
+				return
+			}
+
+			scopes[i] = nn[0]
+		}
+		return
+	}()
+	if err != nil {
+		err = errors.Wrap(err, "failed to decode node scopes")
+		return
+	}
+
+	return
+}
+
+// getReferenceNodes returns all of the nodes referenced by the nodes defined by the filters
+//
+// The nodes are provided as a slice (the same order as the filters) and as a map for easier lookups.
+func (d StoreDecoder) getReferenceNodes(ctx context.Context, s store.Storer, dl dal.FullService, ff []filterWrap) (nodes []map[string]*envoyx.Node, refs []map[string]envoyx.Ref, err error) {
+	nodes = make([]map[string]*envoyx.Node, len(ff))
+	refs = make([]map[string]envoyx.Ref, len(ff))
+	err = func() (err error) {
+		for i, a := range ff {
+			if len(a.f.Refs) == 0 {
+				continue
+			}
+
+			auxr := make(map[string]*envoyx.Node, len(a.f.Refs))
+			auxa := make(map[string]envoyx.Ref)
+			for field, ref := range a.f.Refs {
+				f := ref.ResourceFilter()
+				aux, err := d.decode(ctx, s, dl, envoyx.DecodeParams{
+					Type:   envoyx.DecodeTypeStore,
+					Filter: f,
+				})
+				if err != nil {
+					return err
+				}
+
+				// @todo consider changing this.
+				//       Currently it's required because the .decode may return some
+				//       nested nodes as well.
+				//       Consider a flag or a new function.
+				aux = envoyx.NodesForResourceType(ref.ResourceType, aux...)
+				if len(aux) == 0 {
+					return fmt.Errorf("invalid reference %v", ref)
+				}
+				if len(aux) > 1 {
+					return fmt.Errorf("ambiguous reference: too many resources returned %v", a.f)
+				}
+
+				auxr[field] = aux[0]
+				auxa[field] = aux[0].ToRef()
+			}
+
+			nodes[i] = auxr
+			refs[i] = auxa
+		}
+		return
+	}()
+	if err != nil {
+		err = errors.Wrap(err, "failed to decode node references")
+		return
+	}
+
+	return
+}

server/compose/envoy/store_decode.go

@@ -0,0 +1,301 @@
+package envoy
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cortezaproject/corteza/server/compose/dalutils"
+	"github.com/cortezaproject/corteza/server/compose/service"
+	"github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/cortezaproject/corteza/server/pkg/id"
+	"github.com/cortezaproject/corteza/server/pkg/rbac"
+	"github.com/cortezaproject/corteza/server/store"
+	"github.com/spf13/cast"
+)
+
+func (d StoreDecoder) extendNamespaceFilter(scope *envoyx.Node, refs map[string]*envoyx.Node, auxf envoyx.ResourceFilter, base types.NamespaceFilter) (out types.NamespaceFilter) {
+	out = base
+
+	if scope == nil {
+		return
+	}
+
+	if scope.ResourceType == "" {
+		return
+	}
+
+	// Overwrite it
+	out.NamespaceID = id.Strings(scope.Resource.GetID())
+
+	return
+}
+
+func (d StoreDecoder) extendModuleFilter(scope *envoyx.Node, refs map[string]*envoyx.Node, auxf envoyx.ResourceFilter, base types.ModuleFilter) (out types.ModuleFilter) {
+	out = base
+
+	if scope == nil {
+		return
+	}
+
+	if scope.ResourceType == "" {
+		return
+	}
+
+	// Overwrite it
+	out.NamespaceID = scope.Resource.GetID()
+
+	return
+}
+
+func (d StoreDecoder) extendPageFilter(scope *envoyx.Node, refs map[string]*envoyx.Node, auxf envoyx.ResourceFilter, base types.PageFilter) (out types.PageFilter) {
+	out = base
+
+	if scope == nil {
+		return
+	}
+
+	if scope.ResourceType == "" {
+		return
+	}
+
+	// Overwrite it
+	out.NamespaceID = scope.Resource.GetID()
+
+	return
+}
+
+func (d StoreDecoder) makeModuleFieldFilter(scope *envoyx.Node, refs map[string]*envoyx.Node, auxf envoyx.ResourceFilter) (out types.ModuleFieldFilter) {
+	out.Limit = auxf.Limit
+
+	ids, hh := auxf.Identifiers.Idents()
+	_ = ids
+	_ = hh
+
+	// Refs
+	var (
+		ar *envoyx.Node
+		ok bool
+	)
+	_ = ar
+	_ = ok
+
+	// ar, ok = refs["ModuleID"]
+	// if ok {
+	// 	out.ModuleID = ar.Resource.GetID()
+	// }
+
+	return
+}
+
+func (d StoreDecoder) extendedModuleDecoder(ctx context.Context, s store.Storer, dl dal.FullService, f types.ModuleFilter, base envoyx.NodeSet) (out envoyx.NodeSet, err error) {
+	var ff envoyx.NodeSet
+
+	for _, b := range base {
+		mod := b.Resource.(*types.Module)
+
+		// Get all of the related module fields, append them to the output and
+		// the original module (so other code can have access to the related fields)
+		ff, err = d.decodeModuleField(ctx, s, dl, types.ModuleFieldFilter{ModuleID: []uint64{mod.ID}})
+		if err != nil {
+			return
+		}
+
+		for _, f := range ff {
+			f.Scope = b.Scope
+			f.References = envoyx.MergeRefs(f.References, b.References, map[string]envoyx.Ref{
+				"ModuleID": b.ToRef(),
+			})
+			for k, ref := range f.References {
+				ref.Scope = b.Scope
+				f.References[k] = ref
+			}
+
+			mod.Fields = append(mod.Fields, f.Resource.(*types.ModuleField))
+		}
+
+		out = append(out, ff...)
+	}
+
+	return
+}
+
+func decodeChartRefs(c *types.Chart) (refs map[string]envoyx.Ref) {
+	refs = make(map[string]envoyx.Ref, len(c.Config.Reports))
+
+	for i, r := range c.Config.Reports {
+		if r.ModuleID == 0 {
+			continue
+		}
+
+		refs[fmt.Sprintf("Config.Reports.%d.ModuleID", i)] = envoyx.Ref{
+			ResourceType: types.ModuleResourceType,
+			Identifiers:  envoyx.MakeIdentifiers(r.ModuleID),
+		}
+	}
+
+	return
+}
+
+func decodeModuleFieldRefs(c *types.ModuleField) (refs map[string]envoyx.Ref) {
+	refs = make(map[string]envoyx.Ref, 1)
+
+	refs["NamespaceID"] = envoyx.Ref{
+		ResourceType: types.NamespaceResourceType,
+		Identifiers:  envoyx.MakeIdentifiers(c.NamespaceID),
+	}
+
+	id := c.Options.UInt64("moduleID")
+	if id == 0 {
+		return
+	}
+
+	refs["Options.ModuleID"] = envoyx.Ref{
+		ResourceType: types.ModuleResourceType,
+		Identifiers:  envoyx.MakeIdentifiers(id),
+	}
+
+	return
+}
+
+func decodePageRefs(p *types.Page) (refs map[string]envoyx.Ref) {
+	refs = make(map[string]envoyx.Ref, len(p.Blocks)/2)
+
+	for index, b := range p.Blocks {
+		switch b.Kind {
+		case "RecordList":
+			refs = envoyx.MergeRefs(refs, getPageBlockRecordListRefs(b, index))
+
+		case "Automation":
+			refs = envoyx.MergeRefs(refs, getPageBlockAutomationRefs(b, index))
+
+		case "RecordOrganizer":
+			refs = envoyx.MergeRefs(refs, getPageBlockRecordOrganizerRefs(b, index))
+
+		case "Chart":
+			refs = envoyx.MergeRefs(refs, getPageBlockChartRefs(b, index))
+
+		case "Calendar":
+			refs = envoyx.MergeRefs(refs, getPageBlockCalendarRefs(b, index))
+
+		case "Metric":
+			refs = envoyx.MergeRefs(refs, getPageBlockMetricRefs(b, index))
+
+		case "Comment":
+			refs = envoyx.MergeRefs(refs, getPageBlockCommentRefs(b, index))
+
+		case "Progress":
+			refs = envoyx.MergeRefs(refs, getPageBlockProgressRefs(b, index))
+		}
+	}
+
+	return
+}
+
+func (d StoreDecoder) extendDecoder(ctx context.Context, s store.Storer, dl dal.FullService, p envoyx.DecodeParams, rt string, refs map[string]*envoyx.Node, rf envoyx.ResourceFilter) (out envoyx.NodeSet, err error) {
+	switch rt {
+	// @todo consider hooking into the regular record resource type as well
+	case ComposeRecordDatasourceAuxType:
+		return d.decodeRecordDatasource(ctx, s, dl, p, refs, rf)
+	}
+
+	return
+}
+
+func (d StoreDecoder) decodeRecordDatasource(ctx context.Context, s store.Storer, dl dal.FullService, p envoyx.DecodeParams, refs map[string]*envoyx.Node, rf envoyx.ResourceFilter) (out envoyx.NodeSet, err error) {
+	var (
+		ok bool
+
+		module        *types.Module
+		moduleNode    *envoyx.Node
+		namespace     *types.Namespace
+		namespaceNode *envoyx.Node
+	)
+
+	// Get the refs
+	namespaceNode, ok = refs["NamespaceID"]
+	if !ok {
+		err = fmt.Errorf("missing NamespaceID reference")
+		return
+	}
+	namespace = namespaceNode.Resource.(*types.Namespace)
+
+	moduleNode, ok = refs["ModuleID"]
+	if !ok {
+		err = fmt.Errorf("missing ModuleID reference")
+		return
+	}
+	module = moduleNode.Resource.(*types.Module)
+
+	// Get the iterator
+	iter, _, err := dalutils.ComposeRecordsIterator(ctx, dl, module, types.RecordFilter{
+		Query:       rf.Query,
+		ModuleID:    module.ID,
+		NamespaceID: namespace.ID,
+		Paging: filter.Paging{
+			Limit: rf.Limit,
+		},
+	})
+
+	if err != nil {
+		return
+	}
+
+	mv := make(map[string]bool)
+	dt := make(map[string]bool)
+	for _, f := range module.Fields {
+		if f.Multi {
+			mv[f.Name] = true
+		}
+		if f.Kind == "DateTime" {
+			dt[f.Name] = true
+		}
+	}
+
+	for _, n := range []string{"createdAt", "updatedAt", "deletedAt"} {
+		dt[n] = true
+	}
+
+	rds := &RecordDatasource{
+		Provider:       &iteratorProvider{iter: iter},
+		multivalues:    mv,
+		datetimeFields: dt,
+
+		refToID:     make(map[string]uint64),
+		existingIDs: make(map[uint64]bool),
+		// @todo consider providing defaults from the outside
+		Mapping: envoyx.DatasourceMapping{
+			KeyField:    []string{"id"},
+			Defaultable: true,
+		},
+	}
+
+	// Get access controller to enforce field-level read permissions
+	ac := service.AccessControl(s)
+	if rbac.Global() == nil {
+		ac = nil
+	}
+
+	rds.Provider, err = mkIteratorProvider(ctx, ac, s, dl, iter, module, cast.ToBool(p.Params["resolveRefs"]), cast.ToBool(p.Params["includeRefID"]))
+	if err != nil {
+		return
+	}
+
+	rr := map[string]envoyx.Ref{
+		"NamespaceID": namespaceNode.ToRef(),
+		"ModuleID":    moduleNode.ToRef(),
+	}
+
+	out = append(out, &envoyx.Node{
+		Datasource:   rds,
+		ResourceType: ComposeRecordDatasourceAuxType,
+
+		Identifiers: envoyx.MakeIdentifiers(module.ID, module.Handle),
+		References:  rr,
+		Scope:       moduleNode.Scope,
+	})
+
+	return
+}

server/compose/envoy/store_encode.go

@@ -0,0 +1,233 @@
+package envoy
+
+import (
+	"context"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/compose/service"
+	"github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/store"
+	"github.com/pkg/errors"
+)
+
+func (e StoreEncoder) encode(ctx context.Context, p envoyx.EncodeParams, s store.Storer, rt string, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
+	dl, err := e.grabDal(p)
+	if err != nil {
+		return
+	}
+
+	switch rt {
+	case ComposeRecordDatasourceAuxType:
+		err = e.encodeRecordDatasources(ctx, p, s, dl, nn, tree)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func (e StoreEncoder) setChartDefaults(res *types.Chart) (err error) {
+	if res.CreatedAt.IsZero() {
+		res.CreatedAt = time.Now()
+	}
+	return
+}
+
+func (e StoreEncoder) validateChart(*types.Chart) (err error) {
+	return
+}
+
+func (e StoreEncoder) setModuleDefaults(res *types.Module) (err error) {
+	if res.CreatedAt.IsZero() {
+		res.CreatedAt = time.Now()
+	}
+	return
+}
+
+func (e StoreEncoder) validateModule(*types.Module) (err error) {
+	return
+}
+
+func (e StoreEncoder) setModuleFieldDefaults(res *types.ModuleField) (err error) {
+	if res.CreatedAt.IsZero() {
+		res.CreatedAt = time.Now()
+	}
+
+	if res.Kind == "" {
+		res.Kind = "String"
+	}
+
+	// Update validator ID
+	maxValidatorID := uint64(0)
+	for _, v := range res.Expressions.Validators {
+		if v.ValidatorID > maxValidatorID {
+			maxValidatorID = v.ValidatorID
+		}
+	}
+
+	for _, v := range res.Expressions.Validators {
+		if v.ValidatorID == 0 {
+			v.ValidatorID = maxValidatorID + 1
+			maxValidatorID++
+		}
+	}
+
+	return
+}
+
+func (e StoreEncoder) sanitizeModuleFieldBeforeSave(f *types.ModuleField) (err error) {
+	delete(f.Options, "module")
+	return
+}
+
+func (e StoreEncoder) validateModuleField(*types.ModuleField) (err error) {
+	return
+}
+
+func (e StoreEncoder) setNamespaceDefaults(res *types.Namespace) (err error) {
+	if res.CreatedAt.IsZero() {
+		res.CreatedAt = time.Now()
+	}
+	return
+}
+
+func (e StoreEncoder) validateNamespace(*types.Namespace) (err error) {
+	return
+}
+
+func (e StoreEncoder) setPageDefaults(res *types.Page) (err error) {
+	if res.CreatedAt.IsZero() {
+		res.CreatedAt = time.Now()
+	}
+
+	if res.Title == "" {
+		res.Title = res.Handle
+	}
+
+	// Update pageblock ID
+	maxPageBlockID := uint64(0)
+	for _, b := range res.Blocks {
+		if b.BlockID > maxPageBlockID {
+			maxPageBlockID = b.BlockID
+		}
+	}
+
+	for i, b := range res.Blocks {
+		if b.BlockID == 0 {
+			b.BlockID = maxPageBlockID + 1
+			maxPageBlockID++
+			res.Blocks[i] = b
+		}
+	}
+
+	return
+}
+
+func (e StoreEncoder) validatePage(*types.Page) (err error) {
+	return
+}
+
+func (e StoreEncoder) setPageLayoutDefaults(res *types.PageLayout) (err error) {
+	if res.CreatedAt.IsZero() {
+		res.CreatedAt = time.Now()
+	}
+
+	// @note these pageblocks reference the ones define on the page so nothing
+	//       to do here.
+
+	return
+}
+
+func (e StoreEncoder) validatePageLayout(*types.PageLayout) (err error) {
+	return
+}
+
+func (e StoreEncoder) setRecordDefaults(*types.Record) (err error) {
+	return
+}
+
+func (e StoreEncoder) validateRecord(*types.Record) (err error) {
+	return
+}
+
+func (e StoreEncoder) prepare(ctx context.Context, p envoyx.EncodeParams, s store.Storer, rt string, nn envoyx.NodeSet) (err error) {
+	dl, err := e.grabDal(p)
+	if err != nil {
+		return
+	}
+
+	switch rt {
+	case ComposeRecordDatasourceAuxType:
+		return e.prepareRecordDatasource(ctx, p, s, dl, nn)
+	}
+
+	return
+}
+
+func (e StoreEncoder) encodeModuleExtend(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, nested envoyx.NodeSet, tree envoyx.Traverser) (err error) {
+
+	// Push fields under mod
+	mod := n.Resource.(*types.Module)
+	for _, n := range nested {
+		if n.ResourceType != types.ModuleFieldResourceType {
+			continue
+		}
+
+		mod.Fields = append(mod.Fields, n.Resource.(*types.ModuleField))
+	}
+
+	// Register to DAL
+	dl, err := e.grabDal(p)
+	if err != nil {
+		return
+	}
+
+	nsNode := tree.ParentForRef(n, n.References["NamespaceID"])
+	ns := nsNode.Resource.(*types.Namespace)
+
+	// @todo get connection and things from there
+	models, err := service.ModulesToModelSet(dl, ns, mod)
+	if err != nil {
+		return err
+	}
+
+	// @note there is only one model so this is ok
+	_, err = dl.ReplaceModel(ctx, nil, models[0])
+	return
+}
+
+func (e StoreEncoder) postModulesEncode(ctx context.Context, p envoyx.EncodeParams, s store.Storer, tree envoyx.Traverser, nn envoyx.NodeSet) (err error) {
+	dl, err := e.grabDal(p)
+	if err != nil {
+		return
+	}
+
+	for _, n := range nn {
+		cc := tree.ChildrenForResourceType(n, ComposeRecordDatasourceAuxType)
+		err = e.encodeRecordDatasources(ctx, p, s, dl, cc, tree)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (e *StoreEncoder) grabDal(p envoyx.EncodeParams) (dl dal.FullService, err error) {
+	auxs, ok := p.Params[paramsKeyDAL]
+	if !ok {
+		err = errors.Errorf("store encoder expects a dal conforming to dal.FullService interface")
+		return
+	}
+
+	dl, ok = auxs.(dal.FullService)
+	if !ok {
+		err = errors.Errorf("store encoder expects a dal conforming to dal.FullService interface")
+
+		return
+	}
+
+	return
+}

server/compose/envoy/store_encode_datasource.go

@@ -0,0 +1,420 @@
+package envoy
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/compose/dalutils"
+	"github.com/cortezaproject/corteza/server/compose/service"
+	"github.com/cortezaproject/corteza/server/compose/service/values"
+	"github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/pkg/auth"
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx/datasource"
+	"github.com/cortezaproject/corteza/server/pkg/id"
+	"github.com/cortezaproject/corteza/server/store"
+	systemEnvoy "github.com/cortezaproject/corteza/server/system/envoy"
+	"github.com/spf13/cast"
+)
+
+const (
+	recordBatchMaxChunk = 100
+)
+
+var (
+	rvSanitizer = values.Sanitizer()
+	rvValidator = values.Validator()
+	rvFormatter = values.Formatter()
+)
+
+func (e StoreEncoder) prepareRecordDatasource(ctx context.Context, p envoyx.EncodeParams, s store.Storer, dl dal.FullService, nn envoyx.NodeSet) (err error) {
+	// @todo match existing records; for now use just the ID like V1
+
+	for _, n := range nn {
+		if n.Datasource == nil {
+			panic("unexpected state: cannot call prepareRecordDatasource with nodes without a defined Datasource")
+		}
+
+		ds, ok := n.Datasource.(*RecordDatasource)
+		if !ok {
+			panic("unexpected datasource type: node expecting type of RecordDatasource")
+		}
+
+		err = e.prepareRecords(ctx, p, s, dl, ds, nn)
+		if err != nil {
+			return
+		}
+		err = ds.Reset(ctx)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (e StoreEncoder) prepareRecords(ctx context.Context, p envoyx.EncodeParams, s store.Storer, dl dal.FullService, ds *RecordDatasource, nn envoyx.NodeSet) (err error) {
+	var (
+		aux   = make(datasource.RawRecord)
+		more  bool
+		ident []string
+	)
+
+	ds.refToID = make(map[string]uint64)
+	ds.existingIDs = make(map[uint64]bool)
+
+	// Just so we don't need to do a branch later down the line
+	cex := ds.CheckExisting
+	if cex == nil || len(ds.Mapping.KeyField) == 0 {
+		cex = func(ctx context.Context, ref ...[]string) ([]uint64, error) {
+			return make([]uint64, len(ref)), nil
+		}
+	}
+
+	for {
+		ident, more, err = ds.Next(ctx, aux)
+		if err != nil || !more {
+			return
+		}
+
+		// @todo we'll need to batch these up
+		existing, err := cex(ctx, ident)
+		if err != nil {
+			return err
+		}
+
+		if existing[0] != 0 {
+			ds.existingIDs[existing[0]] = true
+		}
+
+		if existing[0] == 0 {
+			existing[0] = id.Next()
+		}
+
+		ds.AddRef(existing[0], ident...)
+
+		// Construct a simple record for basic validation/preprocessing
+		rec := types.Record{}
+		for k, vv := range aux {
+			for i, v := range vv.Values {
+				// Ignore errors at this point since some values can't yet be properly casted
+				rec.SetValue(k, uint(i), v)
+			}
+		}
+
+		// @note defaults and validation will have to happen again when encoding
+		//       since we won't persist it.
+		//       Consider supporting updating the datasource's data.
+		err = e.setRecordDefaults(&rec)
+		if err != nil {
+			return err
+		}
+
+		err = e.validateRecord(&rec)
+		if err != nil {
+			return err
+		}
+	}
+}
+
+func (e StoreEncoder) encodeRecordDatasources(ctx context.Context, p envoyx.EncodeParams, s store.Storer, dl dal.FullService, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
+	for _, n := range nn {
+		err = e.encodeRecordDatasource(ctx, p, s, dl, n, tree)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (e StoreEncoder) encodeRecordDatasource(ctx context.Context, p envoyx.EncodeParams, s store.Storer, dl dal.FullService, n *envoyx.Node, tree envoyx.Traverser) (err error) {
+	var (
+		auxRec = make(datasource.RawRecord)
+		more   bool
+		ident  []string
+
+		nsNode  *envoyx.Node
+		ns      *types.Namespace
+		modNode *envoyx.Node
+		mod     *types.Module
+
+		// This was already validated so we can blindly cast
+		ds = n.Datasource.(*RecordDatasource)
+	)
+
+	// Get the parent namespace
+	nsNode = tree.ParentForRef(n, n.References["NamespaceID"])
+	if nsNode == nil {
+		err = fmt.Errorf("cannot encode record datasource: missing reference for NamespaceID")
+		return
+	}
+	ns = nsNode.Resource.(*types.Namespace)
+
+	// Get the parent module
+	modNode = tree.ParentForRef(n, n.References["ModuleID"])
+	if modNode == nil {
+		err = fmt.Errorf("cannot encode record datasource: missing reference for ModuleID")
+		return
+	}
+	mod = modNode.Resource.(*types.Module)
+
+	// Prepare getters for related resources
+	recordGetters := e.makeRecordGetters(dl, tree, n)
+	userGetters, sysUserFields := e.makeUserGetters(s, dl, tree, mod, n)
+
+	maykr := e.recordMaker(ns, mod, recordGetters, userGetters, sysUserFields)
+
+	var (
+		rec     types.Record
+		creates types.RecordSet
+		updates types.RecordSet
+		rve     *types.RecordValueErrorSet
+	)
+	for {
+		ident, more, err = ds.Next(ctx, auxRec)
+		if err != nil {
+			return
+		}
+		if !more {
+			break
+		}
+
+		err = func() (err error) {
+			rec, err = maykr(ctx, auxRec)
+			if err != nil {
+				return
+			}
+
+			// Do these at the end so they can't be overwritten
+			rec.NamespaceID = ns.ID
+			rec.ModuleID = mod.ID
+			rec.CreatedAt = time.Now()
+			rec.OwnedBy = service.CalcRecordOwner(0, rec.OwnedBy, p.Encoder.DefaultUserID)
+			rec.CreatedBy = e.getCreatedBy(ctx, rec)
+			rec.ID, err = ds.ResolveRefS(ident...)
+			if err != nil {
+				return err
+			}
+
+			// Standard record processing
+			rec.Values.SetUpdatedFlag(true)
+			//
+			rve = service.RecordValueUpdateOpCheck(ctx, nil, mod, rec.Values)
+			if !rve.IsValid() {
+				return rve
+			}
+			//
+			rve = service.RecordPreparer(ctx, s, rvSanitizer, rvValidator, rvFormatter, mod, &rec)
+			if !rve.IsValid() {
+				return rve
+			}
+
+			ax := rec
+
+			if ds.existingIDs[rec.ID] {
+				updates = append(updates, &ax)
+			} else {
+				creates = append(creates, &ax)
+			}
+
+			if len(creates) > recordBatchMaxChunk {
+				err = dalutils.ComposeRecordCreate(ctx, dl, mod, creates...)
+				if err != nil {
+					return
+				}
+
+				creates = make(types.RecordSet, 0, recordBatchMaxChunk/2)
+			}
+
+			if len(updates) > recordBatchMaxChunk {
+				err = dalutils.ComposeRecordUpdate(ctx, dl, mod, updates...)
+				if err != nil {
+					return
+				}
+
+				updates = make(types.RecordSet, 0, recordBatchMaxChunk/2)
+			}
+			return
+		}()
+
+		if p.Defer != nil {
+			p.Defer()
+		}
+		if err != nil {
+			if p.DeferNok != nil {
+				err = p.DeferNok(err)
+			}
+			if err != nil {
+				return
+			}
+		} else if p.DeferOk != nil {
+			p.DeferOk()
+		}
+	}
+
+	if len(creates) > 0 {
+		err = dalutils.ComposeRecordCreate(ctx, dl, mod, creates...)
+	}
+
+	if len(updates) > 0 {
+		err = dalutils.ComposeRecordUpdate(ctx, dl, mod, updates...)
+	}
+	return
+}
+
+func (e StoreEncoder) getCreatedBy(ctx context.Context, rec types.Record) uint64 {
+	if rec.CreatedBy > 0 {
+		return rec.CreatedBy
+	}
+
+	idt := auth.GetIdentityFromContext(ctx)
+	if idt == nil {
+		return 0
+	}
+
+	return idt.Identity()
+}
+
+// maceRecordGetters returns a map of getters where the key is the field name
+func (e StoreEncoder) makeRecordGetters(dl dal.FullService, tree envoyx.Traverser, n *envoyx.Node) (getters map[string]*recordGetter) {
+	var (
+		modIndex = make(map[string]envoyx.Ref)
+		dsIndex  = make(map[string]envoyx.Ref)
+	)
+
+	// First pass collects all the references
+	for k, v := range n.References {
+		if k == "NamespaceID" || k == "ModuleID" {
+			continue
+		}
+
+		// Encoded as <fieldName>.<refKind>
+		pp := strings.Split(k, ".")
+		f := pp[0]
+		kind := pp[1]
+
+		switch kind {
+		case "module":
+			modIndex[f] = v
+		case "datasource":
+			dsIndex[f] = v
+		}
+	}
+
+	// - second pass makes the getters
+	getters = make(map[string]*recordGetter)
+	for k := range modIndex {
+		getters[k] = makeRecordGetter(dl, tree, n, modIndex[k], dsIndex[k])
+	}
+
+	return
+}
+
+// sysUserRefFields are the record system user fields in the spellings
+// types.Record.SetValue understands
+var sysUserRefFields = []string{
+	"ownedBy", "OwnedBy", "owned_by",
+	"createdBy", "CreatedBy", "created_by",
+	"updatedBy", "UpdatedBy", "updated_by",
+	"deletedBy", "DeletedBy", "deleted_by",
+}
+
+func (e StoreEncoder) makeUserGetters(s store.Storer, dl dal.FullService, tree envoyx.Traverser, mod *types.Module, n *envoyx.Node) (getters map[string]*systemEnvoy.UserGetter, sysFields map[string]bool) {
+	userGetter := systemEnvoy.MakeUserGetter(s, tree)
+
+	getters = make(map[string]*systemEnvoy.UserGetter, len(sysUserRefFields))
+	sysFields = make(map[string]bool, len(sysUserRefFields))
+	for _, name := range sysUserRefFields {
+		getters[name] = userGetter
+		sysFields[name] = true
+	}
+
+	// Module fields take precedence over the system field spellings; only
+	// User kind fields resolve through the getter
+	for _, f := range mod.Fields {
+		if f.Kind == "User" {
+			getters[f.Name] = userGetter
+		} else {
+			delete(getters, f.Name)
+		}
+		delete(sysFields, f.Name)
+	}
+
+	return
+}
+
+func (e StoreEncoder) recordMaker(ns *types.Namespace, mod *types.Module, recordGetters map[string]*recordGetter, userGetters map[string]*systemEnvoy.UserGetter, sysUserFields map[string]bool) func(ctx context.Context, auxRec datasource.RawRecord) (r types.Record, err error) {
+	return func(ctx context.Context, auxRec datasource.RawRecord) (rec types.Record, err error) {
+		// Iterate mapified record values and populate the provided values
+		var auxvv types.RecordValueSet
+		for k, vv := range auxRec {
+			auxvv = nil
+
+			if rg := recordGetters[k]; rg != nil {
+				auxvv, err = resolveRefs(ctx, rg.resolve, k, vv.Values)
+			} else if ug := userGetters[k]; ug != nil {
+				vals := vv.Values
+				// system fields hold a single user; the first value wins
+				if sysUserFields[k] && len(vals) > 1 {
+					vals = vals[:1]
+				}
+				auxvv, err = resolveRefs(ctx, ug.Resolve, k, vals)
+			} else {
+				// Default is regular values
+				for i, v := range vv.Values {
+					err = rec.SetValue(k, uint(i), v)
+					if err != nil {
+						return
+					}
+				}
+				continue
+			}
+			if err != nil {
+				return
+			}
+
+			for _, auxv := range auxvv {
+				err = rec.SetValue(k, auxv.Place, auxv.Ref)
+				if err != nil {
+					return
+				}
+			}
+		}
+
+		return
+	}
+}
+
+// resolveRefs resolves reference values (user or record identifiers) into IDs
+func resolveRefs(ctx context.Context, resolve func(context.Context, any) (uint64, error), k string, vals []string) (out types.RecordValueSet, err error) {
+	var (
+		ref   uint64
+		place uint
+	)
+
+	for _, v := range vals {
+		// mini-csv decoding can produce empty entries; nothing to resolve
+		if strings.TrimSpace(v) == "" {
+			continue
+		}
+
+		ref, err = resolve(ctx, v)
+		if err != nil {
+			return
+		}
+
+		out = append(out, &types.RecordValue{
+			Name:  k,
+			Place: place,
+			Ref:   ref,
+			Value: cast.ToString(ref),
+		})
+		place++
+	}
+
+	return
+}

server/compose/envoy/timezone.go

@@ -0,0 +1,50 @@
+package envoy
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/spf13/cast"
+)
+
+// resolveTimezone reads the "timezone" param and returns the loaded location.
+// Returns a nil location when no timezone is set. An unresolvable timezone is
+// reported as an error so the export fails loudly instead of silently skipping
+// the conversion.
+func resolveTimezone(p envoyx.EncodeParams) (*time.Location, error) {
+	tz := cast.ToString(p.Params["timezone"])
+	if tz == "" {
+		return nil, nil
+	}
+	loc, err := time.LoadLocation(tz)
+	if err != nil {
+		return nil, fmt.Errorf("invalid export timezone %q: %w", tz, err)
+	}
+	return loc, nil
+}
+
+// formatInTimezone reparses v as a datetime and reformats it in loc.
+// On parse failure returns v unchanged so values are never dropped.
+func formatInTimezone(v string, loc *time.Location) string {
+	if loc == nil || v == "" {
+		return v
+	}
+
+	layouts := []string{time.RFC3339Nano, time.RFC3339, "2006-01-02 15:04:05.999999999 -0700 MST", "2006-01-02 15:04:05 -0700 MST", "2006-01-02T15:04:05Z"}
+	for _, ly := range layouts {
+		if t, err := time.Parse(ly, v); err == nil {
+			return t.In(loc).Format(time.RFC3339)
+		}
+	}
+
+	// Fall back to zone-less layouts; assume values are stored in UTC.
+	naiveLayouts := []string{"2006-01-02T15:04:05.999999999", "2006-01-02T15:04:05", "2006-01-02 15:04:05.999999999", "2006-01-02 15:04:05"}
+	for _, ly := range naiveLayouts {
+		if t, err := time.ParseInLocation(ly, v, time.UTC); err == nil {
+			return t.In(loc).Format(time.RFC3339)
+		}
+	}
+
+	return v
+}

server/compose/envoy/yaml_decode.go

@@ -0,0 +1,492 @@
+package envoy
+
+import (
+	"fmt"
+	"strings"
+
+	automationTypes "github.com/cortezaproject/corteza/server/automation/types"
+	"github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/pkg/y7s"
+	"github.com/modern-go/reflect2"
+	"gopkg.in/yaml.v3"
+)
+
+const (
+	ComposeRecordDatasourceAuxType = "corteza::compose:record-datasource"
+)
+
+func (d *auxYamlDoc) unmarshalYAML(k string, n *yaml.Node) (out envoyx.NodeSet, err error) { return }
+
+func (d *auxYamlDoc) unmarshalChartConfigNode(r *types.Chart, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
+	err = y7s.EachMap(n, func(k, v *yaml.Node) error {
+		if k.Value != "reports" {
+			return nil
+		}
+
+		if y7s.IsSeq(v) {
+			var (
+				auxRefs   = make(map[string]envoyx.Ref)
+				auxIdents envoyx.Identifiers
+				i         = -1
+			)
+			err = y7s.EachSeq(v, func(c *yaml.Node) error {
+				i++
+
+				auxRefs, auxIdents, err = d.unmarshalChartConfigReportNode(r, c, i)
+				refs = envoyx.MergeRefs(refs, auxRefs)
+				idents = idents.Merge(auxIdents)
+				return err
+			})
+			if err != nil {
+				return err
+			}
+		} else {
+			refs, idents, err = d.unmarshalChartConfigReportNode(r, v, 0)
+			return err
+		}
+		return nil
+	})
+
+	return
+}
+
+func (d *auxYamlDoc) unmarshalChartConfigReportNode(r *types.Chart, n *yaml.Node, index int) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
+	err = y7s.EachMap(n, func(k, v *yaml.Node) error {
+		switch strings.ToLower(k.Value) {
+		case "module", "mod", "moduleid", "module_id":
+			var auxi any
+			y7s.DecodeScalar(v, "moduleID", &auxi)
+			refs = map[string]envoyx.Ref{
+				fmt.Sprintf("Config.Reports.%d.ModuleID", index): {
+					ResourceType: types.ModuleResourceType,
+					Identifiers:  envoyx.MakeIdentifiers(auxi),
+				},
+			}
+		}
+		return nil
+	})
+	return
+}
+
+func (d *auxYamlDoc) unmarshalPageBlocksNode(r *types.Page, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
+	refs = map[string]envoyx.Ref{}
+
+	for index, b := range r.Blocks {
+		switch b.Kind {
+		case "RecordList":
+			refs = envoyx.MergeRefs(refs, getPageBlockRecordListRefs(b, index))
+
+		case "Automation":
+			refs = envoyx.MergeRefs(refs, getPageBlockAutomationRefs(b, index))
+
+		case "RecordOrganizer":
+			refs = envoyx.MergeRefs(refs, getPageBlockRecordOrganizerRefs(b, index))
+
+		case "Chart":
+			refs = envoyx.MergeRefs(refs, getPageBlockChartRefs(b, index))
+
+		case "Calendar":
+			refs = envoyx.MergeRefs(refs, getPageBlockCalendarRefs(b, index))
+
+		case "Metric":
+			refs = envoyx.MergeRefs(refs, getPageBlockMetricRefs(b, index))
+
+		case "Comment":
+			refs = envoyx.MergeRefs(refs, getPageBlockCommentRefs(b, index))
+
+		case "Progress":
+			refs = envoyx.MergeRefs(refs, getPageBlockProgressRefs(b, index))
+		}
+	}
+
+	return
+}
+
+func getPageBlockRecordListRefs(b types.PageBlock, index int) (refs map[string]envoyx.Ref) {
+	refs = make(map[string]envoyx.Ref)
+
+	id := optString(b.Options, "module", "moduleID")
+	if id == "" || id == "0" {
+		return
+	}
+
+	refs[fmt.Sprintf("Blocks.%d.Options.ModuleID", index)] = envoyx.Ref{
+		ResourceType: types.ModuleResourceType,
+		Identifiers:  envoyx.MakeIdentifiers(id),
+	}
+
+	return
+}
+
+func getPageBlockChartRefs(b types.PageBlock, index int) (refs map[string]envoyx.Ref) {
+	refs = make(map[string]envoyx.Ref)
+
+	id := optString(b.Options, "chart", "chartID")
+	if id == "" || id == "0" {
+		return
+	}
+
+	refs[fmt.Sprintf("Blocks.%d.Options.ChartID", index)] = envoyx.Ref{
+		ResourceType: types.ChartResourceType,
+		Identifiers:  envoyx.MakeIdentifiers(id),
+	}
+
+	return
+}
+
+func getPageBlockCalendarRefs(b types.PageBlock, index int) (refs map[string]envoyx.Ref) {
+	refs = make(map[string]envoyx.Ref)
+
+	ff, _ := b.Options["feeds"].([]interface{})
+	for j, f := range ff {
+		feed, _ := f.(map[string]interface{})
+		opt, _ := (feed["options"]).(map[string]interface{})
+
+		id := optString(opt, "module", "moduleID")
+		if id == "" || id == "0" {
+			return
+		}
+
+		refs[fmt.Sprintf("Blocks.%d.Options.feeds.%d.ModuleID", index, j)] = envoyx.Ref{
+			ResourceType: types.ModuleResourceType,
+			Identifiers:  envoyx.MakeIdentifiers(id),
+		}
+	}
+
+	return
+}
+
+func getPageBlockMetricRefs(b types.PageBlock, index int) (refs map[string]envoyx.Ref) {
+	refs = make(map[string]envoyx.Ref)
+
+	mm, _ := b.Options["metrics"].([]interface{})
+	for j, m := range mm {
+		mops, _ := m.(map[string]interface{})
+
+		id := optString(mops, "module", "moduleID")
+		if id == "" || id == "0" {
+			return
+		}
+
+		refs[fmt.Sprintf("Blocks.%d.Options.metrics.%d.ModuleID", index, j)] = envoyx.Ref{
+			ResourceType: types.ModuleResourceType,
+			Identifiers:  envoyx.MakeIdentifiers(id),
+		}
+	}
+
+	return
+}
+
+func getPageBlockCommentRefs(b types.PageBlock, index int) (refs map[string]envoyx.Ref) {
+	// Same difference
+	return getPageBlockRecordListRefs(b, index)
+}
+
+func getPageBlockProgressRefs(b types.PageBlock, index int) (refs map[string]envoyx.Ref) {
+	refs = make(map[string]envoyx.Ref)
+	var aux *envoyx.Ref
+
+	aux = getPageBlockProgressValueRefs(b.Options["minValue"])
+	if aux != nil {
+		refs[fmt.Sprintf("Blocks.%d.Options.minValue.ModuleID", index)] = *aux
+	}
+
+	aux = getPageBlockProgressValueRefs(b.Options["maxValue"])
+	if aux != nil {
+		refs[fmt.Sprintf("Blocks.%d.Options.maxValue.ModuleID", index)] = *aux
+	}
+
+	aux = getPageBlockProgressValueRefs(b.Options["value"])
+	if aux != nil {
+		refs[fmt.Sprintf("Blocks.%d.Options.value.ModuleID", index)] = *aux
+	}
+
+	return
+}
+
+func getPageBlockProgressValueRefs(val any) (ref *envoyx.Ref) {
+	if reflect2.IsNil(val) {
+		return
+	}
+
+	opt, ok := val.(map[string]interface{})
+	if !ok {
+		return
+	}
+
+	id := optString(opt, "module", "moduleID")
+	if id == "" || id == "0" {
+		return
+	}
+
+	return &envoyx.Ref{
+		ResourceType: types.ModuleResourceType,
+		Identifiers:  envoyx.MakeIdentifiers(id),
+	}
+}
+
+func getPageBlockRecordOrganizerRefs(b types.PageBlock, index int) (refs map[string]envoyx.Ref) {
+	// Same difference
+	return getPageBlockRecordListRefs(b, index)
+}
+
+func getPageBlockAutomationRefs(b types.PageBlock, index int) (refs map[string]envoyx.Ref) {
+	refs = make(map[string]envoyx.Ref)
+
+	bb, _ := b.Options["buttons"].([]interface{})
+	for buttonIx, b := range bb {
+		button, _ := b.(map[string]interface{})
+		id := optString(button, "workflow", "workflowID")
+		if id == "" || id == "0" {
+			return
+		}
+
+		refs[fmt.Sprintf("Blocks.%d.Options.buttons.%d.WorkflowID", index, buttonIx)] = envoyx.Ref{
+			ResourceType: automationTypes.WorkflowResourceType,
+			Identifiers:  envoyx.MakeIdentifiers(id),
+		}
+	}
+
+	return
+}
+
+func optString(opt map[string]interface{}, kk ...string) string {
+	for _, k := range kk {
+		if vr, has := opt[k]; has {
+			v, _ := vr.(string)
+			return v
+		}
+	}
+	return ""
+}
+
+func (d *auxYamlDoc) unmarshalModuleFieldOptionsNode(r *types.ModuleField, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
+	refs = make(map[string]envoyx.Ref)
+
+	err = y7s.EachMap(n, func(k, v *yaml.Node) error {
+		switch strings.ToLower(k.Value) {
+		case "module", "module_id", "moduleid":
+			var aux any
+			err = y7s.DecodeScalar(v, "moduleID", &aux)
+			if err != nil {
+				return err
+			}
+			refs["Options.ModuleID"] = envoyx.Ref{
+				ResourceType: types.ModuleResourceType,
+				Identifiers:  envoyx.MakeIdentifiers(aux),
+			}
+		default:
+			return nil
+		}
+		return nil
+	})
+
+	return
+}
+
+func (d *auxYamlDoc) unmarshalModuleFieldDefaultValueNode(r *types.ModuleField, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
+	var rvs = types.RecordValueSet{}
+
+	switch n.Kind {
+	case yaml.ScalarNode:
+		rvs = rvs.Set(&types.RecordValue{Value: n.Value})
+
+	case yaml.SequenceNode:
+		_ = y7s.EachSeq(n, func(v *yaml.Node) error {
+			rvs = rvs.Set(&types.RecordValue{Value: n.Value, Place: uint(len(rvs))})
+			return nil
+		})
+	}
+
+	r.DefaultValue = rvs
+	return
+}
+
+func (d *auxYamlDoc) unmarshalModuleFieldExpressionsNode(r *types.ModuleField, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
+
+	err = y7s.EachMap(n, func(k *yaml.Node, v *yaml.Node) error {
+		switch k.Value {
+		case "sanitizer":
+			var aux string
+			err = y7s.DecodeScalar(v, "sanitizer", &aux)
+			if err != nil {
+				return err
+			}
+			r.Expressions.Sanitizers = append(r.Expressions.Sanitizers, aux)
+
+		case "sanitizers":
+			return y7s.EachSeq(v, func(san *yaml.Node) error {
+				r.Expressions.Sanitizers = append(r.Expressions.Sanitizers, san.Value)
+				return nil
+			})
+
+		case "validator":
+			var aux types.ModuleFieldValidator
+			err = v.Decode(&aux)
+			if err != nil {
+				return err
+			}
+			r.Expressions.Validators = append(r.Expressions.Validators, aux)
+
+		case "validators":
+			return y7s.Each(v, func(k *yaml.Node, v *yaml.Node) error {
+				var aux types.ModuleFieldValidator
+				if y7s.IsKind(v, yaml.MappingNode) {
+					err = v.Decode(&aux)
+					if err != nil {
+						return err
+					}
+				} else {
+					aux.Test = k.Value
+					aux.Error = v.Value
+				}
+
+				r.Expressions.Validators = append(r.Expressions.Validators, aux)
+				return nil
+			})
+
+		case "formatter":
+			r.Expressions.Formatters = append(r.Expressions.Formatters, v.Value)
+			return nil
+
+		case "formatters":
+			return y7s.EachSeq(v, func(san *yaml.Node) error {
+				r.Expressions.Formatters = append(r.Expressions.Formatters, san.Value)
+				return nil
+			})
+		}
+		return nil
+	})
+
+	return
+}
+
+func (d *auxYamlDoc) postProcessNestedModuleNodes(nn envoyx.NodeSet) (out envoyx.NodeSet, err error) {
+	// Get all references from all module fields
+	refs := make(map[string]envoyx.Ref)
+	for _, n := range nn {
+		if n.ResourceType != types.ModuleFieldResourceType {
+			continue
+		}
+
+		if len(n.References) == 0 {
+			continue
+		}
+
+		r, ok := n.References["Options.ModuleID"]
+		if !ok {
+			continue
+		}
+
+		refs[fmt.Sprintf("%s.module", n.Identifiers.FriendlyIdentifier())] = r
+		refs[fmt.Sprintf("%s.datasource", n.Identifiers.FriendlyIdentifier())] = envoyx.Ref{
+			ResourceType: ComposeRecordDatasourceAuxType,
+			Identifiers:  r.Identifiers,
+			Scope:        r.Scope,
+			Optional:     true,
+		}
+	}
+
+	// Update datasources with references
+
+	for _, n := range nn {
+		if n.Datasource == nil {
+			continue
+		}
+
+		n.References = envoyx.MergeRefs(n.References, refs)
+	}
+	out = nn
+	return
+}
+
+func (d *auxYamlDoc) unmarshalPagesExtendedNode(dctx documentContext, n *yaml.Node, meta ...*yaml.Node) (out envoyx.NodeSet, err error) {
+	return d.unmarshalPageNode(dctx, n, meta...)
+}
+
+func (d *auxYamlDoc) unmarshalSourceExtendedNode(dctx documentContext, n *yaml.Node, meta ...*yaml.Node) (out envoyx.NodeSet, err error) {
+	var r envoyx.DatasourceMapping
+
+	// @todo we're omitting errors because there will be a bunch due to invalid
+	//       resource field types. This might be a bit unstable as other errors may
+	//       also get ignored.
+	//
+	//       A potential fix would be to firstly unmarshal into an any, check errors
+	//       and then unmarshal into the resource while omitting errors.
+	n.Decode(&r)
+
+	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
+		var auxNodeValue any
+		_ = auxNodeValue
+
+		switch strings.ToLower(k.Value) {
+		case "origin", "from":
+			err = y7s.DecodeScalar(n, "origin", &r.SourceIdent)
+			if err != nil {
+				return err
+			}
+
+		case "key", "index", "pk":
+			if !y7s.IsKind(n, yaml.SequenceNode) {
+				r.KeyField = []string{n.Value}
+			} else {
+				r.KeyField = make([]string, 0, 3)
+				y7s.EachSeq(n, func(n *yaml.Node) error {
+					r.KeyField = append(r.KeyField, n.Value)
+					return nil
+				})
+			}
+
+		case "map":
+			return n.Decode(&r.Mapping)
+		}
+
+		return nil
+	})
+
+	// @todo for now we only support record datasources; extend when needed
+	auxN := &envoyx.Node{
+		Datasource: &RecordDatasource{
+			Mapping: r,
+		},
+
+		ResourceType: ComposeRecordDatasourceAuxType,
+		Identifiers:  dctx.parentIdent,
+	}
+	auxN.References, auxN.Scope = d.procMappingRefs(r.References)
+	out = append(out, auxN)
+
+	return
+}
+
+func (d *auxYamlDoc) procMappingRefs(in map[string]string) (out map[string]envoyx.Ref, scope envoyx.Scope) {
+	out = make(map[string]envoyx.Ref)
+
+	nsII := envoyx.MakeIdentifiers(in["namespace"])
+	if len(nsII.Slice) > 0 {
+		scope = envoyx.Scope{
+			ResourceType: types.NamespaceResourceType,
+			Identifiers:  nsII,
+		}
+	}
+
+	if len(nsII.Slice) > 0 {
+		out["NamespaceID"] = envoyx.Ref{
+			ResourceType: types.NamespaceResourceType,
+			Identifiers:  nsII,
+			Scope:        scope,
+		}
+	}
+
+	modII := envoyx.MakeIdentifiers(in["module"])
+	if len(modII.Slice) > 0 {
+		out["ModuleID"] = envoyx.Ref{
+			ResourceType: types.ModuleResourceType,
+			Identifiers:  envoyx.MakeIdentifiers(in["module"]),
+			Scope:        scope,
+		}
+	}
+
+	return
+}

server/compose/envoy/yaml_encode.gen.go

@@ -0,0 +1,721 @@
+package envoy
+
+// This file is auto-generated.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+//
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/pkg/y7s"
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v3"
+)
+
+type (
+	// YamlEncoder is responsible for encoding Corteza resources into
+	// a YAML supported format
+	YamlEncoder struct{}
+)
+
+const (
+	paramsKeyWriter = "writer"
+)
+
+// Encode encodes the given Corteza resources into some YAML supported format
+//
+// Encoding should not do any additional processing apart from matching with
+// dependencies and runtime validation
+//
+// Preparation runs validation, default value initialization, matching with
+// already existing instances, ...
+//
+// The prepare function receives a set of nodes grouped by the resource type.
+// This enables some batching optimization and simplifications when it comes to
+// matching with existing resources.
+//
+// Prepare does not receive any placeholder nodes which are used solely
+// for dependency resolution.
+func (e YamlEncoder) Encode(ctx context.Context, p envoyx.EncodeParams, rt string, nodes envoyx.NodeSet, tt envoyx.Traverser) (err error) {
+	var (
+		out *yaml.Node
+		aux *yaml.Node
+	)
+	_ = aux
+
+	w, err := e.getWriter(p)
+	if err != nil {
+		return
+	}
+
+	switch rt {
+	case types.ChartResourceType:
+		aux, err = e.encodeCharts(ctx, p, nodes, tt)
+		if err != nil {
+			return
+		}
+		// Root level resources are always encoded as a map
+		out, err = y7s.AddMap(out, "chart", aux)
+		if err != nil {
+			return
+		}
+	case types.ModuleResourceType:
+		aux, err = e.encodeModules(ctx, p, nodes, tt)
+		if err != nil {
+			return
+		}
+		// Root level resources are always encoded as a map
+		out, err = y7s.AddMap(out, "module", aux)
+		if err != nil {
+			return
+		}
+	case types.ModuleFieldResourceType:
+		aux, err = e.encodeModuleFields(ctx, p, nodes, tt)
+		if err != nil {
+			return
+		}
+		// Root level resources are always encoded as a map
+		out, err = y7s.AddMap(out, "moduleField", aux)
+		if err != nil {
+			return
+		}
+	case types.NamespaceResourceType:
+		aux, err = e.encodeNamespaces(ctx, p, nodes, tt)
+		if err != nil {
+			return
+		}
+		// Root level resources are always encoded as a map
+		out, err = y7s.AddMap(out, "namespace", aux)
+		if err != nil {
+			return
+		}
+	case types.PageResourceType:
+		aux, err = e.encodePages(ctx, p, nodes, tt)
+		if err != nil {
+			return
+		}
+		// Root level resources are always encoded as a map
+		out, err = y7s.AddMap(out, "page", aux)
+		if err != nil {
+			return
+		}
+	case types.PageLayoutResourceType:
+		aux, err = e.encodePageLayouts(ctx, p, nodes, tt)
+		if err != nil {
+			return
+		}
+		// Root level resources are always encoded as a map
+		out, err = y7s.AddMap(out, "pageLayout", aux)
+		if err != nil {
+			return
+		}
+
+	default:
+		out, err = e.encode(ctx, out, p, rt, nodes, tt)
+		if err != nil {
+			return
+		}
+	}
+
+	// Don't output nil values since that will produce broken yaml docs
+	if out == nil {
+		return
+	}
+
+	return yaml.NewEncoder(w).Encode(out)
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource chart
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (e YamlEncoder) encodeCharts(ctx context.Context, p envoyx.EncodeParams, nodes envoyx.NodeSet, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	var aux *yaml.Node
+	for _, n := range nodes {
+		aux, err = e.encodeChart(ctx, p, n, tt)
+		if err != nil {
+			return
+		}
+
+		out, err = y7s.AddSeq(out, aux)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// encodeChart focuses on the specific resource invoked by the Encode method
+func (e YamlEncoder) encodeChart(ctx context.Context, p envoyx.EncodeParams, node *envoyx.Node, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	res := node.Resource.(*types.Chart)
+
+	// Pre-compute some map values so we can omit error checking when encoding yaml nodes
+	auxConfig, err := e.encodeChartConfigC(ctx, p, tt, node, res, res.Config)
+	if err != nil {
+		return
+	}
+	auxCreatedAt, err := e.encodeTimestamp(p, res.CreatedAt)
+	if err != nil {
+		return
+	}
+	auxDeletedAt, err := e.encodeTimestampNil(p, res.DeletedAt)
+	if err != nil {
+		return
+	}
+
+	auxNamespaceID, err := e.encodeRef(p, res.NamespaceID, "NamespaceID", node, tt)
+	if err != nil {
+		return
+	}
+	auxUpdatedAt, err := e.encodeTimestampNil(p, res.UpdatedAt)
+	if err != nil {
+		return
+	}
+
+	out, err = y7s.AddMap(out,
+		"config", auxConfig,
+		"createdAt", auxCreatedAt,
+		"deletedAt", auxDeletedAt,
+		"handle", res.Handle,
+		"chartID", res.ID,
+		"name", res.Name,
+		"namespaceID", auxNamespaceID,
+		"updatedAt", auxUpdatedAt,
+	)
+	if err != nil {
+		return
+	}
+
+	// Handle nested resources
+	var aux *yaml.Node
+	_ = aux
+
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource module
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (e YamlEncoder) encodeModules(ctx context.Context, p envoyx.EncodeParams, nodes envoyx.NodeSet, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	var aux *yaml.Node
+	for _, n := range nodes {
+		aux, err = e.encodeModule(ctx, p, n, tt)
+		if err != nil {
+			return
+		}
+
+		out, err = y7s.AddSeq(out, aux)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// encodeModule focuses on the specific resource invoked by the Encode method
+func (e YamlEncoder) encodeModule(ctx context.Context, p envoyx.EncodeParams, node *envoyx.Node, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	res := node.Resource.(*types.Module)
+
+	// Pre-compute some map values so we can omit error checking when encoding yaml nodes
+
+	auxCreatedAt, err := e.encodeTimestamp(p, res.CreatedAt)
+	if err != nil {
+		return
+	}
+	auxDeletedAt, err := e.encodeTimestampNil(p, res.DeletedAt)
+	if err != nil {
+		return
+	}
+
+	auxNamespaceID, err := e.encodeRef(p, res.NamespaceID, "NamespaceID", node, tt)
+	if err != nil {
+		return
+	}
+	auxUpdatedAt, err := e.encodeTimestampNil(p, res.UpdatedAt)
+	if err != nil {
+		return
+	}
+
+	out, err = y7s.AddMap(out,
+		"config", res.Config,
+		"createdAt", auxCreatedAt,
+		"deletedAt", auxDeletedAt,
+		"handle", res.Handle,
+		"moduleID", res.ID,
+		"meta", res.Meta,
+		"name", res.Name,
+		"namespaceID", auxNamespaceID,
+		"updatedAt", auxUpdatedAt,
+	)
+	if err != nil {
+		return
+	}
+
+	// Handle nested resources
+	var aux *yaml.Node
+	_ = aux
+
+	// When processing module fields, we need to filter out the ones that
+	// don't belong to this module
+	//
+	// @todo offload this to dependency resolution; this is a hack
+	children := tt.ChildrenForResourceType(node, types.ModuleFieldResourceType)
+
+	var proc envoyx.NodeSet
+	selfRef := node.ToRef()
+	for _, c := range children {
+		if c.References["ModuleID"].Equals(selfRef) {
+			proc = append(proc, c)
+		}
+	}
+
+	aux, err = e.encodeModuleFields(ctx, p, proc, tt)
+	if err != nil {
+		return
+	}
+	out, err = y7s.AddMap(out,
+		"moduleField", aux,
+	)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource moduleField
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (e YamlEncoder) encodeModuleFields(ctx context.Context, p envoyx.EncodeParams, nodes envoyx.NodeSet, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	var aux *yaml.Node
+	for _, n := range nodes {
+		aux, err = e.encodeModuleField(ctx, p, n, tt)
+		if err != nil {
+			return
+		}
+
+		out, err = y7s.AddSeq(out, aux)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// encodeModuleField focuses on the specific resource invoked by the Encode method
+func (e YamlEncoder) encodeModuleField(ctx context.Context, p envoyx.EncodeParams, node *envoyx.Node, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	res := node.Resource.(*types.ModuleField)
+
+	// Pre-compute some map values so we can omit error checking when encoding yaml nodes
+
+	auxCreatedAt, err := e.encodeTimestamp(p, res.CreatedAt)
+	if err != nil {
+		return
+	}
+
+	auxDeletedAt, err := e.encodeTimestampNil(p, res.DeletedAt)
+	if err != nil {
+		return
+	}
+
+	auxModuleID, err := e.encodeRef(p, res.ModuleID, "ModuleID", node, tt)
+	if err != nil {
+		return
+	}
+
+	auxOptions, err := e.encodeModuleFieldOptionsC(ctx, p, tt, node, res, res.Options)
+	if err != nil {
+		return
+	}
+
+	auxUpdatedAt, err := e.encodeTimestampNil(p, res.UpdatedAt)
+	if err != nil {
+		return
+	}
+
+	out, err = y7s.AddMap(out,
+		"config", res.Config,
+		"createdAt", auxCreatedAt,
+		"defaultValue", res.DefaultValue,
+		"deletedAt", auxDeletedAt,
+		"expressions", res.Expressions,
+		"id", res.ID,
+		"kind", res.Kind,
+		"label", res.Label,
+		"moduleID", auxModuleID,
+		"multi", res.Multi,
+		"name", res.Name,
+		"options", auxOptions,
+		"place", res.Place,
+		"required", res.Required,
+		"updatedAt", auxUpdatedAt,
+	)
+	if err != nil {
+		return
+	}
+
+	// Handle nested resources
+	var aux *yaml.Node
+	_ = aux
+
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource namespace
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (e YamlEncoder) encodeNamespaces(ctx context.Context, p envoyx.EncodeParams, nodes envoyx.NodeSet, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	var aux *yaml.Node
+	for _, n := range nodes {
+		aux, err = e.encodeNamespace(ctx, p, n, tt)
+		if err != nil {
+			return
+		}
+
+		out, err = y7s.AddSeq(out, aux)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// encodeNamespace focuses on the specific resource invoked by the Encode method
+func (e YamlEncoder) encodeNamespace(ctx context.Context, p envoyx.EncodeParams, node *envoyx.Node, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	res := node.Resource.(*types.Namespace)
+
+	// Pre-compute some map values so we can omit error checking when encoding yaml nodes
+	auxCreatedAt, err := e.encodeTimestamp(p, res.CreatedAt)
+	if err != nil {
+		return
+	}
+	auxDeletedAt, err := e.encodeTimestampNil(p, res.DeletedAt)
+	if err != nil {
+		return
+	}
+
+	auxUpdatedAt, err := e.encodeTimestampNil(p, res.UpdatedAt)
+	if err != nil {
+		return
+	}
+
+	out, err = y7s.AddMap(out,
+		"createdAt", auxCreatedAt,
+		"deletedAt", auxDeletedAt,
+		"enabled", res.Enabled,
+		"id", res.ID,
+		"meta", res.Meta,
+		"name", res.Name,
+		"slug", res.Slug,
+		"updatedAt", auxUpdatedAt,
+	)
+	if err != nil {
+		return
+	}
+
+	// Handle nested resources
+	var aux *yaml.Node
+	_ = aux
+
+	aux, err = e.encodeCharts(ctx, p, tt.ChildrenForResourceType(node, types.ChartResourceType), tt)
+	if err != nil {
+		return
+	}
+	out, err = y7s.AddMap(out,
+		"chart", aux,
+	)
+	if err != nil {
+		return
+	}
+
+	aux, err = e.encodeModules(ctx, p, tt.ChildrenForResourceType(node, types.ModuleResourceType), tt)
+	if err != nil {
+		return
+	}
+	out, err = y7s.AddMap(out,
+		"module", aux,
+	)
+	if err != nil {
+		return
+	}
+
+	aux, err = e.encodePages(ctx, p, tt.ChildrenForResourceType(node, types.PageResourceType), tt)
+	if err != nil {
+		return
+	}
+	out, err = y7s.AddMap(out,
+		"page", aux,
+	)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource page
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (e YamlEncoder) encodePages(ctx context.Context, p envoyx.EncodeParams, nodes envoyx.NodeSet, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	var aux *yaml.Node
+	for _, n := range nodes {
+		aux, err = e.encodePage(ctx, p, n, tt)
+		if err != nil {
+			return
+		}
+
+		out, err = y7s.AddSeq(out, aux)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// encodePage focuses on the specific resource invoked by the Encode method
+func (e YamlEncoder) encodePage(ctx context.Context, p envoyx.EncodeParams, node *envoyx.Node, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	res := node.Resource.(*types.Page)
+
+	// Pre-compute some map values so we can omit error checking when encoding yaml nodes
+	auxBlocks, err := e.encodePageBlocksC(ctx, p, tt, node, res, res.Blocks)
+	if err != nil {
+		return
+	}
+
+	auxCreatedAt, err := e.encodeTimestamp(p, res.CreatedAt)
+	if err != nil {
+		return
+	}
+	auxDeletedAt, err := e.encodeTimestampNil(p, res.DeletedAt)
+	if err != nil {
+		return
+	}
+
+	auxModuleID, err := e.encodeRef(p, res.ModuleID, "ModuleID", node, tt)
+	if err != nil {
+		return
+	}
+	auxNamespaceID, err := e.encodeRef(p, res.NamespaceID, "NamespaceID", node, tt)
+	if err != nil {
+		return
+	}
+	auxSelfID, err := e.encodeRef(p, res.SelfID, "SelfID", node, tt)
+	if err != nil {
+		return
+	}
+
+	auxUpdatedAt, err := e.encodeTimestampNil(p, res.UpdatedAt)
+	if err != nil {
+		return
+	}
+
+	out, err = y7s.AddMap(out,
+		"blocks", auxBlocks,
+		"children", res.Children,
+		"config", res.Config,
+		"createdAt", auxCreatedAt,
+		"deletedAt", auxDeletedAt,
+		"description", res.Description,
+		"handle", res.Handle,
+		"id", res.ID,
+		"meta", res.Meta,
+		"moduleID", auxModuleID,
+		"namespaceID", auxNamespaceID,
+		"selfID", auxSelfID,
+		"title", res.Title,
+		"updatedAt", auxUpdatedAt,
+		"visible", res.Visible,
+		"weight", res.Weight,
+	)
+	if err != nil {
+		return
+	}
+
+	// Handle nested resources
+	var aux *yaml.Node
+	_ = aux
+
+	aux, err = e.encodePageLayouts(ctx, p, tt.ChildrenForResourceType(node, types.PageLayoutResourceType), tt)
+	if err != nil {
+		return
+	}
+	out, err = y7s.AddMap(out,
+		"pageLayout", aux,
+	)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Functions for resource pageLayout
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (e YamlEncoder) encodePageLayouts(ctx context.Context, p envoyx.EncodeParams, nodes envoyx.NodeSet, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	var aux *yaml.Node
+	for _, n := range nodes {
+		aux, err = e.encodePageLayout(ctx, p, n, tt)
+		if err != nil {
+			return
+		}
+
+		out, err = y7s.AddSeq(out, aux)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// encodePageLayout focuses on the specific resource invoked by the Encode method
+func (e YamlEncoder) encodePageLayout(ctx context.Context, p envoyx.EncodeParams, node *envoyx.Node, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	res := node.Resource.(*types.PageLayout)
+
+	// Pre-compute some map values so we can omit error checking when encoding yaml nodes
+
+	auxCreatedAt, err := e.encodeTimestamp(p, res.CreatedAt)
+	if err != nil {
+		return
+	}
+	auxDeletedAt, err := e.encodeTimestampNil(p, res.DeletedAt)
+	if err != nil {
+		return
+	}
+
+	auxNamespaceID, err := e.encodeRef(p, res.NamespaceID, "NamespaceID", node, tt)
+	if err != nil {
+		return
+	}
+	auxOwnedBy, err := e.encodeRef(p, res.OwnedBy, "OwnedBy", node, tt)
+	if err != nil {
+		return
+	}
+	auxPageID, err := e.encodeRef(p, res.PageID, "PageID", node, tt)
+	if err != nil {
+		return
+	}
+	auxParentID, err := e.encodeRef(p, res.ParentID, "ParentID", node, tt)
+	if err != nil {
+		return
+	}
+	auxUpdatedAt, err := e.encodeTimestampNil(p, res.UpdatedAt)
+	if err != nil {
+		return
+	}
+
+	out, err = y7s.AddMap(out,
+		"blocks", res.Blocks,
+		"config", res.Config,
+		"createdAt", auxCreatedAt,
+		"deletedAt", auxDeletedAt,
+		"handle", res.Handle,
+		"id", res.ID,
+		"meta", res.Meta,
+		"namespaceID", auxNamespaceID,
+		"ownedBy", auxOwnedBy,
+		"pageID", auxPageID,
+		"parentID", auxParentID,
+		"updatedAt", auxUpdatedAt,
+		"weight", res.Weight,
+	)
+	if err != nil {
+		return
+	}
+
+	// Handle nested resources
+	var aux *yaml.Node
+	_ = aux
+
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Encoding utils
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (e YamlEncoder) encodeTimestamp(p envoyx.EncodeParams, t time.Time) (any, error) {
+	if t.IsZero() {
+		return nil, nil
+	}
+
+	tz := p.Encoder.PreferredTimezone
+	if tz != "" {
+		tzL, err := time.LoadLocation(tz)
+		if err != nil {
+			return nil, err
+		}
+		t = t.In(tzL)
+	}
+
+	ly := p.Encoder.PreferredTimeLayout
+	if ly == "" {
+		ly = time.RFC3339
+	}
+
+	return t.Format(ly), nil
+}
+
+func (e YamlEncoder) encodeTimestampNil(p envoyx.EncodeParams, t *time.Time) (any, error) {
+	if t == nil {
+		return nil, nil
+	}
+
+	return e.encodeTimestamp(p, *t)
+}
+
+func (e YamlEncoder) encodeRef(p envoyx.EncodeParams, id uint64, field string, node *envoyx.Node, tt envoyx.Traverser) (any, error) {
+	parent := tt.ParentForRef(node, node.References[field])
+
+	// @todo should we panic instead?
+	//       for now gracefully fallback to the ID
+	if parent == nil {
+		return id, nil
+	}
+
+	return parent.Identifiers.FriendlyIdentifier(), nil
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Utility functions
+// // // // // // // // // // // // // // // // // // // // // // // // //
+
+func (e YamlEncoder) getWriter(p envoyx.EncodeParams) (out io.Writer, err error) {
+	aux, ok := p.Params[paramsKeyWriter]
+	if ok {
+		out, ok = aux.(io.Writer)
+		if ok {
+			return
+		}
+	}
+
+	// @todo consider adding support for managing files from a location
+	err = errors.Errorf("YAML encoder expects a writer conforming to io.Writer interface")
+	return
+}
+
+func safeParentIdentifier(tt envoyx.Traverser, n *envoyx.Node, ref envoyx.Ref) (out string) {
+	aux := tt.ParentForRef(n, ref)
+	if aux == nil {
+		return ref.Identifiers.FriendlyIdentifier()
+	}
+
+	return aux.Identifiers.FriendlyIdentifier()
+}

server/compose/envoy/yaml_encode.go

@@ -0,0 +1,238 @@
+package envoy
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/pkg/y7s"
+	"github.com/modern-go/reflect2"
+	"gopkg.in/yaml.v3"
+)
+
+func (e YamlEncoder) encode(ctx context.Context, base *yaml.Node, p envoyx.EncodeParams, rt string, nodes envoyx.NodeSet, tt envoyx.Traverser) (out *yaml.Node, err error) {
+	return
+}
+
+func (e YamlEncoder) encodeChartConfigC(ctx context.Context, p envoyx.EncodeParams, tt envoyx.Traverser, n *envoyx.Node, chart *types.Chart, cfg types.ChartConfig) (_ any, err error) {
+
+	reports, _ := y7s.MakeSeq()
+
+	for i, r := range cfg.Reports {
+		modRef, ok := n.References[fmt.Sprintf("Config.Reports.%d.ModuleID", i)]
+		if !ok {
+			continue
+		}
+
+		r, err := y7s.MakeMap(
+			"filter", r.Filter,
+			"module", safeParentIdentifier(tt, n, modRef),
+			"metrics", r.Metrics,
+			"dimensions", r.Dimensions,
+			"yAxis", r.YAxis,
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		reports, err = y7s.AddSeq(reports, r)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return y7s.MakeMap(
+		"reports", reports,
+		"colorScheme", cfg.ColorScheme,
+	)
+}
+
+func (e YamlEncoder) encodeModuleFieldsC(ctx context.Context, p envoyx.EncodeParams, tt envoyx.Traverser, n *envoyx.Node, mod *types.Module, fields types.ModuleFieldSet) (_ any, err error) {
+
+	fn := tt.ChildrenForResourceType(n, types.ModuleFieldResourceType)
+
+	out, err := e.encodeModuleFields(ctx, p, fn, tt)
+	return out, err
+}
+
+func (e YamlEncoder) encodeModuleFieldOptionsC(ctx context.Context, p envoyx.EncodeParams, tt envoyx.Traverser, n *envoyx.Node, f *types.ModuleField, opt types.ModuleFieldOptions) (_ any, err error) {
+	if opt == nil {
+		opt = make(types.ModuleFieldOptions)
+	}
+	switch f.Kind {
+	case "Record":
+		modRef := n.References["Options.ModuleID"]
+		opt["module"] = safeParentIdentifier(tt, n, modRef)
+		delete(opt, "moduleID")
+
+	case "User":
+		aux := make([]string, 0, 2)
+		for i := range opt.Strings("roles") {
+			roleRef := n.References[fmt.Sprintf("Options.RoleID.%d", i)]
+			aux = append(aux, safeParentIdentifier(tt, n, roleRef))
+		}
+		opt["roles"] = aux
+		delete(opt, "role")
+		delete(opt, "roleID")
+	}
+
+	nopt, _ := y7s.MakeMap()
+	for k, v := range opt {
+		nopt, err = y7s.AddMap(nopt, k, v)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return nopt, nil
+}
+
+func (e YamlEncoder) encodePageBlocksC(ctx context.Context, p envoyx.EncodeParams, tt envoyx.Traverser, n *envoyx.Node, pg *types.Page, bb types.PageBlocks) (_ any, err error) {
+	out, _ := y7s.MakeSeq()
+
+	var aux any
+	for i, b := range pg.Blocks {
+		aux, err = e.encodePageBlockC(ctx, p, tt, n, pg, i, b)
+		if err != nil {
+			return
+		}
+
+		out, err = y7s.AddSeq(out, aux)
+		if err != nil {
+			return
+		}
+	}
+
+	return out, nil
+}
+
+func (e YamlEncoder) encodePageBlockC(ctx context.Context, p envoyx.EncodeParams, tt envoyx.Traverser, n *envoyx.Node, pg *types.Page, index int, b types.PageBlock) (_ any, err error) {
+
+	switch b.Kind {
+	case "RecordList":
+		b = e.cleanupPageblockRecordList(b)
+
+		modRef := n.References[fmt.Sprintf("Blocks.%d.Options.ModuleID", index)]
+		b.Options["module"] = safeParentIdentifier(tt, n, modRef)
+		delete(b.Options, "moduleID")
+		break
+
+	case "RecordOrganizer":
+		modRef := n.References[fmt.Sprintf("Blocks.%d.Options.ModuleID", index)]
+		b.Options["module"] = safeParentIdentifier(tt, n, modRef)
+		delete(b.Options, "moduleID")
+		break
+
+	case "Chart":
+		chrRef := n.References[fmt.Sprintf("Blocks.%d.Options.ChartID", index)]
+		b.Options["chart"] = safeParentIdentifier(tt, n, chrRef)
+		delete(b.Options, "chartID")
+		break
+
+	case "Calendar":
+		ff, _ := b.Options["feeds"].([]interface{})
+		for i, f := range ff {
+			feed, _ := f.(map[string]interface{})
+			fOpts, _ := (feed["options"]).(map[string]interface{})
+
+			modRef := n.References[fmt.Sprintf("Blocks.%d.Options.feeds.%d.ModuleID", index, i)]
+			fOpts["module"] = safeParentIdentifier(tt, n, modRef)
+			delete(fOpts, "moduleID")
+		}
+		break
+
+	case "Automation":
+		bb, _ := b.Options["buttons"].([]interface{})
+		for i, b := range bb {
+			button, _ := b.(map[string]interface{})
+			if _, has := button["workflowID"]; !has {
+				continue
+			}
+
+			wfRef := n.References[fmt.Sprintf("Blocks.%d.Options.buttons.%d.WorkflowID", index, i)]
+			button["workflow"] = safeParentIdentifier(tt, n, wfRef)
+			delete(button, "workflowID")
+			i++
+		}
+		break
+
+	case "Metric":
+		mm, _ := b.Options["metrics"].([]interface{})
+		for i, m := range mm {
+			modRef := n.References[fmt.Sprintf("Blocks.%d.Options.metrics.%d.ModuleID", index, i)]
+
+			mops, _ := m.(map[string]interface{})
+			mops["module"] = safeParentIdentifier(tt, n, modRef)
+			delete(mops, "moduleID")
+		}
+		break
+
+	case "Comment":
+		modRef := n.References[fmt.Sprintf("Blocks.%d.Options.ModuleID", index)]
+		b.Options["module"] = safeParentIdentifier(tt, n, modRef)
+		delete(b.Options, "moduleID")
+		break
+
+	case "Progress":
+		err = e.encodeProgressPageblockVal("minValue", index, n, tt, &b)
+		if err != nil {
+			return
+		}
+
+		err = e.encodeProgressPageblockVal("maxValue", index, n, tt, &b)
+		if err != nil {
+			return
+		}
+
+		err = e.encodeProgressPageblockVal("value", index, n, tt, &b)
+		if err != nil {
+			return
+		}
+		break
+	}
+
+	return b, nil
+}
+
+func (e YamlEncoder) encodeProgressPageblockVal(k string, index int, n *envoyx.Node, tt envoyx.Traverser, b *types.PageBlock) (err error) {
+	if reflect2.IsNil(b.Options[k]) {
+		return
+	}
+
+	modRef := n.References[fmt.Sprintf("Blocks.%d.Options.%s.ModuleID", index, k)]
+	opt := b.Options[k].(map[string]any)
+	opt["moduleID"] = safeParentIdentifier(tt, n, modRef)
+	delete(opt, "moduleID")
+
+	return
+}
+
+func (e YamlEncoder) cleanupPageblockRecordList(b types.PageBlock) (out types.PageBlock) {
+	out = b
+	rawFF, has := out.Options["fields"]
+	if !has {
+		return
+	}
+
+	ff, ok := rawFF.([]interface{})
+	if !ok {
+		return
+	}
+
+	retFF := make([]interface{}, 0, len(ff))
+	for _, rawF := range ff {
+		switch c := rawF.(type) {
+		case string:
+			retFF = append(retFF, map[string]interface{}{"name": c})
+		case map[string]interface{}, map[string]string:
+			retFF = append(retFF, c)
+		default:
+			retFF = append(retFF, rawF)
+		}
+	}
+
+	out.Options["fields"] = retFF
+
+	return b
+}