snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/pkg/dal/aggregator.go

@@ -0,0 +1,359 @@
+package dal
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/ql"
+	"github.com/modern-go/reflect2"
+	"github.com/spf13/cast"
+)
+
+type (
+	// aggregator performs value aggregation primarily used for the aggregate step
+	// @todo consider moving into a separate package
+	//
+	// The aggregator performs the requested value expressions as well as
+	// aggregation operations over the evaluated values.
+	//
+	// The aggregator computes the values on the fly (for the ones it can).
+	aggregator struct {
+		// aggregates holds output aggregation values
+		// @note we'll use float64 for all values, but it might make more sense to split it up
+		//       in the future. For now, it'll be ok.
+		aggregates []float64
+
+		// counts holds the number of values for each aggregate including multi value fields.
+		// Counts are currently only used for average.
+		counts []int
+
+		// def provides a list of aggregates.
+		//
+		// Each index corresponds to the aggregates and counts slices
+		def []aggregateDef
+		// scanned indicates whether the aggregator has been scanned since
+		// we need to block writes after the first scan.
+		scanned bool
+	}
+
+	// aggregateDef is a wrapper for the provided value aggregation
+	aggregateDef struct {
+		outIdent string
+
+		aggOp string
+
+		inIdent string
+		eval    evaluator
+	}
+)
+
+var (
+	// aggregateFunctionIndex specifies all of the registered aggregate functions
+	//
+	// @todo consider making this expandable via some registry/plugin/...
+	aggregateFunctionIndex = map[string]bool{
+		"count": true,
+		"sum":   true,
+		"min":   true,
+		"max":   true,
+		"avg":   true,
+	}
+)
+
+// Aggregator initializes a new aggregator for the given set of mappings
+//
+// The aggregator is not routine safe; consider defining multiple aggregators
+// and then combining them together.
+func Aggregator() *aggregator {
+	return &aggregator{
+		aggregates: make([]float64, 0, 16),
+		counts:     make([]int, 0, 16),
+	}
+}
+
+// AddAggregateE adds a new aggregate from a raw expression
+func (a *aggregator) AddAggregateE(ident, expr string) (err error) {
+	// @note the converter is reused so we can safely do this
+	n, err := newConverterGval().Parse(expr)
+	if err != nil {
+		return
+	}
+
+	return a.AddAggregate(ident, n)
+}
+
+// AddAggregate adds a new aggregate from an already parsed expression
+func (a *aggregator) AddAggregate(ident string, expr *ql.ASTNode) (err error) {
+	def := aggregateDef{
+		outIdent: ident,
+	}
+
+	inIdent, expr, err := unpackMappingSource(expr)
+	if err != nil {
+		return
+	}
+
+	// Take it from the source
+	if inIdent != "" {
+		def.inIdent = inIdent
+	}
+
+	// Take it from the expression
+	// - agg. op.
+	def.aggOp, expr, err = unpackExpressionNode(expr)
+	if err != nil {
+		return
+	}
+	// Prepare a runner in case we're not simply copying values
+	if inIdent == "" {
+		// - make evaluator
+		if expr != nil {
+			def.eval, err = newRunnerGvalParsed(expr)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	a.aggregates = append(a.aggregates, 0)
+	a.counts = append(a.counts, 0)
+	a.def = append(a.def, def)
+	return
+}
+
+// Aggregate aggregates the given value
+func (a *aggregator) Aggregate(ctx context.Context, v ValueGetter) (err error) {
+	if a.scanned {
+		// If we attempt to add data to the aggregator, the previous data might
+		// no longer be valid.
+		return fmt.Errorf("cannot call Aggregate on an already scanned aggregator")
+	}
+
+	// @todo consider throwing all of the aggregates into a routine.
+	//       I think (for now) performance gains will be negligible.
+	for i, attr := range a.def {
+		err = a.aggregate(ctx, attr, i, v)
+		if err != nil {
+			return
+		}
+	}
+
+	return nil
+}
+
+// Scan scans the aggregated values into the setter
+func (a *aggregator) Scan(s ValueSetter) (err error) {
+	// On first scan, complete partial aggregates
+	if !a.scanned {
+		a.completePartials()
+	}
+
+	a.scanned = true
+
+	// Set the values
+	for i, attr := range a.def {
+		// @note each aggregated value can be at most one so no need for multi-value
+		//       suport here.
+		err = s.SetValue(attr.outIdent, 0, a.aggregates[i])
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// aggregate applies the provided value into the requested aggregate
+func (a *aggregator) aggregate(ctx context.Context, attr aggregateDef, i int, v ValueGetter) (err error) {
+	switch attr.aggOp {
+	case "count":
+		return a.count(ctx, attr, i, v)
+
+	case "sum":
+		return a.sum(ctx, attr, i, v)
+
+	case "min":
+		return a.min(ctx, attr, i, v)
+
+	case "max":
+		return a.max(ctx, attr, i, v)
+
+	case "avg":
+		return a.avg(ctx, attr, i, v)
+	}
+
+	return fmt.Errorf("unsupported aggregate function: %s", attr.aggOp)
+}
+
+// walkValues traverses the available values for the specified attribute
+func (a *aggregator) walkValues(ctx context.Context, r ValueGetter, cc map[string]uint, attr aggregateDef, run func(v any, isNil bool)) (err error) {
+	var out any
+	if attr.inIdent == "" {
+		if attr.eval != nil {
+			out, err = attr.eval.Eval(ctx, r)
+			if err != nil {
+				return
+			}
+		} else {
+			out = r
+		}
+
+		run(out, reflect2.IsNil(out))
+		return nil
+	}
+
+	for i := uint(0); i < cc[attr.inIdent]; i++ {
+		v, err := r.GetValue(attr.inIdent, i)
+		if err != nil {
+			return err
+		}
+		run(v, reflect2.IsNil(v))
+	}
+
+	return nil
+}
+
+// Aggregate methods
+
+func (a *aggregator) count(ctx context.Context, attr aggregateDef, i int, v ValueGetter) (err error) {
+	err = a.walkValues(ctx, v, v.CountValues(), attr, func(v any, isNil bool) {
+		if isNil {
+			return
+		}
+
+		a.aggregates[i]++
+		a.counts[i]++
+	})
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (a *aggregator) sum(ctx context.Context, attr aggregateDef, i int, v ValueGetter) (err error) {
+	err = a.walkValues(ctx, v, v.CountValues(), attr, func(v any, isNil bool) {
+		if isNil {
+			return
+		}
+		a.aggregates[i] += cast.ToFloat64(v)
+		a.counts[i]++
+	})
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (a *aggregator) min(ctx context.Context, attr aggregateDef, i int, v ValueGetter) (err error) {
+	err = a.walkValues(ctx, v, v.CountValues(), attr, func(v any, isNil bool) {
+		if isNil {
+			return
+		}
+
+		if a.counts[i] == 0 {
+			a.aggregates[i] = cast.ToFloat64(v)
+		} else {
+			a.aggregates[i] = math.Min(a.aggregates[i], cast.ToFloat64(v))
+		}
+		a.counts[i]++
+	})
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (a *aggregator) max(ctx context.Context, attr aggregateDef, i int, v ValueGetter) (err error) {
+	err = a.walkValues(ctx, v, v.CountValues(), attr, func(v any, isNil bool) {
+		if isNil {
+			return
+		}
+
+		if a.counts[i] == 0 {
+			a.aggregates[i] = cast.ToFloat64(v)
+		} else {
+			a.aggregates[i] = math.Max(a.aggregates[i], cast.ToFloat64(v))
+		}
+		a.counts[i]++
+	})
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (a *aggregator) avg(ctx context.Context, attr aggregateDef, i int, v ValueGetter) (err error) {
+	err = a.walkValues(ctx, v, v.CountValues(), attr, func(v any, isNil bool) {
+		if isNil {
+			return
+		}
+
+		a.aggregates[i] += cast.ToFloat64(v)
+
+		a.counts[i]++
+	})
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (a *aggregator) completePartials() {
+	a.completeAverage()
+}
+
+func (a *aggregator) completeAverage() {
+	for i, attr := range a.def {
+		if attr.aggOp == "avg" {
+			if a.counts[i] == 0 {
+				return
+			}
+			a.aggregates[i] = a.aggregates[i] / float64(a.counts[i])
+		}
+	}
+}
+
+// Utilities
+
+func unpackMappingSource(n *ql.ASTNode) (ident string, expr *ql.ASTNode, err error) {
+	// Check if first arg of agg. fnc. is an attr.
+	if len(n.Args) == 1 && n.Args[0].Symbol != "" {
+		return n.Args[0].Symbol, n, nil
+	}
+
+	expr = n
+	return
+}
+
+func unpackExpressionNode(n *ql.ASTNode) (aggOp string, expr *ql.ASTNode, err error) {
+	if n.Ref != "" {
+		aggOp = strings.ToLower(n.Ref)
+	}
+	if !aggregateFunctionIndex[aggOp] {
+		err = fmt.Errorf("root expression must be an aggregate function")
+		return
+	}
+
+	if len(n.Args) > 0 {
+		expr = n.Args[0]
+	}
+	return
+}
+
+// reset is a benchmarking utility to reset the aggregator
+//
+// Don't use it in production code.
+func (a *aggregator) reset() {
+	for i := 0; i < len(a.aggregates); i++ {
+		a.aggregates[i] = 0
+		a.counts[i] = 0
+	}
+	a.scanned = false
+}

server/pkg/dal/aggregator_test.go

@@ -0,0 +1,191 @@
+package dal
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestAggregator(t *testing.T) {
+	tcc := []struct {
+		name string
+
+		rows       []simpleRow
+		attrubutes []simpleAttribute
+		out        simpleRow
+	}{
+		// Plain operations
+		{
+			name: "simple count",
+			rows: []simpleRow{
+				{"v": 1},
+				{"v": 5},
+				{"v": 35},
+				{"v": 11.5},
+			},
+			out: simpleRow{"count": float64(4)},
+			attrubutes: []simpleAttribute{
+				{ident: "count", expr: "count(v)"},
+			},
+		},
+		{
+			name: "simple count nulls",
+			rows: []simpleRow{
+				{"v": nil},
+				{"v": 5},
+				{"v": 35},
+				{"v": 11.5},
+			},
+			out: simpleRow{"count": float64(3)},
+			attrubutes: []simpleAttribute{
+				{ident: "count", expr: "count(v)"},
+			},
+		},
+		{
+			name: "simple sum",
+			rows: []simpleRow{
+				{"v": 1},
+				{"v": 5},
+				{"v": 35},
+				{"v": 11.5},
+			},
+			out: simpleRow{"sum": float64(52.5)},
+			attrubutes: []simpleAttribute{
+				{ident: "sum", expr: "sum(v)"},
+			},
+		},
+		{
+			name: "simple sum nulls",
+			rows: []simpleRow{
+				{"v": nil},
+				{"v": 5},
+				{"v": 35},
+				{"v": 11.5},
+			},
+			out: simpleRow{"sum": float64(51.5)},
+			attrubutes: []simpleAttribute{
+				{ident: "sum", expr: "sum(v)"},
+			},
+		},
+		{
+			name: "simple min",
+			rows: []simpleRow{
+				{"v": 1},
+				{"v": 5},
+				{"v": 35},
+				{"v": 11.5},
+			},
+			out: simpleRow{"min": float64(1)},
+			attrubutes: []simpleAttribute{
+				{ident: "min", expr: "min(v)"},
+			},
+		},
+		{
+			name: "simple min nulls",
+			rows: []simpleRow{
+				{"v": nil},
+				{"v": 5},
+				{"v": 35},
+				{"v": 11.5},
+			},
+			out: simpleRow{"min": float64(5)},
+			attrubutes: []simpleAttribute{
+				{ident: "min", expr: "min(v)"},
+			},
+		},
+		{
+			name: "simple max",
+			rows: []simpleRow{
+				{"v": 1},
+				{"v": 5},
+				{"v": 35},
+				{"v": 11.5},
+			},
+			out: simpleRow{"max": float64(35)},
+			attrubutes: []simpleAttribute{
+				{ident: "max", expr: "max(v)"},
+			},
+		},
+		{
+			name: "simple max nulls",
+			rows: []simpleRow{
+				{"v": 1},
+				{"v": 5},
+				{"v": nil},
+				{"v": 11.5},
+			},
+			out: simpleRow{"max": float64(11.5)},
+			attrubutes: []simpleAttribute{
+				{ident: "max", expr: "max(v)"},
+			},
+		},
+		{
+			name: "simple avg",
+			rows: []simpleRow{
+				{"v": 1},
+				{"v": 5},
+				{"v": 35},
+				{"v": 11.5},
+			},
+			out: simpleRow{"avg": float64(13.125)},
+			attrubutes: []simpleAttribute{
+				{ident: "avg", expr: "avg(v)"},
+			},
+		},
+		{
+			name: "simple avg nulls",
+			rows: []simpleRow{
+				{"v": nil},
+				{"v": 5},
+				{"v": 35},
+				{"v": nil},
+			},
+			out: simpleRow{"avg": float64(20)},
+			attrubutes: []simpleAttribute{
+				{ident: "avg", expr: "avg(v)"},
+			},
+		},
+
+		// With a nested expression
+		// @todo tests to assure nil values; omitting due to the gval issue
+		{
+			name: "nested expression",
+			rows: []simpleRow{
+				{"v": 1},
+				{"v": 5},
+				{"v": 35},
+				{"v": 11.5},
+			},
+			out: simpleRow{"sum": float64(60.5)},
+			attrubutes: []simpleAttribute{
+				{ident: "sum", expr: "sum(v + 2)"},
+			},
+		},
+	}
+
+	ctx := context.Background()
+	for _, tc := range tcc {
+		t.Run(tc.name, func(t *testing.T) {
+			agg := Aggregator()
+			for _, a := range tc.attrubutes {
+				require.NoError(t, agg.AddAggregateE(a.ident, a.expr))
+			}
+
+			for _, r := range tc.rows {
+				require.NoError(t, agg.Aggregate(ctx, r))
+			}
+
+			out := make(simpleRow)
+			require.NoError(t, agg.Scan(out))
+			require.Equal(t, tc.out, out)
+		})
+	}
+}
+
+func TestAggregatorInit(t *testing.T) {
+	t.Run("non supported agg. op.", func(t *testing.T) {
+		agg := Aggregator()
+		require.Error(t, agg.AddAggregateE("count", "div(v)"))
+	})
+}

server/pkg/dal/alteration.go

@@ -0,0 +1,370 @@
+package dal
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+type (
+	Alteration struct {
+		ID           uint64
+		BatchID      uint64
+		DependsOn    uint64
+		Resource     string
+		ResourceType string
+		ConnectionID uint64
+
+		AttributeAdd      *AttributeAdd
+		AttributeDelete   *AttributeDelete
+		AttributeReType   *AttributeReType
+		AttributeReEncode *AttributeReEncode
+		ModelAdd          *ModelAdd
+		ModelDelete       *ModelDelete
+	}
+
+	AlterationSet []*Alteration
+
+	AttributeAdd struct {
+		Attr *Attribute `json:"attr"`
+	}
+
+	AttributeDelete struct {
+		Attr *Attribute `json:"attr"`
+	}
+
+	AttributeReType struct {
+		Attr *Attribute `json:"attr"`
+		To   Type       `json:"to"`
+	}
+
+	AttributeReEncode struct {
+		Attr *Attribute `json:"attr"`
+		To   Codec      `json:"to"`
+	}
+
+	ModelAdd struct {
+		Model *Model `json:"model"`
+	}
+
+	ModelDelete struct {
+		Model *Model `json:"model"`
+	}
+
+	// auxAttributeType is a helper struct used for marshaling/unmarshaling
+	//
+	// This is required since the Type inside the Attribute is an interface and we
+	// need to help the encoding/json a bit.
+	auxAttributeReType struct {
+		Attr *Attribute        `json:"attr"`
+		To   *auxAttributeType `json:"to"`
+	}
+
+	// auxAttributeReEncode is a helper struct used for marshaling/unmarshaling
+	//
+	// This is required since the Codec inside the Attribute is an interface and we
+	// need to help the encoding/json a bit.
+	auxAttributeReEncode struct {
+		Attr *Attribute     `json:"attr"`
+		To   *auxStoreCodec `json:"to"`
+	}
+)
+
+// Merge merges the two alteration slices
+func (aa AlterationSet) Merge(bb AlterationSet) (cc AlterationSet) {
+	// @todo the Merge function currently just merges the two slices together
+	//       and removes any duplicates that would occur due to the merge.
+	//       We should also handle overlapping/transitive alterations to reduce
+	//       the amount of needles processing.
+	//
+	// A quick list of overlapping alterations:
+	// * attribute A added and then renamed from A to A'
+	// * attribute A renamed to A' and then renamed to A''
+	// * attribute A deleted and then created
+
+	aux := append(aa, bb...)
+	seen := make(map[int]bool, len(aux)/2)
+
+	for i, a := range aux {
+		if seen[i] {
+			continue
+		}
+
+		found := false
+		for j := i + 1; j < len(aux); j++ {
+			if seen[j] {
+				continue
+			}
+
+			if a.compare(*aux[j]) {
+				if !found {
+					cc = append(cc, aux[j])
+				}
+
+				seen[j] = true
+				found = true
+			}
+		}
+
+		if !found {
+			cc = append(cc, a)
+		}
+	}
+
+	return
+}
+
+func (a AttributeReType) MarshalJSON() ([]byte, error) {
+	aux := auxAttributeReType{
+		Attr: a.Attr,
+		To:   &auxAttributeType{},
+	}
+
+	switch t := a.To.(type) {
+	case *TypeID:
+		aux.To.Type = "ID"
+		aux.To.ID = t
+
+	case *TypeRef:
+		aux.To.Type = "Ref"
+		aux.To.Ref = t
+
+	case *TypeTimestamp:
+		aux.To.Type = "Timestamp"
+		aux.To.Timestamp = t
+
+	case *TypeTime:
+		aux.To.Type = "Time"
+		aux.To.Time = t
+
+	case *TypeDate:
+		aux.To.Type = "Date"
+		aux.To.Date = t
+
+	case *TypeNumber:
+		aux.To.Type = "Number"
+		aux.To.Number = t
+
+	case *TypeText:
+		aux.To.Type = "Text"
+		aux.To.Text = t
+
+	case *TypeBoolean:
+		aux.To.Type = "Boolean"
+		aux.To.Boolean = t
+
+	case *TypeEnum:
+		aux.To.Type = "Enum"
+		aux.To.Enum = t
+
+	case *TypeGeometry:
+		aux.To.Type = "Geometry"
+		aux.To.Geometry = t
+
+	case *TypeJSON:
+		aux.To.Type = "JSON"
+		aux.To.JSON = t
+
+	case *TypeBlob:
+		aux.To.Type = "Blob"
+		aux.To.Blob = t
+
+	case *TypeUUID:
+		aux.To.Type = "UUID"
+		aux.To.UUID = t
+	}
+
+	return json.Marshal(aux)
+}
+
+func (a *AttributeReType) UnmarshalJSON(data []byte) (err error) {
+	aux := &auxAttributeReType{}
+	err = json.Unmarshal(data, &aux)
+	if err != nil {
+		return err
+	}
+
+	if a == nil {
+		*a = AttributeReType{}
+	}
+
+	a.Attr = aux.Attr
+
+	switch aux.To.Type {
+	case "ID":
+		a.To = aux.To.ID
+
+	case "Ref":
+		a.To = aux.To.Ref
+
+	case "Timestamp":
+		a.To = aux.To.Timestamp
+
+	case "Time":
+		a.To = aux.To.Time
+
+	case "Date":
+		a.To = aux.To.Date
+
+	case "Number":
+		a.To = aux.To.Number
+
+	case "Text":
+		a.To = aux.To.Text
+
+	case "Boolean":
+		a.To = aux.To.Boolean
+
+	case "Enum":
+		a.To = aux.To.Enum
+
+	case "Geometry":
+		a.To = aux.To.Geometry
+
+	case "JSON":
+		a.To = aux.To.JSON
+
+	case "Blob":
+		a.To = aux.To.Blob
+
+	case "UUID":
+		a.To = aux.To.UUID
+	}
+
+	return
+}
+
+func (a AttributeReEncode) MarshalJSON() ([]byte, error) {
+	aux := auxAttributeReEncode{
+		Attr: a.Attr,
+		To:   &auxStoreCodec{},
+	}
+
+	switch t := a.To.(type) {
+	case *CodecPlain:
+		aux.To.Type = "CodecPlain"
+		aux.To.CodecPlain = t
+	case *CodecRecordValueSetJSON:
+		aux.To.Type = "CodecRecordValueSetJSON"
+		aux.To.CodecRecordValueSetJSON = t
+	case *CodecAlias:
+		aux.To.Type = "CodecAlias"
+		aux.To.CodecAlias = t
+	}
+
+	return json.Marshal(aux)
+}
+
+func (a *AttributeReEncode) UnmarshalJSON(data []byte) (err error) {
+	aux := &auxAttributeReEncode{}
+	err = json.Unmarshal(data, &aux)
+	if err != nil {
+		return err
+	}
+
+	if a == nil {
+		*a = AttributeReEncode{}
+	}
+
+	a.Attr = aux.Attr
+
+	switch aux.To.Type {
+	case "CodecPlain":
+		a.To = aux.To.CodecPlain
+
+	case "CodecRecordValueSetJSON":
+		a.To = aux.To.CodecRecordValueSetJSON
+
+	case "CodecAlias":
+		a.To = aux.To.CodecAlias
+	}
+
+	return
+}
+
+func (a Alteration) compare(b Alteration) (cmp bool) {
+	if a.AttributeAdd == nil && b.AttributeAdd != nil {
+		return false
+	}
+	if a.AttributeDelete == nil && b.AttributeDelete != nil {
+		return false
+	}
+	if a.AttributeReType == nil && b.AttributeReType != nil {
+		return false
+	}
+	if a.AttributeReEncode == nil && b.AttributeReEncode != nil {
+		return false
+	}
+	if a.ModelAdd == nil && b.ModelAdd != nil {
+		return false
+	}
+	if a.ModelDelete == nil && b.ModelDelete != nil {
+		return false
+	}
+
+	switch {
+	case a.AttributeAdd != nil:
+		return a.compareAttributeAdd(b)
+	case a.AttributeDelete != nil:
+		return a.compareAttributeDelete(b)
+	case a.AttributeReType != nil:
+		return a.compareAttributeReType(b)
+	case a.AttributeReEncode != nil:
+		return a.compareAttributeReEncode(b)
+	case a.ModelAdd != nil:
+		return a.compareModelAdd(b)
+	case a.ModelDelete != nil:
+		return a.compareModelDelete(b)
+	}
+
+	panic(fmt.Sprintf("unsupported alteration type %v", a))
+}
+
+func (a Alteration) compareAttributeAdd(b Alteration) bool {
+	if a.AttributeAdd == nil || b.AttributeAdd == nil {
+		return a.AttributeAdd == b.AttributeAdd
+	}
+	return a.AttributeAdd.Attr.Compare(b.AttributeAdd.Attr)
+}
+
+func (a Alteration) compareAttributeDelete(b Alteration) bool {
+	if a.AttributeDelete == nil || b.AttributeDelete == nil {
+		return a.AttributeDelete == b.AttributeDelete
+	}
+	return a.AttributeDelete.Attr.Compare(b.AttributeDelete.Attr)
+}
+
+func (a Alteration) compareAttributeReType(b Alteration) bool {
+	if !a.AttributeReType.Attr.Compare(b.AttributeReType.Attr) {
+		return false
+	}
+
+	if a.AttributeReType == nil || b.AttributeReType == nil {
+		return a.AttributeReType == b.AttributeReType
+	}
+	return a.AttributeReType.To.Type() == b.AttributeReType.To.Type()
+}
+
+func (a Alteration) compareAttributeReEncode(b Alteration) bool {
+	if !a.AttributeReEncode.Attr.Compare(b.AttributeReEncode.Attr) {
+		return false
+	}
+
+	if a.AttributeReEncode == nil || b.AttributeReEncode == nil {
+		return a.AttributeReEncode == b.AttributeReEncode
+	}
+	return a.AttributeReEncode.To.Type() == b.AttributeReEncode.To.Type()
+}
+
+func (a Alteration) compareModelAdd(b Alteration) bool {
+	if a.ModelAdd == nil || b.ModelAdd == nil {
+		return a.ModelAdd == b.ModelAdd
+	}
+	return a.ModelAdd.Model.Compare(*b.ModelAdd.Model)
+}
+
+func (a Alteration) compareModelDelete(b Alteration) bool {
+	if a.ModelDelete == nil || b.ModelDelete == nil {
+		return a.ModelDelete == b.ModelDelete
+	}
+	return a.ModelDelete.Model.Compare(*b.ModelDelete.Model)
+}

server/pkg/dal/alteration_test.go

@@ -0,0 +1,600 @@
+package dal
+
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMerge(t *testing.T) {
+	tcc := []struct {
+		name string
+		aa   AlterationSet
+		bb   AlterationSet
+		cc   AlterationSet
+	}{
+		{
+			name: "empty",
+			aa:   AlterationSet{},
+			bb:   AlterationSet{},
+			cc:   AlterationSet{},
+		},
+		{
+			name: "aa empty",
+			aa:   AlterationSet{},
+			bb: AlterationSet{
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+			cc: AlterationSet{
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "bb empty",
+			aa: AlterationSet{
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+			bb: AlterationSet{},
+			cc: AlterationSet{
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "remove duplicates",
+			aa:   AlterationSet{},
+			bb: AlterationSet{
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+			cc: AlterationSet{
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "un matching types",
+			aa: AlterationSet{
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+			bb: AlterationSet{
+				&Alteration{
+					AttributeDelete: &AttributeDelete{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+			cc: AlterationSet{
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+				&Alteration{
+					AttributeDelete: &AttributeDelete{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "match AttributeAdd",
+			aa: AlterationSet{
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+			bb: AlterationSet{
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+			cc: AlterationSet{
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "not match AttributeAdd",
+			aa: AlterationSet{
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+			bb: AlterationSet{
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "bar",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+			cc: AlterationSet{
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+				&Alteration{
+					AttributeAdd: &AttributeAdd{
+						Attr: &Attribute{
+							Ident: "bar",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+		},
+
+		{
+			name: "match AttributeDelete",
+			aa: AlterationSet{
+				&Alteration{
+					AttributeDelete: &AttributeDelete{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+			bb: AlterationSet{
+				&Alteration{
+					AttributeDelete: &AttributeDelete{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+			cc: AlterationSet{
+				&Alteration{
+					AttributeDelete: &AttributeDelete{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "not match AttributeDelete",
+			aa: AlterationSet{
+				&Alteration{
+					AttributeDelete: &AttributeDelete{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+			bb: AlterationSet{
+				&Alteration{
+					AttributeDelete: &AttributeDelete{
+						Attr: &Attribute{
+							Ident: "bar",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+			cc: AlterationSet{
+				&Alteration{
+					AttributeDelete: &AttributeDelete{
+						Attr: &Attribute{
+							Ident: "foo",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+				&Alteration{
+					AttributeDelete: &AttributeDelete{
+						Attr: &Attribute{
+							Ident: "bar",
+							Type:  &TypeJSON{Nullable: false},
+						},
+					},
+				},
+			},
+		},
+
+		{
+			name: "match AttributeReType",
+			aa: AlterationSet{
+				&Alteration{
+					AttributeReType: &AttributeReType{
+						Attr: &Attribute{
+							Ident: "foo",
+						},
+						To: &TypeBoolean{Nullable: false},
+					},
+				},
+			},
+			bb: AlterationSet{
+				&Alteration{
+					AttributeReType: &AttributeReType{
+						Attr: &Attribute{
+							Ident: "foo",
+						},
+						To: &TypeBoolean{Nullable: false},
+					},
+				},
+			},
+			cc: AlterationSet{
+				&Alteration{
+					AttributeReType: &AttributeReType{
+						Attr: &Attribute{
+							Ident: "foo",
+						},
+						To: &TypeBoolean{Nullable: false},
+					},
+				},
+			},
+		},
+		{
+			name: "not match AttributeReType",
+			aa: AlterationSet{
+				&Alteration{
+					AttributeReType: &AttributeReType{
+						Attr: &Attribute{
+							Ident: "foo",
+						},
+						To: &TypeBoolean{Nullable: false},
+					},
+				},
+			},
+			bb: AlterationSet{
+				&Alteration{
+					AttributeReType: &AttributeReType{
+						Attr: &Attribute{
+							Ident: "foo",
+						},
+						To: &TypeText{Nullable: false},
+					},
+				},
+			},
+			cc: AlterationSet{
+				&Alteration{
+					AttributeReType: &AttributeReType{
+						Attr: &Attribute{
+							Ident: "foo",
+						},
+						To: &TypeBoolean{Nullable: false},
+					},
+				},
+				&Alteration{
+					AttributeReType: &AttributeReType{
+						Attr: &Attribute{
+							Ident: "foo",
+						},
+						To: &TypeText{Nullable: false},
+					},
+				},
+			},
+		},
+
+		{
+			name: "match AttributeReEncode",
+			aa: AlterationSet{
+				&Alteration{
+					AttributeReEncode: &AttributeReEncode{
+						Attr: &Attribute{Ident: "foo"},
+						To:   &CodecPlain{},
+					},
+				},
+			},
+			bb: AlterationSet{
+				&Alteration{
+					AttributeReEncode: &AttributeReEncode{
+						Attr: &Attribute{Ident: "foo"},
+						To:   &CodecPlain{},
+					},
+				},
+			},
+			cc: AlterationSet{
+				&Alteration{
+					AttributeReEncode: &AttributeReEncode{
+						Attr: &Attribute{Ident: "foo"},
+						To:   &CodecPlain{},
+					},
+				},
+			},
+		},
+		{
+			name: "not match AttributeReEncode",
+			aa: AlterationSet{
+				&Alteration{
+					AttributeReEncode: &AttributeReEncode{
+						Attr: &Attribute{Ident: "foo"},
+						To:   &CodecPlain{},
+					},
+				},
+			},
+			bb: AlterationSet{
+				&Alteration{
+					AttributeReEncode: &AttributeReEncode{
+						Attr: &Attribute{Ident: "foo"},
+						To:   &CodecAlias{Ident: "foo2"},
+					},
+				},
+			},
+			cc: AlterationSet{
+				&Alteration{
+					AttributeReEncode: &AttributeReEncode{
+						Attr: &Attribute{Ident: "foo"},
+						To:   &CodecPlain{},
+					},
+				},
+				&Alteration{
+					AttributeReEncode: &AttributeReEncode{
+						Attr: &Attribute{Ident: "foo"},
+						To:   &CodecAlias{Ident: "foo2"},
+					},
+				},
+			},
+		},
+
+		{
+			name: "match ModelAdd",
+			aa: AlterationSet{
+				&Alteration{
+					ModelAdd: &ModelAdd{
+						Model: &Model{Ident: "foo"},
+					},
+				},
+			},
+			bb: AlterationSet{
+				&Alteration{
+					ModelAdd: &ModelAdd{
+						Model: &Model{Ident: "foo"},
+					},
+				},
+			},
+			cc: AlterationSet{
+				&Alteration{
+					ModelAdd: &ModelAdd{
+						Model: &Model{Ident: "foo"},
+					},
+				},
+			},
+		},
+		{
+			name: "not match ModelAdd",
+			aa: AlterationSet{
+				&Alteration{
+					ModelAdd: &ModelAdd{
+						Model: &Model{Ident: "foo"},
+					},
+				},
+			},
+			bb: AlterationSet{
+				&Alteration{
+					ModelAdd: &ModelAdd{
+						Model: &Model{Ident: "bar"},
+					},
+				},
+			},
+			cc: AlterationSet{
+				&Alteration{
+					ModelAdd: &ModelAdd{
+						Model: &Model{Ident: "foo"},
+					},
+				},
+				&Alteration{
+					ModelAdd: &ModelAdd{
+						Model: &Model{Ident: "bar"},
+					},
+				},
+			},
+		},
+
+		{
+			name: "match ModelDelete",
+			aa: AlterationSet{
+				&Alteration{
+					ModelDelete: &ModelDelete{
+						Model: &Model{Ident: "foo"},
+					},
+				},
+			},
+			bb: AlterationSet{
+				&Alteration{
+					ModelDelete: &ModelDelete{
+						Model: &Model{Ident: "foo"},
+					},
+				},
+			},
+			cc: AlterationSet{
+				&Alteration{
+					ModelDelete: &ModelDelete{
+						Model: &Model{Ident: "foo"},
+					},
+				},
+			},
+		},
+		{
+			name: "not match ModelDelete",
+			aa: AlterationSet{
+				&Alteration{
+					ModelDelete: &ModelDelete{
+						Model: &Model{Ident: "foo"},
+					},
+				},
+			},
+			bb: AlterationSet{
+				&Alteration{
+					ModelDelete: &ModelDelete{
+						Model: &Model{Ident: "bar"},
+					},
+				},
+			},
+			cc: AlterationSet{
+				&Alteration{
+					ModelDelete: &ModelDelete{
+						Model: &Model{Ident: "foo"},
+					},
+				},
+				&Alteration{
+					ModelDelete: &ModelDelete{
+						Model: &Model{Ident: "bar"},
+					},
+				},
+			},
+		},
+	}
+
+	req := require.New(t)
+
+	for _, tc := range tcc {
+		t.Run(tc.name, func(t *testing.T) {
+			cc := tc.aa.Merge(tc.bb)
+			req.Len(cc, len(tc.cc))
+			for i, c := range cc {
+				cmp := c.compare(*tc.cc[i])
+				req.True(cmp)
+			}
+		})
+	}
+}
+
+func TestReTypeMarshling(t *testing.T) {
+	a := &AttributeReType{
+		Attr: &Attribute{
+			Ident: "foo",
+			Type:  &TypeBlob{},
+			Store: &CodecPlain{},
+		},
+		To: &TypeBoolean{},
+	}
+
+	bb, err := json.Marshal(a)
+	require.NoError(t, err)
+
+	b := &AttributeReType{}
+	err = json.Unmarshal(bb, &b)
+	require.NoError(t, err)
+
+	require.True(t, reflect.DeepEqual(a, b))
+}
+
+func TestReEncodeMarshling(t *testing.T) {
+	a := &AttributeReEncode{
+		Attr: &Attribute{
+			Ident: "foo",
+			Type:  &TypeBlob{},
+			Store: &CodecPlain{},
+		},
+		To: &CodecAlias{Ident: "bar"},
+	}
+
+	bb, err := json.Marshal(a)
+	require.NoError(t, err)
+
+	b := &AttributeReEncode{}
+	err = json.Unmarshal(bb, &b)
+	require.NoError(t, err)
+
+	require.True(t, reflect.DeepEqual(a, b))
+}

server/pkg/dal/attribute_codec.go

@@ -0,0 +1,69 @@
+package dal
+
+type (
+	AttributeCodecType string
+
+	// Codec defines how values for a specific model attribute
+	// are retrieved or stored
+	//
+	// If attribute does not have store strategy set
+	// store driver should use attribute name to determinate
+	// source/destination of the value (table column, json document property)
+	Codec interface {
+		Type() AttributeCodecType
+		SingleValueOnly() bool
+	}
+
+	CodecPlain struct{}
+
+	// CodecRecordValueSetJSON defines that values are encoded/decoded into
+	// a JSON simple document { [_: string]: Array<unknown> }
+	//
+	// This only solves
+	// Attribute{Ident: "foo", Store: StoreCodecRecordValueJSON{ Ident: "bar" }
+	// => "bar"->'foo'->0
+	CodecRecordValueSetJSON struct {
+		Ident string
+	}
+
+	// handling complex JSON documents
+	//StoreCodecJSON struct {
+	//	Ident string
+	//	Path  []any
+	//}
+	//
+	// { "@value": ... "@type": .... }
+	// StoreCodecJSONLD struct { Ident  string; Path   []any }
+
+	// StoreCodecXML
+	//StoreCodecXML struct {}
+
+	// CodecAlias defines case when value is not stored
+	// under the same column (in case of an SQL table)
+	//
+	// Value of CodecAlias.Ident is used as base
+	// and value of Attribute.Ident holding CodecAlias is used
+	// as an alias!
+	//
+	// Attribute{Ident: "foo", Store: CodecAlias{ Ident: "bar" }
+	// => "bar" as "foo"
+	CodecAlias struct {
+		Ident string
+	}
+)
+
+const (
+	AttributeCodecPlain              AttributeCodecType = "corteza::dal:attribute-codec:plain"
+	AttributeCodecRecordValueSetJSON AttributeCodecType = "corteza::dal:attribute-codec:record-value-set-json"
+	AttributeCodecAlias              AttributeCodecType = "corteza::dal:attribute-codec:alias"
+)
+
+func (*CodecPlain) Type() AttributeCodecType { return AttributeCodecPlain }
+func (*CodecRecordValueSetJSON) Type() AttributeCodecType {
+	return AttributeCodecRecordValueSetJSON
+}
+func (*CodecAlias) Type() AttributeCodecType { return AttributeCodecAlias }
+
+func (*CodecPlain) SingleValueOnly() bool              { return true }
+func (*CodecRecordValueSetJSON) SingleValueOnly() bool { return false }
+func (*CodecAlias) SingleValueOnly() bool              { return true }

server/pkg/dal/attribute_types.go

@@ -0,0 +1,203 @@
+package dal
+
+type (
+	AttributeType string
+
+	// temp
+	Type interface {
+		Type() AttributeType
+		IsNullable() bool
+	}
+
+	// TypeID handles ID (uint64) coding
+	//
+	// Encoding/decoding might be different depending on
+	//  1) underlying store (and dialect)
+	//  2) value codec (raw, json ...)
+	//
+	// It is always Corteza ID
+	TypeID struct {
+		GeneratedByStore bool
+		Nullable         bool
+
+		HasDefault   bool
+		DefaultValue uint64
+	}
+
+	// TypeRef handles ID (uint64) coding + reference info
+	//
+	// Encoding/decoding might be different depending on
+	//  1) underlying store (and dialect)
+	//  2) value codec (raw, json ...)
+	TypeRef struct {
+		// defaults to ID
+		RefAttribute string
+		RefModel     *ModelRef
+
+		Nullable bool
+
+		HasDefault   bool
+		DefaultValue uint64
+	}
+
+	// TypeTimestamp handles timestamp coding
+	//
+	// Encoding/decoding might be different depending on
+	//  1) underlying store (and dialect)
+	//  2) value codec (raw, json ...)
+	TypeTimestamp struct {
+		Timezone  bool
+		Precision int
+		Nullable  bool
+
+		DefaultCurrentTimestamp bool
+	}
+
+	// TypeTime handles time coding
+	//
+	// Encoding/decoding might be different depending on
+	//  1) underlying store (and dialect)
+	//  2) value codec (raw, json ...)
+	TypeTime struct {
+		Timezone          bool
+		TimezonePrecision bool
+		Precision         int
+		Nullable          bool
+
+		DefaultCurrentTimestamp bool
+	}
+
+	// TypeDate handles date coding
+	//
+	// Encoding/decoding might be different depending on
+	//  1) underlying store (and dialect)
+	//  2) value codec (raw, jsonb...)
+	TypeDate struct {
+		//
+		Nullable bool
+
+		DefaultCurrentTimestamp bool
+	}
+
+	// TypeNumber handles number coding
+	//
+	// Encoding/decoding might be different depending on
+	//  1) underlying store (and dialect)
+	//  2) value codec (raw, jsonb...)
+	TypeNumber struct {
+		Precision int
+		Scale     int
+		Nullable  bool
+
+		HasDefault   bool
+		DefaultValue float64
+		Meta         map[string]any
+	}
+
+	// TypeText handles string coding
+	//
+	// Encoding/decoding might be different depending on
+	//  1) underlying store (and dialect)
+	//  2) value codec (raw, jsonb...)
+	TypeText struct {
+		Length   uint
+		Nullable bool
+
+		HasDefault   bool
+		DefaultValue string
+	}
+
+	// TypeBoolean
+	TypeBoolean struct {
+		//
+		Nullable bool
+
+		HasDefault   bool
+		DefaultValue bool
+	}
+
+	// TypeEnum
+	TypeEnum struct {
+		Values   []string
+		Nullable bool
+
+		HasDefault   bool
+		DefaultValue string
+	}
+
+	// TypeGeometry
+	TypeGeometry struct {
+		//
+		Nullable bool
+	}
+
+	// TypeJSON handles coding of arbitrary data into JSON structure
+	// NOT TO BE CONFUSED with encodedField
+	//
+	// Encoding/decoding might be different depending on
+	//  1) underlying store (and dialect)
+	//  2) value codec (raw, json ...)
+	TypeJSON struct {
+		//
+		Nullable bool
+
+		HasDefault   bool
+		DefaultValue any
+	}
+
+	// TypeBlob store/return data as
+	TypeBlob struct {
+		//
+		Nullable bool
+	}
+
+	TypeUUID struct {
+		Nullable bool
+	}
+
+	TypeNumberStoreNativeType string
+)
+
+const (
+	AttributeTypeID        AttributeType = "corteza::dal:attribute-type:id"
+	AttributeTypeRef       AttributeType = "corteza::dal:attribute-type:ref"
+	AttributeTypeTimestamp AttributeType = "corteza::dal:attribute-type:timestamp"
+	AttributeTypeTime      AttributeType = "corteza::dal:attribute-type:time"
+	AttributeTypeDate      AttributeType = "corteza::dal:attribute-type:date"
+	AttributeTypeNumber    AttributeType = "corteza::dal:attribute-type:number"
+	AttributeTypeText      AttributeType = "corteza::dal:attribute-type:text"
+	AttributeTypeBoolean   AttributeType = "corteza::dal:attribute-type:boolean"
+	AttributeTypeEnum      AttributeType = "corteza::dal:attribute-type:enum"
+	AttributeTypeGeometry  AttributeType = "corteza::dal:attribute-type:geometry"
+	AttributeTypejson      AttributeType = "corteza::dal:attribute-type:json"
+	AttributeTypeBlob      AttributeType = "corteza::dal:attribute-type:blob"
+	AttributeTypeUUID      AttributeType = "corteza::dal:attribute-type:uuid"
+)
+
+func (t TypeID) IsNullable() bool        { return t.Nullable }
+func (t TypeRef) IsNullable() bool       { return t.Nullable }
+func (t TypeTimestamp) IsNullable() bool { return t.Nullable }
+func (t TypeTime) IsNullable() bool      { return t.Nullable }
+func (t TypeDate) IsNullable() bool      { return t.Nullable }
+func (t TypeNumber) IsNullable() bool    { return t.Nullable }
+func (t TypeText) IsNullable() bool      { return t.Nullable }
+func (t TypeBoolean) IsNullable() bool   { return t.Nullable }
+func (t TypeEnum) IsNullable() bool      { return t.Nullable }
+func (t TypeGeometry) IsNullable() bool  { return t.Nullable }
+func (t TypeJSON) IsNullable() bool      { return t.Nullable }
+func (t TypeBlob) IsNullable() bool      { return t.Nullable }
+func (t TypeUUID) IsNullable() bool      { return t.Nullable }
+
+func (t TypeID) Type() AttributeType        { return AttributeTypeID }
+func (t TypeRef) Type() AttributeType       { return AttributeTypeRef }
+func (t TypeTimestamp) Type() AttributeType { return AttributeTypeTimestamp }
+func (t TypeTime) Type() AttributeType      { return AttributeTypeTime }
+func (t TypeDate) Type() AttributeType      { return AttributeTypeDate }
+func (t TypeNumber) Type() AttributeType    { return AttributeTypeNumber }
+func (t TypeText) Type() AttributeType      { return AttributeTypeText }
+func (t TypeBoolean) Type() AttributeType   { return AttributeTypeBoolean }
+func (t TypeEnum) Type() AttributeType      { return AttributeTypeEnum }
+func (t TypeGeometry) Type() AttributeType  { return AttributeTypeGeometry }
+func (t TypeJSON) Type() AttributeType      { return AttributeTypejson }
+func (t TypeBlob) Type() AttributeType      { return AttributeTypeBlob }
+func (t TypeUUID) Type() AttributeType      { return AttributeTypeUUID }

server/pkg/dal/buffer.go

@@ -0,0 +1,36 @@
+package dal
+
+import (
+	"context"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+)
+
+type (
+	// OrderedBuffer provides the buffered data in the specified order
+	OrderedBuffer interface {
+		Buffer
+
+		// InOrder makes the buffer provide the stored data in the specified order
+		InOrder(ss ...*filter.SortExpr) (err error)
+	}
+
+	// Buffer provides a place where you can buffer the data provided by DAL
+	// @note Buffers are currently primarily used for testing
+	Buffer interface {
+		// Seek moves the index pointer to the specified location
+		// After the Seek call, a Next() call is required
+		Seek(context.Context, int) error
+
+		// Len returns the number of elements in the buffer
+		Len() int
+
+		Iterator
+		Adder
+	}
+
+	Adder interface {
+		// Add adds a new ValueGetter to the buffer
+		Add(context.Context, ValueGetter) (err error)
+	}
+)

server/pkg/dal/conn.go

@@ -0,0 +1,42 @@
+package dal
+
+import "regexp"
+
+type (
+	ConnectionWrap struct {
+		ID uint64
+
+		connection Connection
+		params     ConnectionParams
+		Config     ConnectionConfig
+	}
+
+	ConnectionConfig struct {
+		SensitivityLevelID uint64
+		Label              string
+
+		// When model does not specify the ident (table name for example), fallback to this
+		ModelIdent string
+
+		// when a new model is added on a connection, it's ident
+		// is verified against this regexp
+		//
+		// ident is considered valid if it matches one of the expressions
+		// or if the list of checks is empty
+		ModelIdentCheck []*regexp.Regexp
+	}
+)
+
+func checkIdent(ident string, rr ...*regexp.Regexp) bool {
+	if len(rr) == 0 {
+		return true
+	}
+
+	for _, r := range rr {
+		if r.MatchString(ident) {
+			return true
+		}
+	}
+
+	return false
+}

server/pkg/dal/conn_test.go

@@ -0,0 +1,53 @@
+package dal
+
+import (
+	"regexp"
+	"testing"
+)
+
+func Test_checkIdent(t *testing.T) {
+	tests := []struct {
+		name  string
+		ident string
+		rr    []*regexp.Regexp
+		want  bool
+	}{
+		{
+			name:  "empty",
+			ident: "",
+			rr:    []*regexp.Regexp{},
+			want:  true,
+		},
+		{
+			name:  "one",
+			ident: "foo",
+			rr:    []*regexp.Regexp{regexp.MustCompile("foo")},
+			want:  true,
+		},
+		{
+			name:  "false",
+			ident: "foo",
+			rr:    []*regexp.Regexp{regexp.MustCompile("bar")},
+			want:  false,
+		},
+		{
+			name:  "two",
+			ident: "bar",
+			rr:    []*regexp.Regexp{regexp.MustCompile("foo"), regexp.MustCompile("bar")},
+			want:  true,
+		},
+		{
+			name:  "two failed",
+			ident: "foo",
+			rr:    []*regexp.Regexp{regexp.MustCompile("bar"), regexp.MustCompile("baz")},
+			want:  false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := checkIdent(tt.ident, tt.rr...); got != tt.want {
+				t.Errorf("checkIdent() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

server/pkg/dal/def_aggregate.go

@@ -0,0 +1,299 @@
+package dal
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/cortezaproject/corteza/server/pkg/ql"
+)
+
+type (
+	// Aggregate produces a series of aggregated rows from the provided sources based
+	// on the specified group.
+	Aggregate struct {
+		Ident     string
+		RelSource string
+		Filter    filter.Filter
+		filter    internalFilter
+
+		Group         []AggregateAttr
+		OutAttributes []AggregateAttr
+
+		SourceAttributes []AttributeMapping
+
+		rel      PipelineStep
+		plan     aggregatePlan
+		analysis map[string]OpAnalysis
+	}
+
+	// aggregatePlan outlines how the optimizer determined the dataset should be
+	// aggregated
+	aggregatePlan struct {
+		// partialScan indicates we can partially pull data from the two sources
+		// as the data is provided in the correct order.
+		partialScan bool
+	}
+
+	groupKey       []any
+	aggregateGroup struct {
+		key groupKey
+		agg *aggregator
+	}
+
+	// AggregateAttr is a simple wrapper to outline aggregated attribute definitions
+	AggregateAttr struct {
+		MultiValue bool
+		Key        bool
+
+		// @todo change; temporary for compose service
+		RawExpr string
+
+		Identifier string
+		Label      string
+		Expression *ql.ASTNode
+		Type       Type
+		Store      Codec
+	}
+)
+
+func (def *Aggregate) Identifier() string {
+	return def.Ident
+}
+
+func (def *Aggregate) Sources() []string {
+	return []string{def.RelSource}
+}
+
+func (def *Aggregate) Attributes() [][]AttributeMapping {
+	aa := append(def.Group, def.OutAttributes...)
+	out := make([]AttributeMapping, 0, len(aa))
+
+	for _, a := range aa {
+		out = append(out, a.toSimpleAttr())
+	}
+
+	return [][]AttributeMapping{out}
+}
+
+func (def *Aggregate) Analyze(ctx context.Context) (err error) {
+	// @todo proper analysis; for now we'll leave this as defaults
+	def.analysis = map[string]OpAnalysis{
+		OpAnalysisIterate: {
+			ScanCost:   CostUnknown,
+			SearchCost: CostUnknown,
+			FilterCost: CostUnknown,
+			SortCost:   CostUnknown,
+			OutputSize: SizeUnknown,
+		},
+	}
+	return
+}
+
+func (def *Aggregate) Analysis() map[string]OpAnalysis {
+	return def.analysis
+}
+
+func (def *Aggregate) Optimize(reqFilter internalFilter) (rspFilter internalFilter, err error) {
+	err = fmt.Errorf("not implemented")
+	return
+}
+
+// iterator initializes an iterator based on the provided pipeline step definition
+func (def *Aggregate) iterator(ctx context.Context, src Iterator) (out Iterator, err error) {
+	exec, err := def.init(ctx, src)
+	if err != nil {
+		return
+	}
+
+	return exec, exec.init(ctx)
+}
+
+// dryrun performs step execution without interacting with the data
+// @todo consider rewording this
+func (def *Aggregate) dryrun(ctx context.Context) (err error) {
+	_, err = def.init(ctx, nil)
+	return
+}
+
+func (def *Aggregate) init(ctx context.Context, src Iterator) (exec *aggregate, err error) {
+	exec = &aggregate{
+		source: src,
+	}
+
+	// Convert the provided filter into an internal filter
+	if def.Filter != nil {
+		def.filter, err = toInternalFilter(def.Filter)
+		if err != nil {
+			return
+		}
+	}
+
+	// Collect attributes from the underlaying step in case own are not provided
+	if len(def.SourceAttributes) == 0 {
+		def.SourceAttributes = collectAttributes(def.rel)
+	}
+
+	// Index source attributes for group/aggregate definition validation
+	srcAttrs := indexAttrs(def.SourceAttributes...)
+	pp := newQlParser(func(ident ql.Ident) (_ ql.Ident, err error) {
+		if _, ok := srcAttrs[ident.Value]; !ok {
+			return ident, fmt.Errorf("unknown attribute %s", ident.Value)
+		}
+		return ident, nil
+	})
+
+	prepAttr := func(attr AggregateAttr) (_ AggregateAttr, err error) {
+		if attr.RawExpr != "" {
+			// Parse (it already validates) raw expressions
+			attr.Expression, err = pp.Parse(attr.RawExpr)
+			if err != nil {
+				return
+			}
+		} else {
+			// Manually validate already parsed expressions
+			err = attr.Expression.Traverse(func(a *ql.ASTNode) (bool, *ql.ASTNode, error) {
+				if a.Symbol == "" {
+					return true, a, nil
+				}
+				if _, ok := srcAttrs[a.Symbol]; !ok {
+					return false, nil, fmt.Errorf("unknown attribute %s", a.Symbol)
+				}
+
+				return true, a, nil
+			})
+		}
+		if err != nil {
+			return
+		}
+		return def.determineAttrType(attr, def.SourceAttributes)
+	}
+
+	// Convert & validate group definitions
+	// - groups
+	outAttrs := make(map[string]bool, len(def.Group)+len(def.OutAttributes))
+	for i, attr := range def.Group {
+		attr, err = prepAttr(attr)
+		if err != nil {
+			return
+		}
+
+		def.Group[i] = attr
+		idtf := attr.Identifier
+
+		exec.groupDefs = append(exec.groupDefs, attr)
+		outAttrs[idtf] = true
+	}
+	// - aggregates
+	for i, attr := range def.OutAttributes {
+		// @todo change when needed; currently, all aggregates are numbers
+		attr.Type = &TypeNumber{}
+		attr, err = prepAttr(attr)
+		if err != nil {
+			return
+		}
+
+		def.OutAttributes[i] = attr
+		idtf := attr.Identifier
+
+		exec.aggregateDefs = append(exec.aggregateDefs, attr)
+		outAttrs[idtf] = true
+	}
+
+	// Generic validation
+	if len(def.Group) == 0 {
+		err = fmt.Errorf("no group attributes specified")
+		return
+	}
+
+	if len(def.OutAttributes) == 0 {
+		err = fmt.Errorf("no output attributes specified")
+		return
+	}
+
+	if len(def.SourceAttributes) == 0 {
+		err = fmt.Errorf("no source attributes specified")
+		return
+	}
+
+	// order
+	for _, s := range def.filter.OrderBy() {
+		if _, ok := outAttrs[s.Column]; !ok {
+			err = fmt.Errorf("order by attribute %s does not exist", s.Column)
+			return
+		}
+	}
+
+	// Finishup
+	exec.filter = def.filter
+	exec.def = *def
+	return
+}
+
+// determineAttrType determines the type of the AggregateAttr based on it's definition
+// and source attributes
+func (def *Aggregate) determineAttrType(base AggregateAttr, ss []AttributeMapping) (out AggregateAttr, err error) {
+	out = base
+	if out.Type != nil {
+		return
+	}
+
+	var root *ql.ASTNode
+	var t Type
+
+	// If we have a symbol, then we'll use it to determine the type.
+	// All current operations should return the same output type as the input one.
+	//
+	// In case of a function, use the output type of the root most function which has
+	// a known type.
+	//
+	// Note, some refs (group and add for example) may not know their types so we need
+	// to dig deeper.
+	base.Expression.Traverse(func(a *ql.ASTNode) (bool, *ql.ASTNode, error) {
+		if a.Symbol != "" {
+			root = a
+			return false, a, nil
+		}
+
+		if a.Ref != "" {
+			tmp := refToGvalExp[strings.ToLower(a.Ref)]
+			if tmp == nil || tmp.OutType == nil || tmp.OutTypeUnknown {
+				return true, a, nil
+			}
+
+			if tmp.OutType != nil {
+				t = tmp.OutType
+				return false, a, nil
+			}
+		}
+		return true, a, nil
+	})
+
+	if root != nil {
+		for _, s := range ss {
+			if s.Identifier() == root.Symbol {
+				t = s.Properties().Type
+				break
+			}
+		}
+	}
+
+	out.Type = t
+	return
+}
+
+func (a AggregateAttr) toSimpleAttr() SimpleAttr {
+	return SimpleAttr{
+		Ident: a.Identifier,
+		Props: MapProperties{
+			Label:     a.Label,
+			IsPrimary: a.Key,
+			Type:      a.Type,
+		},
+
+		// @todo won't matter for now
+		Expr: "",
+		Src:  "",
+	}
+}

server/pkg/dal/def_datasource.go

@@ -0,0 +1,362 @@
+package dal
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cortezaproject/corteza/server/pkg/expr"
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/cortezaproject/corteza/server/pkg/ql"
+)
+
+type (
+	// Datasource is a simple passthrough step for underlaying datasources.
+	// It exists primarily to make operations consistent.
+	Datasource struct {
+		Ident    string
+		Filter   filter.Filter
+		ModelRef ModelRef
+		filter   internalFilter
+
+		OutAttributes []AttributeMapping
+
+		analysis   map[string]OpAnalysis
+		connection *ConnectionWrap
+		model      *Model
+
+		// clobbered lists all of the steps that are offloaded into the datasource.
+		// The list is provided in order; the first step is the first step which should execute
+		// in the report pipeline.
+		//
+		// @todo change to generic step; doing this because I can for now
+		clobbered []*Aggregate
+
+		// provided in the init step so we can omit some code in the exec step
+		// @todo consider removing this
+		auxIter Iterator
+	}
+)
+
+func (def *Datasource) Identifier() string {
+	return def.Ident
+}
+
+func (def *Datasource) Sources() []string {
+	return []string{}
+}
+
+func (def *Datasource) Attributes() [][]AttributeMapping {
+	if len(def.clobbered) > 0 {
+		return def.offloadedAttributes()
+	}
+
+	return def.ownAttributes()
+}
+
+func (def *Datasource) Analyze(ctx context.Context) (err error) {
+	a, err := def.connection.connection.Analyze(ctx, def.model)
+	if err != nil {
+		return
+	}
+
+	def.analysis = map[string]OpAnalysis{
+		OpAnalysisIterate: {
+			ScanCost:   CostUnknown,
+			SearchCost: CostUnknown,
+			FilterCost: CostUnknown,
+			SortCost:   CostUnknown,
+			OutputSize: SizeUnknown,
+		},
+	}
+
+	if _, ok := a[OpAnalysisAggregate]; ok {
+		def.analysis[OpAnalysisAggregate] = a[OpAnalysisAggregate]
+	}
+	if _, ok := a[OpAnalysisJoin]; ok {
+		def.analysis[OpAnalysisJoin] = a[OpAnalysisJoin]
+	}
+
+	return
+}
+
+func (def *Datasource) Analysis() map[string]OpAnalysis {
+	return def.analysis
+}
+
+func (def *Datasource) Optimize(req internalFilter) (res internalFilter, err error) {
+	return internalFilter{}, fmt.Errorf("optimization not implemented")
+}
+
+func (def *Datasource) init(ctx context.Context) (err error) {
+	if def.model == nil {
+		return fmt.Errorf("cannot initialize datasource: model not set")
+	}
+	if def.connection == nil {
+		return fmt.Errorf("cannot initialize datasource: connection not set")
+	}
+
+	if def.Filter != nil {
+		def.filter, err = toInternalFilter(def.Filter)
+		if err != nil {
+			return
+		}
+	}
+
+	if len(def.OutAttributes) == 0 {
+		def.OutAttributes = def.outAttrsFromModel(def.model)
+	}
+
+	pp := make([]string, 0, len(def.OutAttributes)/2+1)
+	for _, a := range def.OutAttributes {
+		if a.Properties().IsPrimary {
+			pp = append(pp, a.Identifier())
+		}
+	}
+
+	// Assure and attempt to correct the provided sort to conform with the data set and the
+	// paging cursor (if any)
+	def.filter, err = assureSort(def.filter, pp)
+	if err != nil {
+		return
+	}
+
+	// Firstly validate the base
+	err = def.validate()
+	if err != nil {
+		return
+	}
+
+	if len(def.clobbered) > 0 {
+		// @todo currently, we can only do one; change this when we tweak the DB
+		//       offloading code.
+		ag := def.clobbered[0]
+
+		// Invoke the aggregation's init to perform the validation and preparation logic
+		var wa *aggregate
+		wa, err = ag.init(ctx, nil)
+		if err != nil {
+			return
+		}
+
+		// Preprocess the filters to conform to connection API
+		f, having, err := def.getAggregationFilters(def.filter, wa.filter)
+		if err != nil {
+			return err
+		}
+
+		def.auxIter, err = def.connection.connection.Aggregate(ctx, def.model, f, wa.groupDefs, wa.aggregateDefs, having)
+		return err
+	}
+
+	def.auxIter, err = def.connection.connection.Search(ctx, def.model, def.filter)
+	return err
+}
+
+func (def *Datasource) exec(ctx context.Context) (out Iterator, err error) {
+	if def.auxIter == nil {
+		return nil, fmt.Errorf("datasource not initialized")
+	}
+
+	return def.auxIter, nil
+}
+
+func (def *Datasource) validate() (err error) {
+	err = func() (err error) {
+		if len(def.OutAttributes) == 0 {
+			return fmt.Errorf("no attributes specified")
+		}
+
+		return
+	}()
+	if err != nil {
+		return fmt.Errorf("invalid definition: %v", err)
+	}
+
+	return
+}
+
+func (def *Datasource) outAttrsFromModel(model *Model) (attrs []AttributeMapping) {
+	for _, attr := range model.Attributes {
+		if attr.Type == nil {
+			panic(fmt.Sprintf("impossible state: attribute %s has no type", attr.Ident))
+		}
+
+		attrs = append(attrs, SimpleAttr{
+			Ident: attr.Ident,
+			Src:   attr.Ident,
+			Props: MapProperties{
+				IsPrimary:    attr.PrimaryKey,
+				IsSystem:     attr.System,
+				IsFilterable: attr.Filterable,
+				IsSortable:   attr.Sortable,
+				Nullable:     attr.Type.IsNullable(),
+				Type:         attr.Type,
+			},
+		})
+	}
+
+	return
+}
+
+func (def *Datasource) shouldClobberAggregation() bool {
+	costs, ok := def.analysis[OpAnalysisAggregate]
+	if !ok {
+		return false
+	}
+
+	// @todo more to it; check and compare costs; for now we know that all rdbms
+	//       offloads will be faster.
+
+	_ = costs
+
+	return true
+}
+
+func (def *Datasource) ownAttributes() [][]AttributeMapping {
+	return [][]AttributeMapping{def.OutAttributes}
+}
+
+func (def *Datasource) offloadedAttributes() [][]AttributeMapping {
+	// The last offloaded step's attributes is what this step's iterator returns
+	return def.clobbered[len(def.clobbered)-1].Attributes()
+}
+
+func (def *Datasource) clobber(s PipelineStep) (ok bool) {
+	switch cs := s.(type) {
+	case *Aggregate:
+		if !def.shouldClobberAggregation() {
+			return false
+		}
+
+		def.clobbered = append(def.clobbered, cs)
+		return true
+	}
+
+	return
+}
+
+// getAggregationFilters transforms the base and the aggregation filters to conform
+// to the store/dal's model.Aggregate API
+//
+// - The full filter.Filter return parameter is applied to the base dataset
+// - The QL node return parameter is applied only to the aggregated dataset
+//
+// @todo should we change the store's API to accept 2x filter.Filter? I think
+//       that would allow the underlaying driver to decide how to handle them
+//       instead of relying on what SQL does.
+func (def *Datasource) getAggregationFilters(base, agg internalFilter) (filter internalFilter, having *ql.ASTNode, err error) {
+	var typedV expr.TypedValue
+	filter = base
+
+	// Move the aggregation's order and limit to the base filter because those
+	// are applied to the final output (per SQL)
+	//
+	// Leave base filtering parameters (filter, constraints, ...) as those are apploed
+	// to the dataset before aggregation (which is correct).
+	filter.orderBy = agg.orderBy
+	filter.limit = agg.limit
+
+	// Convert the rest of the filtering parameters defined on the aggregation's filter
+	// to the QL node.
+	// ALl of that is applied after the aggregation which is correct.
+	var nConstraints *ql.ASTNode
+	if len(agg.constraints) > 0 {
+		nConstraints = &ql.ASTNode{
+			Ref: "and",
+		}
+		for k, c := range agg.constraints {
+			arg := &ql.ASTNode{
+				Ref: "or",
+			}
+
+			for _, v := range c {
+				typedV, err = expr.Typify(v)
+				if err != nil {
+					return
+				}
+				arg.Args = append(arg.Args, &ql.ASTNode{
+					Symbol: k,
+					Value:  ql.WrapValue(typedV),
+				})
+			}
+
+			nConstraints.Args = append(nConstraints.Args, &ql.ASTNode{
+				Ref:  "group",
+				Args: ql.ASTNodeSet{arg},
+			})
+		}
+	}
+
+	var nStateConstraints *ql.ASTNode
+	if len(agg.stateConstraints) > 0 {
+		nStateConstraints = &ql.ASTNode{
+			Ref: "and",
+		}
+		for k, c := range agg.stateConstraints {
+			typedV, err = expr.Typify(c)
+			if err != nil {
+				return
+			}
+
+			nStateConstraints.Args = append(nStateConstraints.Args, &ql.ASTNode{
+				Symbol: k,
+				Value:  ql.WrapValue(typedV),
+			})
+		}
+	}
+
+	var nMetaConstraints *ql.ASTNode
+	if len(agg.stateConstraints) > 0 {
+		nMetaConstraints = &ql.ASTNode{
+			Ref: "and",
+		}
+		for k, c := range agg.stateConstraints {
+			typedV, err = expr.Typify(c)
+			if err != nil {
+				return
+			}
+
+			nMetaConstraints.Args = append(nMetaConstraints.Args, &ql.ASTNode{
+				Symbol: k,
+				Value:  ql.WrapValue(typedV),
+			})
+		}
+	}
+
+	nExpression := agg.expParsed
+
+	var nCursor *ql.ASTNode
+	if agg.cursor != nil {
+		nCursor, err = agg.cursor.ToAST(nil, nil)
+		if err != nil {
+			return
+		}
+	}
+
+	having = &ql.ASTNode{
+		Ref: "and",
+	}
+
+	if nConstraints != nil {
+		having.Args = append(having.Args, nConstraints)
+	}
+	if nStateConstraints != nil {
+		having.Args = append(having.Args, nStateConstraints)
+	}
+	if nMetaConstraints != nil {
+		having.Args = append(having.Args, nMetaConstraints)
+	}
+	if nExpression != nil {
+		having.Args = append(having.Args, nExpression)
+	}
+	if nCursor != nil {
+		having.Args = append(having.Args, nCursor)
+	}
+
+	// In case everything is empty, no need for the having part
+	if len(having.Ref) == 0 {
+		having = nil
+	}
+
+	return
+}

server/pkg/dal/def_join.go

@@ -0,0 +1,218 @@
+package dal
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+)
+
+type (
+	// Join produces a series of joined rows from the provided sources based
+	// on the JoinPredicate.
+	//
+	// The join step produces an SQL left join-like output where all right rows
+	// have a corresponding left row.
+	Join struct {
+		Ident    string
+		RelLeft  string
+		RelRight string
+		// @todo allow multiple join predicates; for now (for easier indexing)
+		// only allow one (this is the same as we had before)
+		On     JoinPredicate
+		Filter filter.Filter
+		filter internalFilter
+
+		OutAttributes   []AttributeMapping
+		LeftAttributes  []AttributeMapping
+		RightAttributes []AttributeMapping
+
+		relLeft  PipelineStep
+		relRight PipelineStep
+		plan     joinPlan
+		analysis map[string]OpAnalysis
+	}
+
+	// JoinPredicate determines the attributes the two datasets should get joined on
+	JoinPredicate struct {
+		Left  string
+		Right string
+	}
+
+	// joinPlan outlines how the optimizer determined the two datasets should be
+	// joined on.
+	joinPlan struct {
+		// @todo add strategy when we have different strategies implemented
+		// strategy string
+
+		// partialScan indicates we can partially pull data from the two sources
+		// as the data is provided in the correct order.
+		partialScan bool
+	}
+)
+
+func (def *Join) Identifier() string {
+	return def.Ident
+}
+
+func (def *Join) Sources() []string {
+	return []string{def.RelLeft, def.RelRight}
+}
+
+func (def *Join) Attributes() [][]AttributeMapping {
+	return [][]AttributeMapping{def.OutAttributes}
+}
+
+func (def *Join) Analyze(ctx context.Context) (err error) {
+	// @todo proper analysis; for now we'll leave this as defaults
+	def.analysis = map[string]OpAnalysis{
+		OpAnalysisIterate: {
+			ScanCost:   CostUnknown,
+			SearchCost: CostUnknown,
+			FilterCost: CostUnknown,
+			SortCost:   CostUnknown,
+			OutputSize: SizeUnknown,
+		},
+	}
+
+	return
+}
+
+func (def *Join) Analysis() map[string]OpAnalysis {
+	return def.analysis
+}
+
+func (def *Join) Optimize(req internalFilter) (res internalFilter, err error) {
+	err = fmt.Errorf("not implemented")
+	return
+}
+
+// iterator initializes an iterator based on the provided pipeline step definition
+func (def *Join) iterator(ctx context.Context, left, right Iterator) (out Iterator, err error) {
+	exec, err := def.init(ctx, left, right)
+	if err != nil {
+		return
+	}
+
+	// Get pred. type
+	// @todo should validate that both pred. types are the same/compatible
+	var pt Type
+	for _, a := range def.LeftAttributes {
+		if a.Identifier() == def.On.Left {
+			pt = a.Properties().Type
+			break
+		}
+	}
+
+	return exec, exec.init(ctx, pt)
+}
+
+// dryrun performs step execution without interacting with the data
+// @todo consider rewording this
+func (def *Join) dryrun(ctx context.Context) (err error) {
+	_, err = def.init(ctx, nil, nil)
+	return
+}
+
+func (def *Join) init(ctx context.Context, left, right Iterator) (exec *joinLeft, err error) {
+	exec = &joinLeft{
+		leftSource:  left,
+		rightSource: right,
+	}
+
+	// Convert the provided filter into an internal filter
+	if def.Filter != nil {
+		def.filter, err = toInternalFilter(def.Filter)
+		if err != nil {
+			return
+		}
+	}
+
+	// Collect attributes from the underlaying step in case own are not provided
+	if len(def.LeftAttributes) == 0 {
+		def.LeftAttributes = collectAttributes(def.relLeft)
+	}
+	if len(def.RightAttributes) == 0 {
+		def.RightAttributes = collectAttributes(def.relRight)
+	}
+
+	// When the output attributes are not provided, we determine them based on the input
+	// attributes.
+	//
+	// The join step prefixes each attribute with the identifier of the source they came from.
+	// This avoids name collisions.
+	// @todo consider improving this to only prefix when there is a name collision
+	//       and allow the nested attr. to be more permissive; for example, instead of
+	//       requiring a.b.c only b.c or c would be enough.
+	if len(def.OutAttributes) == 0 {
+		ins := func(srcIdent string, aa []AttributeMapping) {
+			for _, a := range aa {
+				p := a.Properties()
+				def.OutAttributes = append(def.OutAttributes, SimpleAttr{
+					Ident: fmt.Sprintf("%s%s%s", srcIdent, attributeNestingSeparator, a.Identifier()),
+					Expr:  a.Expression(),
+					Src:   a.Identifier(),
+					Props: p,
+				})
+			}
+		}
+
+		def.OutAttributes = make([]AttributeMapping, 0, len(def.LeftAttributes)+len(def.RightAttributes))
+		ins(def.RelLeft, def.LeftAttributes)
+		ins(def.RelRight, def.RightAttributes)
+	}
+
+	// Assure and attempt to correct the provided sort to conform with the data set and the
+	// paging cursor (if any)
+	def.filter, err = assureSort(def.filter, exec.collectPrimaryAttributes(def.OutAttributes))
+	if err != nil {
+		return
+	}
+
+	// Index attrs for validations
+	leftSrcAttrs := indexAttrs(def.LeftAttributes...)
+	rightSrcAttrs := indexAttrs(def.RightAttributes...)
+	outAttrs := indexAttrs(def.OutAttributes...)
+
+	for _, a := range def.OutAttributes {
+		if !leftSrcAttrs[a.Source()] && !rightSrcAttrs[a.Source()] {
+			return nil, fmt.Errorf("unknown attribute %s", a.Source())
+		}
+	}
+
+	// Generic validation
+	if !leftSrcAttrs[def.On.Left] {
+		return nil, fmt.Errorf("unknown join predicate attribute %s", def.On.Left)
+	}
+	if !rightSrcAttrs[def.On.Right] {
+		return nil, fmt.Errorf("unknown join predicate attribute %s", def.On.Right)
+	}
+
+	if len(def.OutAttributes) == 0 {
+		return nil, fmt.Errorf("no attributes specified")
+	}
+	if len(def.LeftAttributes) == 0 {
+		return nil, fmt.Errorf("no left attributes specified")
+	}
+	if len(def.RightAttributes) == 0 {
+		return nil, fmt.Errorf("no right attributes specified")
+	}
+
+	if def.On.Left == "" {
+		return nil, fmt.Errorf("no left attribute in the join predicate specified")
+	}
+	if def.On.Right == "" {
+		return nil, fmt.Errorf("no right attribute in the join predicate specified")
+	}
+
+	// order
+	for _, s := range def.filter.OrderBy() {
+		if _, ok := outAttrs[s.Column]; !ok {
+			return nil, fmt.Errorf("order attribute %s does not exist", s.Column)
+		}
+	}
+
+	exec.filter = def.filter
+	exec.def = *def
+	return
+}

server/pkg/dal/def_link.go

@@ -0,0 +1,289 @@
+package dal
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+)
+
+type (
+	// Link produces a series of left and corresponding right rows based on the
+	// provided sources and the LinkPredicate.
+	//
+	// The Link step produces an SQL left join-like output where left and right
+	// rows are served separately and the left rows are not duplicated.
+	Link struct {
+		Ident    string
+		RelLeft  string
+		RelRight string
+		// @todo allow multiple link predicates; for now (for easier indexing)
+		// only allow one (this is the same as we had before)
+		On LinkPredicate
+		// @todo consider splitting filter into left and right filter
+		filter internalFilter
+		Filter filter.Filter
+
+		OutLeftAttributes  []AttributeMapping
+		OutRightAttributes []AttributeMapping
+		LeftAttributes     []AttributeMapping
+		RightAttributes    []AttributeMapping
+
+		relLeft  PipelineStep
+		relRight PipelineStep
+
+		plan     linkPlan
+		analysis map[string]OpAnalysis
+	}
+
+	// LinkPredicate determines the attributes the two datasets should get joined on
+	LinkPredicate struct {
+		Left  string
+		Right string
+	}
+
+	// linkPlan outlines how the optimizer determined the two datasets should be
+	// joined on.
+	linkPlan struct {
+		// @todo add strategy when we have different strategies implemented
+		// strategy string
+
+		// partialScan indicates we can partially pull data from the two sources
+		// as the data is provided in the correct order.
+		partialScan bool
+	}
+
+	rowLink struct {
+		a, b ValueGetter
+	}
+)
+
+const (
+	LinkRefIdent = "$sys.ref"
+)
+
+func (def *Link) Identifier() string {
+	return def.Ident
+}
+
+func (def *Link) Sources() []string {
+	return []string{def.RelLeft, def.RelRight}
+}
+
+func (def *Link) Attributes() [][]AttributeMapping {
+	return [][]AttributeMapping{def.OutLeftAttributes, def.OutRightAttributes}
+}
+
+func (def *Link) Analyze(ctx context.Context) (err error) {
+	// @todo proper analysis; for now we'll leave this as defaults
+	def.analysis = map[string]OpAnalysis{
+		OpAnalysisIterate: {
+			ScanCost:   CostUnknown,
+			SearchCost: CostUnknown,
+			FilterCost: CostUnknown,
+			SortCost:   CostUnknown,
+			OutputSize: SizeUnknown,
+		},
+	}
+
+	return
+}
+
+func (def *Link) Analysis() map[string]OpAnalysis {
+	return def.analysis
+}
+
+func (def *Link) Optimize(req internalFilter) (res internalFilter, err error) {
+	err = fmt.Errorf("not implemented")
+	return
+}
+
+// iterator initializes an iterator based on the provided pipeline step definition
+func (def *Link) iterator(ctx context.Context, left, right Iterator) (out Iterator, err error) {
+	x, err := def.init(ctx, left, right)
+	if err != nil {
+		return
+	}
+
+	// Get pred. type
+	// @todo should validate that both pred. types are the same/compatible
+	var pt Type
+	for _, a := range def.LeftAttributes {
+		if a.Identifier() == def.On.Left {
+			pt = a.Properties().Type
+			break
+		}
+	}
+
+	// Collect order by attributes which appear in the right source
+	rightAttrs := indexAttrs(def.RightAttributes...)
+	rightOrderAttrs := make([]string, 0, len(def.filter.OrderBy()))
+	for _, o := range def.filter.OrderBy() {
+		if rightAttrs[o.Column] {
+			rightOrderAttrs = append(rightOrderAttrs, o.Column)
+		}
+	}
+
+	return x, x.init(ctx, pt, rightOrderAttrs)
+}
+
+// dryrun performs step execution without interacting with the data
+// @todo consider rewording this
+func (def *Link) dryrun(ctx context.Context) (err error) {
+	_, err = def.init(ctx, nil, nil)
+	return
+}
+
+func (def *Link) init(ctx context.Context, left, right Iterator) (exec *linkLeft, err error) {
+	exec = &linkLeft{
+		leftSource:  left,
+		rightSource: right,
+	}
+
+	// Convert the provided filter into an internal filter
+	if def.Filter != nil {
+		def.filter, err = toInternalFilter(def.Filter)
+		if err != nil {
+			return
+		}
+	}
+
+	// Collect attributes from the underlaying step in case own are not provided
+	if len(def.LeftAttributes) == 0 {
+		def.LeftAttributes = collectAttributes(def.relLeft)
+	}
+	if len(def.RightAttributes) == 0 {
+		def.RightAttributes = collectAttributes(def.relRight)
+	}
+
+	// @todo this isn't quite ok -- the ident of left/right must become the src of the out.
+	//       An edge-case but it should be covered.
+	if len(def.OutLeftAttributes) == 0 {
+		def.OutLeftAttributes = def.LeftAttributes
+	}
+	if len(def.OutRightAttributes) == 0 {
+		def.OutRightAttributes = def.RightAttributes
+	}
+
+	// Index source attributes for group/aggregate definition validation
+	leftSrcAttrs := indexAttrs(def.LeftAttributes...)
+	rightSrcAttrs := indexAttrs(def.RightAttributes...)
+	outAttrs := indexAttrs(def.OutLeftAttributes...)
+	indexAttrsInto(outAttrs, def.OutRightAttributes...)
+
+	// Check if source attributes are present
+	for _, a := range append(def.OutLeftAttributes, def.OutRightAttributes...) {
+		if !leftSrcAttrs[a.Source()] && !rightSrcAttrs[a.Source()] {
+			return nil, fmt.Errorf("unknown attribute %s", a.Source())
+		}
+	}
+
+	// Check link predicates
+	if !leftSrcAttrs[def.On.Left] {
+		return nil, fmt.Errorf("left link predicate %s does not exist", def.On.Left)
+	}
+	if !rightSrcAttrs[def.On.Right] {
+		return nil, fmt.Errorf("right link predicate %s does not exist", def.On.Right)
+	}
+
+	// General validation
+	if len(def.OutLeftAttributes) == 0 {
+		return nil, fmt.Errorf("no left output attributes specified")
+	}
+	if len(def.OutRightAttributes) == 0 {
+		return nil, fmt.Errorf("no right output attributes specified")
+	}
+	if len(def.LeftAttributes) == 0 {
+		return nil, fmt.Errorf("no left attributes specified")
+	}
+	if len(def.RightAttributes) == 0 {
+		return nil, fmt.Errorf("no right attributes specified")
+	}
+
+	if def.On.Left == "" {
+		return nil, fmt.Errorf("no left attribute in the link predicate specified")
+	}
+	if def.On.Right == "" {
+		return nil, fmt.Errorf("no right attribute in the link predicate specified")
+	}
+
+	for _, s := range def.filter.OrderBy() {
+		if _, ok := outAttrs[s.Column]; !ok {
+			return nil, fmt.Errorf("order attribute %s does not exist", s.Column)
+		}
+	}
+
+	exec.filter = def.filter
+	exec.def = *def
+	return
+}
+
+func (r *rowLink) SelectGVal(ctx context.Context, k string) (interface{}, error) {
+	// @todo this way of determining multivalue is not entirely ok but we can get away with it
+	//       since we have special functions to work with those.
+	//       We need to somehow inform this bit as to what fields are multi value.
+	row, counts := r.getGetter(k)
+	if row == nil {
+		return nil, nil
+	}
+
+	cc := counts[k]
+	if cc == 0 {
+		return nil, nil
+	}
+
+	if cc == 1 {
+		return row.GetValue(k, 0)
+	}
+
+	out := make([]any, cc)
+	for i := uint(0); i < cc; i++ {
+		out[i], _ = row.GetValue(k, i)
+	}
+	return out, nil
+}
+
+func (r *rowLink) GetValue(name string, pos uint) (v any, err error) {
+	row, a := r.getGetter(name)
+	if row == nil {
+		return
+	}
+
+	if cc, ok := a[name]; ok {
+		if pos >= cc {
+			return nil, nil
+		}
+		return row.GetValue(name, pos)
+	}
+
+	return
+}
+
+func (r *rowLink) CountValues() (out map[string]uint) {
+	out = make(map[string]uint)
+
+	for k, c := range r.a.CountValues() {
+		out[k] = c
+	}
+	for k, c := range r.b.CountValues() {
+		out[k] = c
+	}
+
+	return
+}
+
+// getGetter returns the ValueGetter based on if the value is in a or b
+// If neither, it returns nil
+func (r *rowLink) getGetter(k string) (row ValueGetter, cc map[string]uint) {
+	cc = r.a.CountValues()
+	if _, ok := cc[k]; ok {
+		return r.a, cc
+	}
+
+	cc = r.b.CountValues()
+	if _, ok := cc[k]; ok {
+		return r.b, cc
+	}
+
+	return nil, nil
+}

server/pkg/dal/diff.go

@@ -0,0 +1,189 @@
+package dal
+
+type (
+	modelDiffType string
+	// ModelDiff defines one identified missmatch between two models
+	ModelDiff struct {
+		Type modelDiffType
+		// Original will be nil when a new attribute is being added
+		Original *Attribute
+		// Asserted will be nil wen an existing attribute is being removed
+		Asserted *Attribute
+	}
+
+	ModelDiffSet []*ModelDiff
+)
+
+const (
+	AttributeMissing             modelDiffType = "attributeMissing"
+	AttributeTypeMissmatch       modelDiffType = "typeMissmatch"
+	AttributeSensitivityMismatch modelDiffType = "sensitivityMismatch"
+	AttributeCodecMismatch       modelDiffType = "codecMismatch"
+)
+
+// Diff calculates the diff between models a and b where a is used as base
+func (a *Model) Diff(b *Model) (out ModelDiffSet) {
+	if a == nil {
+		a = &Model{}
+	}
+	if b == nil {
+		b = &Model{}
+	}
+
+	bIndex := make(map[string]struct {
+		found bool
+		attr  *Attribute
+	})
+	for _, _attr := range b.Attributes {
+		attr := _attr
+		bIndex[attr.Ident] = struct {
+			found bool
+			attr  *Attribute
+		}{
+			attr: attr,
+		}
+	}
+
+	aIndex := make(map[string]struct {
+		found bool
+		attr  *Attribute
+	})
+	for _, _attr := range a.Attributes {
+		attr := _attr
+		aIndex[attr.Ident] = struct {
+			found bool
+			attr  *Attribute
+		}{
+			attr: attr,
+		}
+	}
+
+	// Deleted and update ones
+	for _, _attrA := range a.Attributes {
+		attrA := _attrA
+		// store is an interface to something that could be a pointer.
+		// we need to copy it to make sure we don't get a nil pointer
+		// make sure not to modify this since it would modify the original
+		attrA.Store = _attrA.Store
+
+		// Missmatches
+		attrBAux, ok := bIndex[attrA.Ident]
+		if !ok {
+			out = append(out, &ModelDiff{
+				Type:     AttributeMissing,
+				Original: attrA,
+			})
+			continue
+		}
+
+		// Typecheck
+		if attrA.Type.Type() != attrBAux.attr.Type.Type() {
+			out = append(out, &ModelDiff{
+				Type:     AttributeTypeMissmatch,
+				Original: attrA,
+				Asserted: attrBAux.attr,
+			})
+		}
+
+		// Other stuff
+		// @todo improve; for now it'll do
+		if attrA.SensitivityLevelID != attrBAux.attr.SensitivityLevelID {
+			out = append(out, &ModelDiff{
+				Type:     AttributeSensitivityMismatch,
+				Original: attrA,
+				Asserted: attrBAux.attr,
+			})
+		}
+		if attrA.Store.Type() != attrBAux.attr.Store.Type() {
+			out = append(out, &ModelDiff{
+				Type:     AttributeCodecMismatch,
+				Original: attrA,
+				Asserted: attrBAux.attr,
+			})
+		}
+	}
+
+	// New
+	for _, _attrB := range b.Attributes {
+		attrB := _attrB
+
+		// Missmatches
+		_, ok := aIndex[attrB.Ident]
+		if !ok {
+			out = append(out, &ModelDiff{
+				Type:     AttributeMissing,
+				Original: nil,
+				Asserted: attrB,
+			})
+			continue
+		}
+	}
+
+	return
+}
+
+func (dd ModelDiffSet) Alterations() (out []*Alteration) {
+	add := func(a *Alteration) {
+		out = append(out, a)
+	}
+
+	for _, d := range dd {
+		switch d.Type {
+		case AttributeMissing:
+			if d.Asserted == nil {
+				// @todo if this was the last attribute we can consider dropping this column
+				if d.Original.Store.Type() == AttributeCodecRecordValueSetJSON {
+					break
+				}
+
+				add(&Alteration{
+					AttributeDelete: &AttributeDelete{
+						Attr: d.Original,
+					},
+				})
+			} else {
+				if d.Asserted.Store.Type() == AttributeCodecRecordValueSetJSON {
+					add(&Alteration{
+						AttributeAdd: &AttributeAdd{
+							Attr: &Attribute{
+								Ident: d.Asserted.StoreIdent(),
+								Type:  &TypeJSON{Nullable: false},
+								Store: &CodecPlain{},
+							},
+						},
+					})
+				} else {
+					add(&Alteration{
+						AttributeAdd: &AttributeAdd{
+							Attr: d.Asserted,
+						},
+					})
+				}
+
+			}
+
+		case AttributeTypeMissmatch:
+			// @todo we might have to do some validation earlier on
+			if d.Original.Store.Type() == AttributeCodecRecordValueSetJSON {
+				break
+			}
+
+			add(&Alteration{
+				AttributeReType: &AttributeReType{
+					Attr: d.Asserted,
+					To:   d.Asserted.Type,
+				},
+			})
+
+		case AttributeCodecMismatch:
+			add(&Alteration{
+				AttributeReEncode: &AttributeReEncode{
+					Attr: d.Asserted,
+					To:   d.Asserted.Store,
+				},
+			})
+		}
+	}
+
+	return
+}

server/pkg/dal/diff_test.go

@@ -0,0 +1,116 @@
+package dal
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDiff_same(t *testing.T) {
+	a := &Model{
+		Attributes: AttributeSet{{
+			Ident: "F1",
+			Type:  TypeText{},
+			Store: &CodecPlain{},
+		}},
+	}
+
+	dd := a.Diff(a)
+	require.Len(t, dd, 0)
+}
+
+func TestDiff_wrongAttrType(t *testing.T) {
+	a := &Model{
+		Attributes: AttributeSet{{
+			Ident: "F1",
+			Type:  TypeText{},
+			Store: &CodecPlain{},
+		}},
+	}
+	b := &Model{
+		Attributes: AttributeSet{{
+			Ident: "F1",
+			Type:  TypeBlob{},
+			Store: &CodecPlain{},
+		}},
+	}
+
+	dd := a.Diff(b)
+	require.Len(t, dd, 1)
+	require.Equal(t, AttributeTypeMissmatch, dd[0].Type)
+}
+
+func TestDiff_removedAttr(t *testing.T) {
+	a := &Model{
+		Attributes: AttributeSet{{
+			Ident: "F1",
+			Type:  TypeText{},
+			Store: &CodecPlain{},
+		}, {
+			Ident: "F2",
+			Type:  TypeText{},
+			Store: &CodecPlain{},
+		}},
+	}
+	b := &Model{
+		Attributes: AttributeSet{{
+			Ident: "F1",
+			Type:  TypeText{},
+			Store: &CodecPlain{},
+		}},
+	}
+
+	dd := a.Diff(b)
+	require.Len(t, dd, 1)
+	require.Equal(t, AttributeMissing, dd[0].Type)
+	require.NotNil(t, dd[0].Original)
+	require.Nil(t, dd[0].Asserted)
+}
+
+func TestDiff_addedAttr(t *testing.T) {
+	a := &Model{
+		Attributes: AttributeSet{{
+			Ident: "F1",
+			Type:  TypeText{},
+			Store: &CodecPlain{},
+		}},
+	}
+	b := &Model{
+		Attributes: AttributeSet{{
+			Ident: "F1",
+			Type:  TypeText{},
+			Store: &CodecPlain{},
+		}, {
+			Ident: "F2",
+			Type:  TypeText{},
+			Store: &CodecPlain{},
+		}},
+	}
+
+	dd := a.Diff(b)
+	require.Len(t, dd, 1)
+	require.Equal(t, AttributeMissing, dd[0].Type)
+	require.Nil(t, dd[0].Original)
+	require.NotNil(t, dd[0].Asserted)
+}
+
+func TestDiff_changedCodec(t *testing.T) {
+	a := &Model{
+		Attributes: AttributeSet{{
+			Ident: "F1",
+			Type:  TypeText{},
+			Store: &CodecPlain{},
+		}},
+	}
+	b := &Model{
+		Attributes: AttributeSet{{
+			Ident: "F1",
+			Type:  TypeText{},
+			Store: &CodecRecordValueSetJSON{},
+		}},
+	}
+
+	dd := a.Diff(b)
+	require.Len(t, dd, 1)
+	require.Equal(t, AttributeCodecMismatch, dd[0].Type)
+}

server/pkg/dal/driver.go

@@ -0,0 +1,257 @@
+package dal
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/cortezaproject/corteza/server/pkg/ql"
+	"go.uber.org/zap"
+)
+
+type (
+	PKValues map[string]any
+
+	ConnectionParams struct {
+		Type   string         `json:"type"`
+		Params map[string]any `json:"params"`
+	}
+
+	Connection interface {
+		// Meta
+
+		// Models returns all the models the underlying connection already supports
+		//
+		// This is useful when adding support for new models since we can find out what
+		// can work out of the box.
+		Models(context.Context) (ModelSet, error)
+
+		// Operations returns all of the operations the given store supports
+		Operations() OperationSet
+
+		// Can returns true if this store can handle the given operations
+		Can(operations ...Operation) bool
+
+		// DML stuff
+
+		// Create stores the given data into the underlying database
+		Create(ctx context.Context, m *Model, rr ...ValueGetter) error
+
+		// Update updates the given value in the underlying connection
+		Update(ctx context.Context, m *Model, r ValueGetter) error
+
+		// Lookup returns one bit of data
+		Lookup(context.Context, *Model, ValueGetter, ValueSetter) error
+
+		// Search returns an iterator which can be used to access all if the bits
+		Search(context.Context, *Model, filter.Filter) (Iterator, error)
+
+		// Count returns the total number of rows matching the filter
+		Count(context.Context, *Model, filter.Filter) (uint, error)
+
+		// Analyze returns the operation analysis the connection can perform for the model
+		Analyze(ctx context.Context, m *Model) (map[string]OpAnalysis, error)
+
+		// Aggregate returns the iterator with aggregated data from the base model
+		Aggregate(ctx context.Context, m *Model, f filter.Filter, groupBy []AggregateAttr, aggrExpr []AggregateAttr, having *ql.ASTNode) (i Iterator, _ error)
+
+		// Delete deletes the given value
+		Delete(ctx context.Context, m *Model, pkv ValueGetter) error
+
+		// Truncate deletes all the data for the given model
+		Truncate(ctx context.Context, m *Model) error
+
+		// DDL stuff
+
+		// // returns all attribute types that driver supports
+		// AttributeTypes() []data.AttributeType
+
+		// CreateModel adds support for the given models to the underlying database
+		//
+		// The operation returns an error if any of the models already exists.
+		CreateModel(context.Context, ...*Model) error
+
+		// DeleteModel removes support for the given model from the underlying database
+		DeleteModel(context.Context, ...*Model) error
+
+		// UpdateModel requests for metadata changes to the existing model
+		//
+		// Only metadata (such as idents) are affected; attributes can not be changed here
+		UpdateModel(ctx context.Context, old *Model, new *Model) error
+
+		// AssertSchemaAlterations returns a new set of Alterations based on what the underlying
+		// schema already provides -- it discards alterations for column additions that already exist, etc.
+		AssertSchemaAlterations(ctx context.Context, sch *Model, aa ...*Alteration) ([]*Alteration, error)
+
+		// ApplyAlteration applies the given alterations to the underlying schema
+		ApplyAlteration(ctx context.Context, sch *Model, aa ...*Alteration) []error
+	}
+
+	ConnectionCloser interface {
+		// Close closes the store connection allowing the driver to perform potential
+		// cleanup operations
+		Close(ctx context.Context) error
+	}
+
+	// Store provides an interface which CRS uses to interact with the underlying database
+
+	ValueGetter interface {
+		CountValues() map[string]uint
+		GetValue(string, uint) (any, error)
+	}
+
+	ValueSetter interface {
+		SetValue(string, uint, any) error
+	}
+
+	ConnectorFn func(ctx context.Context, dsn string) (Connection, error)
+
+	DriverConnectionParam struct {
+		Key        string `json:"key"`
+		ValueType  string `json:"valueType"`
+		MultiValue bool   `json:"multiValue"`
+	}
+
+	DriverConnectionConfig struct {
+		Type   string                  `json:"type"`
+		Params []DriverConnectionParam `json:"params"`
+	}
+
+	Driver struct {
+		Type       string                 `json:"type"`
+		Connection DriverConnectionConfig `json:"connection"`
+		Operations OperationSet           `json:"operations"`
+	}
+)
+
+var (
+	registeredConnectors = make(map[string]ConnectorFn)
+	registeredDrivers    = make(map[string]Driver)
+)
+
+func (pkv PKValues) CountValues() map[string]uint {
+	c := make(map[string]uint)
+	for k := range pkv {
+		c[k] = 1
+	}
+
+	return c
+}
+
+func (pkv PKValues) GetValue(key string, _ uint) (any, error) {
+	if val, has := pkv[key]; has {
+		return val, nil
+	} else {
+		return nil, fmt.Errorf("missing")
+	}
+}
+
+// RegisterConnector registers a new connector for the given DSN schema
+//
+// In case of a duplicate schema the latter overwrites the prior
+func RegisterConnector(fn ConnectorFn, tt ...string) {
+	for _, t := range tt {
+		registeredConnectors[t] = fn
+	}
+}
+
+func RegisterDriver(d Driver) {
+	registeredDrivers[d.Type] = d
+}
+
+// connect opens a new StoreConnection for the given CRS
+func connect(ctx context.Context, log *zap.Logger, isDevelopment bool, cp ConnectionParams) (Connection, error) {
+	if cp.Type != "corteza::dal:connection:dsn" {
+		return nil, fmt.Errorf("cannot open connection: only DSN connections supported (got: %q)", cp.Type)
+	}
+	if cp.Params == nil {
+		return nil, fmt.Errorf("cannot open connection: connection parameters not defined")
+	}
+	if _, ok := cp.Params["dsn"]; !ok {
+		return nil, fmt.Errorf("cannot open connection: DSN not provided")
+	}
+
+	dsn := cp.Params["dsn"].(string)
+
+	if isDevelopment {
+		if strings.Contains(dsn, "{version}") {
+			log.Warn("You're using DB_DSN with {version}, It is still in EXPERIMENTAL phase")
+			log.Warn("Should be used only for development mode")
+			log.Warn("You may experience instability")
+		}
+		expr := regexp.MustCompile(`[.\-]+`)
+		version := expr.ReplaceAllString(os.Getenv("BUILD_VERSION"), "_")
+		dsn = strings.Replace(dsn, "{version}", version, 1)
+	}
+
+	var storeType = strings.SplitN(dsn, "://", 2)[0]
+	if storeType == "" {
+		// Backward compatibility
+		storeType = "mysql"
+	}
+
+	if conn, ok := registeredConnectors[storeType]; ok {
+		return conn(ctx, dsn)
+	} else {
+		return nil, fmt.Errorf("unknown store type used: %q (check your database configuration)", storeType)
+	}
+}
+
+func NewDSNDriverConnectionConfig() DriverConnectionConfig {
+	return DriverConnectionConfig{
+		Type: "corteza::dal:connection:dsn",
+		Params: []DriverConnectionParam{{
+			Key:       "dsn",
+			ValueType: "string",
+		}},
+	}
+}
+
+//func NewHTTPDriverConnectionConfig() DriverConnectionConfig {
+//	panic("not implemented NewHTTPDriverConnectionConfig")
+//	return DriverConnectionConfig{
+//		Type:   "corteza::dal:connection:http",
+//		Params: []DriverConnectionParam{{}},
+//	}
+//}
+//func NewFederatedNodeDriverConnectionConfig() DriverConnectionConfig {
+//	panic("not implemented NewFederatedNodeDriverConnectionConfig")
+//	return DriverConnectionConfig{
+//		Type:   "corteza::dal:connection:federated-node",
+//		Params: []DriverConnectionParam{{}},
+//	}
+//}
+
+//func NewDSNConnection(dsn string) ConnectionParams {
+//	return ConnectionParams{
+//		Type: "corteza::dal:connection:dsn",
+//		Params: map[string]any{
+//			"dsn": dsn,
+//		},
+//	}
+//}
+
+//func NewHTTPConnection(url string, headers, query map[string][]string) ConnectionParams {
+//	return ConnectionParams{
+//		Type: "corteza::dal:connection:http",
+//		Params: map[string]any{
+//			"url":     url,
+//			"headers": headers,
+//			"query":   query,
+//		},
+//	}
+//}
+//
+//func NewFederatedNodeConnection(url string, pairToken, authToken string) ConnectionParams {
+//	return ConnectionParams{
+//		Type: "corteza::dal:connection:federation-node",
+//		Params: map[string]any{
+//			"baseURL":   url,
+//			"pairToken": pairToken,
+//			"authToken": authToken,
+//		},
+//	}
+//}

server/pkg/dal/errors.go

@@ -0,0 +1,166 @@
+package dal
+
+import (
+	"fmt"
+
+	"github.com/cortezaproject/corteza/server/pkg/errors"
+)
+
+// Generic errors
+
+func errModelNotFound(modelID uint64) error {
+	return errors.NotFound("model %d does not exist", modelID)
+}
+
+func errConnectionNotFound(connectionID uint64) error {
+	return errors.NotFound("connection %d does not exist", connectionID)
+}
+
+// Sensitivity level errors
+// - remove
+func errSensitivityLevelRemoveNotFound(sensitivityLevelID uint64) error {
+	return fmt.Errorf("cannot remove sensitivity level %d: sensitivity level does not exist", sensitivityLevelID)
+}
+
+// Connection errors
+// - create
+func errConnectionCreateMissingSensitivityLevel(connectionID, sensitivityLevelID uint64) error {
+	return fmt.Errorf("cannot create connection %d: sensitivity level does not exist %d", connectionID, sensitivityLevelID)
+}
+func errConnectionCreateConnectionFailed(connectionID uint64, err error) error {
+	return fmt.Errorf("cannot create connection %d: connection failed: %v", connectionID, err)
+}
+func errConnectionDeleteNotFound(connectionID uint64) error {
+	return fmt.Errorf("cannot delete connection %d: connection does not exist", connectionID)
+}
+func errConnectionDeleteCloserFailed(connectionID uint64, err error) error {
+	return fmt.Errorf("cannot delete connection %d: connection's driver failed to close: %v", connectionID, err)
+}
+
+// - update
+func errConnectionUpdateNotFound(connectionID uint64) error {
+	return fmt.Errorf("cannot update connection %d: connection does not exist", connectionID)
+}
+func errConnectionUpdateMissingSensitivityLevel(connectionID, sensitivityLevelID uint64) error {
+	return fmt.Errorf("cannot update connection %d: sensitivity level %d does not exist", connectionID, sensitivityLevelID)
+}
+
+// Model errors
+// - create
+func errModelCreateProblematicConnection(connectionID, modelID uint64) error {
+	return fmt.Errorf("cannot create model %d on connection %d: connection has issues", modelID, connectionID)
+}
+func errModelCreateMissingConnection(connectionID, modelID uint64) error {
+	return fmt.Errorf("cannot create model %d on connection %d: connection does not exist", modelID, connectionID)
+}
+func errModelCreateMissingSensitivityLevel(connectionID, modelID, sensitivityLevelID uint64) error {
+	return fmt.Errorf("cannot create model %d on connection %d: sensitivity level %d does not exist", modelID, connectionID, sensitivityLevelID)
+}
+func errModelCreateGreaterSensitivityLevel(connectionID, modelID, modelSensitivityLevelID, connSensitivityLevelID uint64) error {
+	return fmt.Errorf("cannot create model %d on connection %d: sensitivity level %d exceeds connection supported sensitivity level %d", modelID, connectionID, modelSensitivityLevelID, connSensitivityLevelID)
+}
+func errModelCreateMissingAttributeSensitivityLevel(connectionID, modelID, sensitivityLevelID uint64) error {
+	return fmt.Errorf("cannot create model %d on connection %d: attribute sensitivity level %d does not exist", modelID, connectionID, sensitivityLevelID)
+}
+func errModelCreateGreaterAttributeSensitivityLevel(connectionID, modelID, attrSensitivityLevelID, modelSensitivityLevelID uint64) error {
+	return fmt.Errorf("cannot create model %d on connection %d: attribute sensitivity level %d exceeds model supported sensitivity level %d", modelID, connectionID, attrSensitivityLevelID, modelSensitivityLevelID)
+}
+func errModelCreateConnectionModelUnsupported(connectionID, modelID uint64) error {
+	return fmt.Errorf("cannot create model %d on connection %d: model already exists for connection but is not compatible with provided definition", modelID, connectionID)
+}
+func errModelCreateInvalidIdent(connectionID, modelID uint64, ident string) error {
+	return fmt.Errorf("cannot create model %d on connection %d: malformed model ident %s", modelID, connectionID, ident)
+}
+
+// - update
+func errModelUpdateProblematicConnection(connectionID, modelID uint64) error {
+	return fmt.Errorf("cannot update model %d on connection %d: connection has issues", modelID, connectionID)
+}
+func errModelUpdateMissingConnection(connectionID, modelID uint64) error {
+	return fmt.Errorf("cannot update model %d on connection %d: connection does not exist", modelID, connectionID)
+}
+func errModelUpdateConnectionModelUnsupported(connectionID, modelID uint64) error {
+	return fmt.Errorf("cannot update model %d on connection %d: model already exists for connection but is not compatible with provided definition", modelID, connectionID)
+}
+func errModelUpdateMissingOldModel(connectionID, modelID uint64) error {
+	return fmt.Errorf("cannot update model %d on connection %d: model does not exist", modelID, connectionID)
+}
+func errModelUpdateDuplicate(connectionID, modelID uint64) error {
+	return fmt.Errorf("cannot update model %d on connection %d: model already exists", modelID, connectionID)
+}
+func errModelUpdateConnectionMissmatch(connectionID, modelID uint64) error {
+	return fmt.Errorf("cannot update model %d on connection %d: cannot change model connection", modelID, connectionID)
+}
+func errModelUpdateMissingSensitivityLevel(connectionID, modelID, sensitivityLevelID uint64) error {
+	return fmt.Errorf("cannot update model %d on connection %d: sensitivity level %d does not exist", modelID, connectionID, sensitivityLevelID)
+}
+func errModelUpdateGreaterSensitivityLevel(connectionID, modelID, modelSensitivityLevelID, connSensitivityLevelID uint64) error {
+	return fmt.Errorf("cannot update model %d on connection %d: sensitivity level %d exceeds connection supported sensitivity level %d", modelID, connectionID, modelSensitivityLevelID, connSensitivityLevelID)
+}
+
+// - alterations
+func errModelRequiresAlteration(connectionID, modelID, batchID uint64) error {
+	return fmt.Errorf("model %d on connection %d requires schema alterations: alteration batchID %d", modelID, connectionID, batchID)
+}
+
+// Attribute errors
+// - Update
+func errAttributeUpdateProblematicConnection(connectionID, modelID uint64) error {
+	return fmt.Errorf("cannot update attribute for model %d on connection %d: connection has issues", modelID, connectionID)
+}
+func errAttributeUpdateMissingModel(connectionID, modelID uint64) error {
+	return fmt.Errorf("cannot update attribute for model %d on connection %d: model does not exist", modelID, connectionID)
+}
+func errAttributeUpdateMissingSensitivityLevel(connectionID, modelID, sensitivityLevelID uint64) error {
+	return fmt.Errorf("cannot update attribute for model %d on connection %d: sensitivity level %d does not exist", modelID, connectionID, sensitivityLevelID)
+}
+func errAttributeUpdateGreaterSensitivityLevel(connectionID, modelID, attrSensitivityLevelID, modelSensitivityLevelID uint64) error {
+	return fmt.Errorf("cannot update attribute for model %d on connection %d: sensitivity level %d exceeds model supported sensitivity level %d", modelID, connectionID, attrSensitivityLevelID, modelSensitivityLevelID)
+}
+
+// Record errors
+
+func errRecordOpProblematicConnection(connectionID uint64) error {
+	return fmt.Errorf("cannot perform record operation: connection %d has issues", connectionID)
+}
+func errRecordOpProblematicModel(modelID uint64) error {
+	return fmt.Errorf("cannot perform record operation: model %d has issues", modelID)
+}
+
+// func errModelHigherSensitivity(model, connection string) error {
+// 	return errors.New(
+// 		errors.KindSensitiveData,
+
+// 		"model sensitivity surpasses connection sensitivity",
+
+// 		errors.Meta("type", "invalid sensitivity"),
+
+// 		// Translation namespace & key
+// 		errors.Meta(locale.ErrorMetaNamespace{}, "internal"),
+// 		errors.Meta(locale.ErrorMetaKey{}, "dal.sensitivity.model-exceeds-connection"),
+// 		errors.Meta("model", model),
+// 		errors.Meta("connection", connection),
+
+// 		errors.StackSkip(1),
+// 		errors.StackTrimAtFn("http.HandlerFunc.ServeHTTP"),
+// 	)
+// }
+
+// func errAttributeHigherSensitivity(model, attribute string) error {
+// 	return errors.New(
+// 		errors.KindSensitiveData,
+
+// 		"attribute sensitivity surpasses model sensitivity",
+
+// 		errors.Meta("type", "invalid sensitivity"),
+
+// 		// Translation namespace & key
+// 		errors.Meta(locale.ErrorMetaNamespace{}, "internal"),
+// 		errors.Meta(locale.ErrorMetaKey{}, "dal.sensitivity.attribute-exceeds-model"),
+// 		errors.Meta("model", model),
+// 		errors.Meta("attribute", attribute),
+
+// 		errors.StackSkip(1),
+// 		errors.StackTrimAtFn("http.HandlerFunc.ServeHTTP"),
+// 	)
+// }

server/pkg/dal/exec_aggregate.go

@@ -0,0 +1,589 @@
+package dal
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/cortezaproject/corteza/server/pkg/ql"
+	"github.com/tidwall/btree"
+)
+
+type (
+	aggregate struct {
+		def    Aggregate
+		filter internalFilter
+
+		source  Iterator
+		err     error
+		scanRow *Row
+		planned bool
+
+		groupDefs     []AggregateAttr
+		aggregateDefs []AggregateAttr
+
+		rowTester tester
+		keyWalker keyWalker
+
+		// Keep track of the registered groupIndex so we can easier implement the iterator.
+		// Simple index tracking won't be enough, because btree is self-balancing -- things change
+		//
+		// @todo worth considering using the inmembuffer and fork it around...
+		groupIndex *btree.Generic[*aggregateGroup]
+		groups     []*aggregateGroup
+		i          int
+
+		ctr int
+	}
+
+	keyWalker func(context.Context, ValueGetter, func(context.Context, groupKey, ValueGetter) error) error
+)
+
+// init initializes the execution step's state
+func (xs *aggregate) init(ctx context.Context) (err error) {
+	// Initialize state variables
+	xs.scanRow = xs.initScanRow()
+	xs.groupIndex = btree.NewGeneric[*aggregateGroup](xs.compareGroupKeys)
+	xs.groups = make([]*aggregateGroup, 0, 128)
+	xs.rowTester, err = prepareGenericRowTester(xs.filter)
+	if err != nil {
+		return
+	}
+
+	// Initialize the key maker
+	kk := make([]*ql.ASTNode, 0, len(xs.groupDefs))
+	for _, a := range xs.groupDefs {
+		kk = append(kk, a.Expression)
+	}
+
+	xs.keyWalker, err = aggregateGroupKeyWalker(kk...)
+	if err != nil {
+		return
+	}
+
+	// Apply the decided aggregation plan
+	return xs.applyPlan(ctx)
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Iterator methods
+
+func (xs *aggregate) Next(ctx context.Context) (more bool) {
+	// Assure aggregation plan was taken into account
+	xs.err = xs.applyPlan(ctx)
+	if xs.err != nil {
+		return false
+	}
+
+	// Go next...
+	more, xs.err = xs.next(ctx)
+	return
+}
+
+// next prepares the next scannable row
+//
+// The method is adjusted based on the defined aggregation plan.
+// The method performs appropriate filtering/sorting (if needed).
+func (xs *aggregate) next(ctx context.Context) (more bool, err error) {
+	var g *aggregateGroup
+	for {
+		if xs.limitExceeded() {
+			return false, nil
+		}
+
+		// Make sure it's cleared out and ready for fresh data
+		xs.scanRow.Reset()
+
+		// Make sure we have data
+		err = xs.prepareNext(ctx)
+		if err != nil {
+			return false, err
+		}
+
+		// Next group
+		g, err = xs.nextGroup(ctx)
+		if err != nil || g == nil {
+			return false, err
+		}
+
+		// Make sure the key is in there
+		// @todo potential optimization where we don't scan entire composited keys
+		err = xs.scanKey(g, xs.scanRow)
+		if err != nil {
+			return
+		}
+		// Scan the data
+		err = g.agg.Scan(xs.scanRow)
+		if err != nil {
+			return
+		}
+
+		// Check if we want to keep it
+		k, err := xs.keep(ctx, xs.scanRow)
+		if err != nil {
+			return false, err
+		}
+		if !k {
+			continue
+		}
+
+		xs.ctr++
+		break
+	}
+
+	return true, nil
+}
+
+func (xs *aggregate) limitExceeded() bool {
+	return xs.filter.limit > 0 && xs.ctr >= int(xs.filter.limit)
+}
+
+func (xs *aggregate) More(limit uint, v ValueGetter) (err error) {
+	// Redo the cursor
+	xs.filter.cursor, err = filter.PagingCursorFrom(xs.filter.OrderBy(), v, xs.collectPrimaryAttributes()...)
+	if err != nil {
+		return
+	}
+
+	// Redo the row tester
+	xs.rowTester, err = prepareGenericRowTester(xs.filter)
+	if err != nil {
+		return
+	}
+
+	// Redo the state
+	// @todo adjust based on aggregation plan; reuse buffered, etc.
+	xs.scanRow.Reset()
+	xs.groupIndex = btree.NewGeneric[*aggregateGroup](xs.compareGroupKeys)
+	xs.groups = make([]*aggregateGroup, 0, 128)
+	xs.planned = false
+	xs.i = 0
+
+	return
+}
+
+func (s *aggregate) Err() error { return s.err }
+
+func (s *aggregate) Scan(dst ValueSetter) (err error) {
+	if s.i < 0 {
+		return fmt.Errorf("@todo err not initialized; next first")
+	}
+
+	var v any
+	for name, cc := range s.scanRow.CountValues() {
+		for i := uint(0); i < cc; i++ {
+			// omitting err here since it won't happen
+			v, _ = s.scanRow.GetValue(name, i)
+			err = dst.SetValue(name, i, v)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (s *aggregate) Close() error {
+	if s.source != nil {
+		return s.source.Close()
+	}
+	return nil
+}
+
+func (s *aggregate) BackCursor(v ValueGetter) (*filter.PagingCursor, error) {
+	c, err := filter.PagingCursorFrom(s.filter.OrderBy(), v, s.collectPrimaryAttributes()...)
+	if err != nil {
+		return nil, err
+	}
+
+	c.ROrder = true
+	c.LThen = s.filter.OrderBy().Reversed()
+
+	return c, nil
+}
+
+func (s *aggregate) ForwardCursor(v ValueGetter) (*filter.PagingCursor, error) {
+	c, err := filter.PagingCursorFrom(s.filter.OrderBy(), v, s.collectPrimaryAttributes()...)
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Utilities
+
+// applyPlan runs additional initialization code based on the defined aggregation plan
+func (xs *aggregate) applyPlan(ctx context.Context) (err error) {
+	if xs.planned || xs.err != nil {
+		return xs.err
+	}
+
+	xs.planned = true
+	switch {
+	case !xs.def.plan.partialScan:
+		return xs.pullEntireSource(ctx)
+	}
+
+	return
+}
+
+func (xs *aggregate) prepareNext(ctx context.Context) (err error) {
+	// Pull next chunk from source if not entirely buffered
+	if xs.def.plan.partialScan {
+		err = xs.pullNextChunk(ctx)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+// pullEntireSource pulls the entire source into aggregators
+//
+// The entire source should be pulled when underlaying datasources can't provide
+// more appropriately ordered data due to costs.
+func (xs *aggregate) pullEntireSource(ctx context.Context) (err error) {
+	if xs.source == nil {
+		return fmt.Errorf("unable to pull data: no source defined")
+	}
+
+	// @todo consider pre-populating the hashmaps
+	r := &Row{
+		counters: make(map[string]uint),
+		values:   make(valueSet),
+	}
+
+	// Drain the source
+	for xs.source.Next(ctx) {
+		err = xs.source.Scan(r)
+		if err != nil {
+			return
+		}
+
+		// Get the key for this row
+		// @todo we probably can reuse the key or at least cache keys and avoid re-computation.
+		//       My fairly hacky attempt boosted performance by ~20%
+		err = xs.keyWalker(ctx, r, xs.addToGroup)
+		if err != nil {
+			return
+		}
+
+		r.Reset()
+	}
+	xs.err = xs.source.Err()
+	if xs.err != nil {
+		return xs.err
+	}
+
+	xs.sortGroups()
+
+	return
+}
+
+// pullNextChunk pulls the next chunk into the aggregators
+//
+// Source should be pulled in chunks when the underlaying datasource can provide
+// more appropriately ordered data.
+func (xs *aggregate) pullNextChunk(ctx context.Context) (err error) {
+	return fmt.Errorf("not implemented")
+}
+
+func (s *aggregate) addToGroup(ctx context.Context, key groupKey, r ValueGetter) (err error) {
+	if s.groupIndex == nil {
+		s.groupIndex = btree.NewGeneric[*aggregateGroup](s.compareGroupKeys)
+	}
+
+	// Try to get the existing one; if it doesn't exist, push a new one
+	// @todo this causes a bit of a memory pressure; investigate
+	g, ok := s.groupIndex.Get(&aggregateGroup{key: key})
+	if !ok {
+		g, err = s.wrapGroup(ctx, key)
+		if err != nil {
+			return err
+		}
+		s.groupIndex.Set(g)
+
+		s.groups = append(s.groups, g)
+	}
+
+	// if it exists/was added, add it to the aggregator
+	err = g.agg.Aggregate(ctx, r)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// Group key comparator
+// @todo can this be moved to the struct's Less method?
+func (t *aggregate) compareGroupKeys(a, b *aggregateGroup) (out bool) {
+	for i := range a.key {
+		va := a.key[i]
+		vb := b.key[i]
+		out = out || compareValues(va, vb) < 0
+	}
+
+	return
+}
+
+func (s *aggregate) wrapGroup(ctx context.Context, key groupKey) (g *aggregateGroup, err error) {
+	agg := Aggregator()
+
+	for _, a := range s.aggregateDefs {
+		err = agg.AddAggregate(a.Identifier, a.Expression)
+		if err != nil {
+			return
+		}
+	}
+
+	g = &aggregateGroup{
+		key: key,
+		agg: agg,
+	}
+
+	return
+}
+
+func (xs *aggregate) nextGroup(ctx context.Context) (_ *aggregateGroup, err error) {
+	if xs.i >= len(xs.groups) {
+		return
+	}
+
+	xs.i++
+	return xs.groups[xs.i-1], nil
+}
+
+func (xs *aggregate) scanKey(g *aggregateGroup, dst *Row) (err error) {
+	for i, attr := range xs.groupDefs {
+		// @todo multi value support?
+		// omitting err; internal row won't raise them
+		dst.SetValue(attr.Identifier, 0, g.key[i])
+	}
+
+	return nil
+}
+
+func (s *aggregate) keep(ctx context.Context, r *Row) (bool, error) {
+	if s.rowTester == nil {
+		return true, nil
+	}
+	return s.rowTester.Test(ctx, r)
+}
+
+// Each group key is a PK candidate; all together form a composite key
+func (s *aggregate) collectPrimaryAttributes() (out []string) {
+	out = make([]string, 0, 2)
+	for _, m := range s.def.Group {
+		out = append(out, m.Identifier)
+	}
+
+	return
+}
+
+func (s *aggregate) sortGroups() {
+	sort.SliceStable(s.groups, func(i, j int) bool {
+		ga := s.groups[i]
+		gb := s.groups[j]
+
+		var (
+			va any
+			vb any
+		)
+
+		for _, o := range s.filter.OrderBy() {
+			x := inKeys(s.def.Group, o.Column)
+			if x > -1 {
+				va = ga.key[x]
+			} else {
+				x := inKeys(s.def.OutAttributes, o.Column)
+				va = ga.agg.aggregates[x]
+			}
+
+			x = inKeys(s.def.Group, o.Column)
+			if x > -1 {
+				vb = gb.key[x]
+			} else {
+				x := inKeys(s.def.OutAttributes, o.Column)
+				vb = gb.agg.aggregates[x]
+			}
+
+			cmp := compareValues(va, vb)
+			if cmp != 0 {
+				if o.Descending {
+					return cmp > 0
+				}
+				return cmp < 0
+			}
+		}
+		return false
+	})
+}
+
+func inKeys(kk []AggregateAttr, ident string) int {
+	for i, k := range kk {
+		if k.Identifier == ident {
+			return i
+		}
+	}
+	return -1
+}
+
+func (xs *aggregate) initScanRow() (out *Row) {
+	// base
+	out = &Row{
+		counters: make(map[string]uint),
+		values:   make(valueSet),
+	}
+
+	// pre-populate with known attrs
+	for _, attr := range append(xs.def.Group, xs.def.OutAttributes...) {
+		out.values[attr.Identifier] = make([]any, 0, 2)
+		out.counters[attr.Identifier] = 0
+	}
+
+	return
+}
+
+// aggregateGroupKeyWalker prepares a function which runs the provided function over all group keys
+//
+// This is required to handle multi-value attributes when used in group keys.
+//
+// Algorithm TL;DR
+//
+// We're going backwards in the slice of idents we need to handle (going the other way around)
+// should also work but I chose to do it like so.
+//
+// For every ident, keep track of the number of elements and the index of the last unhandled ident.
+//
+// For every iteration, collect all of the values pointed to by the unused index.
+// After the key is processed, increment the counter for the current ident.
+// If the counter exceeds the number of elements, reset the counter for the current ident and move
+// the ident pointer backwards (repeat if that ident's counter also exceeds the limit).
+//
+// When we find an indent which still has some items to process (counter doesn't exceed limit),
+// reset the ident pointer to the end of the slice and repeat the whole thing.
+func aggregateGroupKeyWalker(kk ...*ql.ASTNode) (out keyWalker, err error) {
+	// @todo option to copy constants and idents
+	runners, err := makeExprRunners(kk...)
+	if err != nil {
+		return
+	}
+
+	// We'll sort the idents to keep the output consistent; the order doesn't matter
+	// but it will simplify testing.
+	idents, hasConstants := keysFromExpr(kk...)
+	sort.Strings(idents)
+
+	out = func(ctx context.Context, vg ValueGetter, run func(context.Context, groupKey, ValueGetter) error) error {
+		// Edgecase for when all of the expressions return constant values
+		if len(idents) == 0 {
+			// This should be impossible but better safe then sorry
+			if !hasConstants {
+				return nil
+			}
+
+			k, err := makeGroupKey(ctx, runners, vg)
+			if err != nil {
+				return err
+			}
+
+			return run(ctx, k, vg)
+		}
+
+		// For every value combination of idents used in agg. key expressions
+		// construct a key and run the runner.
+
+		limits := vg.CountValues()
+		ptr := len(idents) - 1
+		counts := make(map[string]uint, len(limits))
+		for k := range limits {
+			counts[k] = 0
+		}
+
+		handle := func() (err error) {
+			aux := make(map[string]any, len(limits))
+
+			for k, i := range counts {
+				aux[k], err = vg.GetValue(k, i)
+				if err != nil {
+					return
+				}
+			}
+
+			kk, err := makeGroupKey(ctx, runners, aux)
+			if err != nil {
+				return err
+			}
+
+			return run(ctx, kk, vg)
+		}
+
+	outer:
+		for ptr >= 0 {
+			handle()
+
+			counts[idents[ptr]]++
+			for {
+				// There are still values to process so we can skip the rest
+				if counts[idents[ptr]] < limits[idents[ptr]] {
+					continue outer
+				}
+
+				// Reset the counter for the current ptr since we'll move back.
+				// The outer loop will have this reset all of the counters after the current ptr.
+				counts[idents[ptr]] = 0
+
+				ptr--
+				if ptr < 0 {
+					break outer
+				}
+
+				counts[idents[ptr]]++
+
+				if counts[idents[ptr]] >= limits[idents[ptr]] {
+					continue
+				}
+
+				// We need to reset to the end so the next ident gets all of the values
+				// that appear after it.
+				ptr = len(idents) - 1
+				break
+			}
+		}
+
+		return nil
+	}
+
+	return
+}
+
+func makeExprRunners(kk ...*ql.ASTNode) (out []*runnerGval, err error) {
+	out = make([]*runnerGval, len(kk))
+
+	for i, k := range kk {
+		out[i], err = newRunnerGvalParsed(k)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func makeGroupKey(ctx context.Context, runners []*runnerGval, vals any) (gk groupKey, err error) {
+	gk = make(groupKey, len(runners))
+	for i, r := range runners {
+		v, err := r.Eval(ctx, vals)
+		if err != nil {
+			return nil, err
+		}
+
+		gk[i] = v
+	}
+	return gk, nil
+}

server/pkg/dal/exec_aggregate_bench_test.go

@@ -0,0 +1,83 @@
+package dal
+
+import (
+	"context"
+	"math/rand"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func benchmarkExecAggregate(b *testing.B, n int) {
+	b.StopTimer()
+	ctx := context.Background()
+	group := []simpleAttribute{{ident: "g", expr: "g"}}
+	outAttributes := []simpleAttribute{{ident: "sum", expr: "sum(s1_v)"}}
+	sourceAttributes := []simpleAttribute{
+		{ident: "k"},
+		{ident: "g"},
+		{ident: "s1_v"},
+	}
+
+	ggs := []string{"a", "b", "c", "d", "e", "f", "g"}
+
+	// Inmem buffer for example
+	buff := InMemoryBuffer()
+	for i := 0; i < n; i++ {
+		require.NoError(b, buff.Add(ctx, simpleRow{"k": i + 1, "g": ggs[rand.Intn(len(ggs))], "s1_v": rand.Intn(200)}))
+	}
+
+	b.ResetTimer()
+
+	for n := 0; n < b.N; n++ {
+		def := Aggregate{
+			Ident:            "agg",
+			Group:            saToAggAttr(group...),
+			OutAttributes:    saToAggAttr(outAttributes...),
+			SourceAttributes: saToMapping(sourceAttributes...),
+		}
+
+		b.StartTimer()
+		_, err := def.iterator(ctx, buff)
+		require.NoError(b, err)
+		b.StopTimer()
+
+		buff.Seek(ctx, 0)
+	}
+}
+
+// goos: linux
+// goarch: amd64
+// pkg: github.com/cortezaproject/corteza/server/pkg/dal
+// cpu: Intel(R) Core(TM) i7-8750H CPU @ 2.20GHz
+// BenchmarkExecAggregate_20000-12     	      63	  17902131 ns/op	 1423343 B/op	   80432 allocs/op
+// BenchmarkExecAggregate_40000-12     	      33	  36179481 ns/op	 2783435 B/op	  160432 allocs/op
+// BenchmarkExecAggregate_60000-12     	      21	  53681403 ns/op	 4143471 B/op	  240433 allocs/op
+// BenchmarkExecAggregate_80000-12     	      15	  72234005 ns/op	 5503370 B/op	  320432 allocs/op
+// BenchmarkExecAggregate_100000-12    	      12	  92145195 ns/op	 6863674 B/op	  400434 allocs/op
+// BenchmarkExecAggregate_120000-12    	      10	 109842507 ns/op	 8223711 B/op	  480433 allocs/op
+// BenchmarkExecAggregate_140000-12    	       8	 129250726 ns/op	 9583616 B/op	  560433 allocs/op
+// BenchmarkExecAggregate_160000-12    	       7	 144896012 ns/op	10943776 B/op	  640434 allocs/op
+// BenchmarkExecAggregate_180000-12    	       7	 162705594 ns/op	12303718 B/op	  720434 allocs/op
+// BenchmarkExecAggregate_200000-12    	       6	 181970479 ns/op	13663848 B/op	  800434 allocs/op
+// BenchmarkExecAggregate_220000-12    	       5	 204168375 ns/op	15024054 B/op	  880435 allocs/op
+// BenchmarkExecAggregate_240000-12    	       5	 218573247 ns/op	16383470 B/op	  960432 allocs/op
+// BenchmarkExecAggregate_260000-12    	       5	 238612939 ns/op	17744016 B/op	 1040435 allocs/op
+// BenchmarkExecAggregate_280000-12    	       4	 261442973 ns/op	19103558 B/op	 1120433 allocs/op
+// BenchmarkExecAggregate_300000-12    	       4	 342276668 ns/op	20464244 B/op	 1200436 allocs/op
+
+func BenchmarkExecAggregate_20000(b *testing.B)  { benchmarkExecAggregate(b, 20000) }
+func BenchmarkExecAggregate_40000(b *testing.B)  { benchmarkExecAggregate(b, 40000) }
+func BenchmarkExecAggregate_60000(b *testing.B)  { benchmarkExecAggregate(b, 60000) }
+func BenchmarkExecAggregate_80000(b *testing.B)  { benchmarkExecAggregate(b, 80000) }
+func BenchmarkExecAggregate_100000(b *testing.B) { benchmarkExecAggregate(b, 100000) }
+func BenchmarkExecAggregate_120000(b *testing.B) { benchmarkExecAggregate(b, 120000) }
+func BenchmarkExecAggregate_140000(b *testing.B) { benchmarkExecAggregate(b, 140000) }
+func BenchmarkExecAggregate_160000(b *testing.B) { benchmarkExecAggregate(b, 160000) }
+func BenchmarkExecAggregate_180000(b *testing.B) { benchmarkExecAggregate(b, 180000) }
+func BenchmarkExecAggregate_200000(b *testing.B) { benchmarkExecAggregate(b, 200000) }
+func BenchmarkExecAggregate_220000(b *testing.B) { benchmarkExecAggregate(b, 220000) }
+func BenchmarkExecAggregate_240000(b *testing.B) { benchmarkExecAggregate(b, 240000) }
+func BenchmarkExecAggregate_260000(b *testing.B) { benchmarkExecAggregate(b, 260000) }
+func BenchmarkExecAggregate_280000(b *testing.B) { benchmarkExecAggregate(b, 280000) }
+func BenchmarkExecAggregate_300000(b *testing.B) { benchmarkExecAggregate(b, 300000) }

server/pkg/dal/exec_join_left.go

@@ -0,0 +1,417 @@
+package dal
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/tidwall/btree"
+)
+
+type (
+	// Considerations for optimizations
+	// - Rework how values are stored internally.
+	//   Potentially rework the pipeline input/output interfaces, how ValueGetter/Setter work.
+	// - Skip initial left row scan/sort when sorting not requested.
+	// - Rework row struct to use slices instead of hashmaps.
+	//   With how values are now the performance gain is not that impactful and the
+	//   rework complexity is a bit too high for now.
+	// - Use `.More` to filter rows by key.
+	// - When data is provided in a satisfactory order, use that to pull data in chunks.
+	joinLeft struct {
+		def    Join
+		filter internalFilter
+
+		leftSource  Iterator
+		rightSource Iterator
+		err         error
+		scanRow     ValueGetter
+		planned     bool
+		filtered    bool
+
+		rowTester tester
+
+		// Index to keep track of related rows
+		relIndex *relIndex
+
+		// Output placeholder for sorted rows
+		// @todo consider a generic slice for cases when sorting is not needed.
+		//       This will probably save up on memory/time since we don't even need
+		//       to pull everything.
+		outSorted *btree.Generic[ValueGetter]
+		i         int
+	}
+)
+
+func (xs *joinLeft) init(ctx context.Context, joinPredType Type) (err error) {
+	xs.relIndex, err = newRelIndex(joinPredType)
+	if err != nil {
+		return
+	}
+
+	xs.rowTester, err = prepareGenericRowTester(xs.filter)
+	if err != nil {
+		return
+	}
+
+	// @note careful here if you throw routines into the mix; see the NoLocks flag.
+	//       Enabling locks does have a performance impact so you might be better off by
+	//       constructing multiple of these but then you'll also need to complicate
+	//       the .Next methods a bit.
+	xs.outSorted = btree.NewGenericOptions[ValueGetter](makeRowComparator(xs.filter.OrderBy()...), btree.Options{NoLocks: true})
+
+	return xs.applyPlan(ctx)
+}
+
+func (xs *joinLeft) Next(ctx context.Context) (more bool) {
+	xs.err = xs.applyPlan(ctx)
+	if xs.err != nil {
+		return false
+	}
+
+	more, xs.err = xs.next(ctx)
+	return
+}
+
+func (xs *joinLeft) More(limit uint, v ValueGetter) (err error) {
+	xs.filter.cursor, err = filter.PagingCursorFrom(xs.filter.OrderBy(), v, xs.collectPrimaryAttributes(xs.def.OutAttributes)...)
+	if err != nil {
+		return
+	}
+
+	// Redo row tester
+	xs.rowTester, err = prepareGenericRowTester(xs.filter)
+	if err != nil {
+		return
+	}
+
+	// Redo the state
+	// @todo adjust based on aggregation plan; reuse buffered, etc.
+	xs.relIndex.Clear()
+	xs.outSorted = btree.NewGenericOptions[ValueGetter](makeRowComparator(xs.filter.OrderBy()...), btree.Options{NoLocks: true})
+	xs.scanRow = nil
+	xs.planned = false
+	xs.i = 0
+
+	return
+}
+
+func (xs *joinLeft) Err() error { return xs.err }
+
+func (xs *joinLeft) Scan(s ValueSetter) (err error) {
+	for k, cc := range xs.scanRow.CountValues() {
+		for i := uint(0); i < cc; i++ {
+			// @note internal row won't raise errors so we can safely omit them
+			v, _ := xs.scanRow.GetValue(k, i)
+			err = s.SetValue(k, i, v)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	return
+}
+
+func (xs *joinLeft) Close() (err error) {
+	if xs == nil {
+		return
+	}
+
+	cc := []io.Closer{
+		xs.leftSource,
+		xs.rightSource,
+	}
+
+	for _, c := range cc {
+		if c != nil {
+			err = c.Close()
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return
+}
+
+func (xs *joinLeft) BackCursor(v ValueGetter) (pc *filter.PagingCursor, err error) {
+	pc, err = filter.PagingCursorFrom(xs.filter.OrderBy(), v, xs.collectPrimaryAttributes(xs.def.OutAttributes)...)
+	if err != nil {
+		return nil, err
+	}
+
+	pc.ROrder = true
+	pc.LThen = xs.filter.OrderBy().Reversed()
+
+	return
+}
+
+func (xs *joinLeft) ForwardCursor(v ValueGetter) (pc *filter.PagingCursor, err error) {
+	pc, err = filter.PagingCursorFrom(xs.filter.OrderBy(), v, xs.collectPrimaryAttributes(xs.def.OutAttributes)...)
+	if err != nil {
+		return nil, err
+	}
+
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Utility methods
+
+// next prepares the next scan row based on the defined join plan
+func (xs *joinLeft) next(ctx context.Context) (more bool, err error) {
+	if xs.limitExceeded() {
+		return false, nil
+	}
+
+	more, err = xs.pullNext(ctx)
+	if !more || err != nil {
+		return more, err
+	}
+
+	xs.scanRow, _ = xs.outSorted.GetAt(xs.i)
+	xs.i++
+	return true, nil
+}
+
+func (xs *joinLeft) limitExceeded() bool {
+	return xs.filter.limit > 0 && xs.i >= int(xs.filter.limit)
+}
+
+// pullNext pulls additional data so we can produce more
+//
+// This step may be omitted based on the join plan.
+func (xs *joinLeft) pullNext(ctx context.Context) (more bool, err error) {
+	// Pull next chunk from source if not entirely buffered
+	if xs.def.plan.partialScan {
+		// @todo this case is currently not implemented so we're erroring it out
+		return false, fmt.Errorf("partialScan join plan strategy not implemented")
+	}
+
+	// Check if buffer has more
+	if xs.i >= xs.outSorted.Len() {
+		return false, nil
+	}
+	return true, nil
+}
+
+// applyPlan runs plan specific logic to prepare the state
+func (xs *joinLeft) applyPlan(ctx context.Context) (err error) {
+	if xs.planned || xs.err != nil {
+		return
+	}
+
+	xs.planned = true
+	switch {
+	case !xs.def.plan.partialScan:
+		return xs.pullEntireSource(ctx)
+	}
+
+	return
+}
+
+// pullEntireSource pulls both sources into memory and indexes them for later use
+func (xs *joinLeft) pullEntireSource(ctx context.Context) (err error) {
+	// This bit does the filtering so just mark it of as such here
+	xs.filtered = true
+
+	// First the right source
+	err = xs.pullEntireRightSource(ctx)
+	if err != nil {
+		return
+	}
+
+	// Next the left source
+	err = xs.pullEntireLeftSource(ctx)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// pullEntireRightSource pulls and indexes all of the right bits
+func (xs *joinLeft) pullEntireRightSource(ctx context.Context) (err error) {
+	for xs.rightSource.Next(ctx) {
+		r := &Row{
+			counters: make(map[string]uint),
+			values:   make(valueSet),
+		}
+
+		err = xs.rightSource.Scan(r)
+		if err != nil {
+			return
+		}
+
+		err = xs.indexRightRow(r)
+		if err != nil {
+			return
+		}
+	}
+	return xs.rightSource.Err()
+}
+
+// pullEntireLeftSource pulls left bits and attempts to do as much of the joining
+// work on this stage
+func (xs *joinLeft) pullEntireLeftSource(ctx context.Context) (err error) {
+	for xs.leftSource.Next(ctx) {
+		l := &Row{
+			counters: make(map[string]uint),
+			values:   make(valueSet),
+		}
+
+		err = xs.leftSource.Scan(l)
+		if err != nil {
+			return
+		}
+
+		err = xs.joinRight(ctx, l)
+		if err != nil {
+			return
+		}
+	}
+	return xs.leftSource.Err()
+}
+
+// joinRight finds related right rows for the given left row and matches them up
+//
+// @note for sorting, we use a b-tree as it's self sorting.
+//			 Benchmarking shows that using a slice is negligibly faster if faster at all.
+func (xs *joinLeft) joinRight(ctx context.Context, left *Row) (err error) {
+	bb, ok, err := xs.getRelatedBuffers(left)
+	if err != nil || !ok {
+		return
+	}
+
+	for _, b := range bb {
+		for _, right := range b.rows {
+			// Merge the two
+			xs.mergeRows(xs.def.OutAttributes, right, left, right)
+
+			// Assert if we want to keep
+			k, err := xs.keep(ctx, right)
+			if err != nil {
+				return err
+			}
+			if !k {
+				continue
+			}
+
+			xs.outSorted.Set(right)
+		}
+	}
+
+	return
+}
+
+// getRelatedBuffers returns all of the right rows corresponding to the given left row
+func (xs *joinLeft) getRelatedBuffers(l *Row) (out []*relIndexBuffer, ok bool, err error) {
+	var aux *relIndexBuffer
+	for c := uint(0); c < l.CountValues()[xs.def.On.Left]; c++ {
+		// @note internal Row struct never errors
+		v, _ := l.GetValue(xs.def.On.Left, c)
+		aux, ok = xs.relIndex.Get(v)
+		if !ok {
+			continue
+		}
+		out = append(out, aux)
+	}
+
+	return
+}
+
+// indexRightRow pushes the provided row onto the rel index
+// @todo consider moving most of this logic to the relIndex struct.
+func (xs *joinLeft) indexRightRow(r *Row) (err error) {
+	for i := uint(0); i < r.CountValues()[xs.def.On.Right]; i++ {
+		v, err := r.GetValue(xs.def.On.Right, i)
+		if err != nil {
+			return err
+		}
+
+		xs.relIndex.Add(v, r)
+	}
+
+	return
+}
+
+// keep checks if the row should be kept or discarded
+func (xs *joinLeft) keep(ctx context.Context, r *Row) (bool, error) {
+	if xs.rowTester == nil {
+		return true, nil
+	}
+
+	return xs.rowTester.Test(ctx, r)
+}
+
+// collectPrimaryAttributes returns all of the attributes of the composited key
+//
+// For joins, all primary attributes from both of the sides should be in here
+// since that is what always uniquely identifies a joined row.
+//
+// @todo consider applying PK candidates and filter out some of these. I don't
+//       think it'll provide much of a performance boost but worth a shot later on.
+func (xs *joinLeft) collectPrimaryAttributes(mm []AttributeMapping) (out []string) {
+	out = make([]string, 0, 2)
+	for _, m := range mm {
+		if m.Properties().IsPrimary {
+			out = append(out, m.Identifier())
+		}
+	}
+
+	return
+}
+
+// mergeRows merges the left and right rows based on the provided attrs
+func (xs *joinLeft) mergeRows(attrs []AttributeMapping, out, left, right *Row) {
+	var (
+		identSide int
+		srcIdent  string
+	)
+
+	for _, attr := range attrs {
+		srcIdent = attr.Identifier()
+		identSide = xs.identSide(srcIdent)
+
+		if identSide == -1 {
+			xs.mergeValuesFrom(attr.Source(), attr.Identifier(), out, left)
+		} else if identSide == 1 {
+			xs.mergeValuesFrom(attr.Source(), attr.Identifier(), out, right)
+		} else {
+			xs.mergeValuesFrom(attr.Source(), attr.Identifier(), out, left, right)
+		}
+	}
+}
+
+func (xs *joinLeft) mergeValuesFrom(inIdent, outIdent string, out *Row, sources ...*Row) {
+	var (
+		aux any
+	)
+	for _, src := range sources {
+		for c := uint(0); c < src.CountValues()[inIdent]; c++ {
+			aux, _ = src.GetValue(inIdent, c)
+			out.SetValue(outIdent, c, aux)
+		}
+	}
+}
+
+// identSide returns -1 if the ident belongs to the left source, 1 if it belongs
+// to the right side, and 0 if it's either.
+//
+// @todo consider adding an additional flag to identify what side it is on.
+//       For now, this should be fine.
+func (xs *joinLeft) identSide(ident string) int {
+	pp := strings.Split(ident, attributeNestingSeparator)
+	if len(pp) > 1 {
+		if pp[0] == xs.def.RelLeft {
+			return -1
+		} else if pp[0] == xs.def.RelRight {
+			return 1
+		}
+	}
+
+	return 0
+}

server/pkg/dal/exec_join_left_bench_test.go

@@ -0,0 +1,116 @@
+package dal
+
+import (
+	"context"
+	"math/rand"
+	"testing"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/stretchr/testify/require"
+)
+
+func benchmarkExecJoin_local(b *testing.B, n int) {
+	ctx := context.Background()
+	attrs := []simpleAttribute{
+		{ident: "l_k"},
+		{ident: "l_v1"},
+		{ident: "l_v2"},
+		{ident: "f_k"},
+		{ident: "f_ref"},
+		{ident: "f_v1"},
+		{ident: "f_v2"},
+	}
+
+	la := []simpleAttribute{
+		{ident: "l_k", t: TypeID{}},
+		{ident: "l_v1"},
+		{ident: "l_v2"},
+	}
+	fa := []simpleAttribute{
+		{ident: "f_k", t: TypeID{}},
+		{ident: "f_ref", t: TypeID{}},
+		{ident: "f_v1"},
+		{ident: "f_v2"},
+	}
+
+	// Inmem buffer for example
+	l := InMemoryBuffer()
+	f := InMemoryBuffer()
+	for i := 0; i < n; i++ {
+		require.NoError(b, l.Add(ctx, simpleRow{"l_k": i + 1, "l_v1": "a", "l_v2": rand.Intn(200)}))
+		require.NoError(b, f.Add(ctx, simpleRow{"f_k": i + 1, "f_ref": i + 1, "f_v1": "a", "f_v2": rand.Intn(200)}))
+	}
+
+	b.ResetTimer()
+
+	for n := 0; n < b.N; n++ {
+		def := Join{
+			Ident:           "join",
+			OutAttributes:   saToMapping(attrs...),
+			LeftAttributes:  saToMapping(la...),
+			RightAttributes: saToMapping(fa...),
+			filter:          internalFilter{orderBy: filter.SortExprSet{{Column: "f_k"}}},
+			On:              JoinPredicate{Left: "l_k", Right: "f_ref"},
+		}
+
+		def.iterator(ctx, l, f)
+
+		l.Seek(ctx, 0)
+		f.Seek(ctx, 0)
+	}
+}
+
+// goos: linux
+// goarch: amd64
+// pkg: github.com/cortezaproject/corteza/server/pkg/dal
+// cpu: Intel(R) Core(TM) i7-8750H CPU @ 2.20GHz
+// BenchmarkExecJoin_local_200-12     	    1620	    718632 ns/op
+// BenchmarkExecJoin_local_400-12     	     801	   1474321 ns/op
+// BenchmarkExecJoin_local_600-12     	     504	   2344433 ns/op
+// BenchmarkExecJoin_local_800-12     	     388	   3098124 ns/op
+// BenchmarkExecJoin_local_1000-12    	     304	   3876453 ns/op
+// BenchmarkExecJoin_local_1200-12    	     258	   4653349 ns/op
+// BenchmarkExecJoin_local_1400-12    	     218	   5406631 ns/op
+// BenchmarkExecJoin_local_1600-12    	     192	   6215687 ns/op
+// BenchmarkExecJoin_local_1800-12    	     168	   7185540 ns/op
+// BenchmarkExecJoin_local_2000-12    	     148	   8021597 ns/op
+// BenchmarkExecJoin_local_2200-12    	     122	   9350466 ns/op
+// BenchmarkExecJoin_local_2400-12    	     100	  10035371 ns/op
+// BenchmarkExecJoin_local_2600-12    	     117	  10247223 ns/op
+// BenchmarkExecJoin_local_2800-12    	     100	  11282374 ns/op
+// BenchmarkExecJoin_local_3000-12    	      99	  12591232 ns/op
+// BenchmarkExecJoin_local_3200-12    	      86	  13047472 ns/op
+// BenchmarkExecJoin_local_3400-12    	      80	  13859590 ns/op
+// BenchmarkExecJoin_local_3600-12    	      76	  14986832 ns/op
+// BenchmarkExecJoin_local_3800-12    	      70	  15826467 ns/op
+// BenchmarkExecJoin_local_4000-12    	      61	  16537483 ns/op
+// BenchmarkExecJoin_local_4200-12    	      68	  17226297 ns/op
+// BenchmarkExecJoin_local_4400-12    	      67	  17996956 ns/op
+// BenchmarkExecJoin_local_4600-12    	      62	  18622211 ns/op
+// BenchmarkExecJoin_local_4800-12    	      61	  19948914 ns/op
+// BenchmarkExecJoin_local_5000-12    	      57	  20614964 ns/op
+func BenchmarkExecJoin_local_200(b *testing.B)  { benchmarkExecJoin_local(b, 200) }
+func BenchmarkExecJoin_local_400(b *testing.B)  { benchmarkExecJoin_local(b, 400) }
+func BenchmarkExecJoin_local_600(b *testing.B)  { benchmarkExecJoin_local(b, 600) }
+func BenchmarkExecJoin_local_800(b *testing.B)  { benchmarkExecJoin_local(b, 800) }
+func BenchmarkExecJoin_local_1000(b *testing.B) { benchmarkExecJoin_local(b, 1000) }
+func BenchmarkExecJoin_local_1200(b *testing.B) { benchmarkExecJoin_local(b, 1200) }
+func BenchmarkExecJoin_local_1400(b *testing.B) { benchmarkExecJoin_local(b, 1400) }
+func BenchmarkExecJoin_local_1600(b *testing.B) { benchmarkExecJoin_local(b, 1600) }
+func BenchmarkExecJoin_local_1800(b *testing.B) { benchmarkExecJoin_local(b, 1800) }
+func BenchmarkExecJoin_local_2000(b *testing.B) { benchmarkExecJoin_local(b, 2000) }
+func BenchmarkExecJoin_local_2200(b *testing.B) { benchmarkExecJoin_local(b, 2200) }
+func BenchmarkExecJoin_local_2400(b *testing.B) { benchmarkExecJoin_local(b, 2400) }
+func BenchmarkExecJoin_local_2600(b *testing.B) { benchmarkExecJoin_local(b, 2600) }
+func BenchmarkExecJoin_local_2800(b *testing.B) { benchmarkExecJoin_local(b, 2800) }
+func BenchmarkExecJoin_local_3000(b *testing.B) { benchmarkExecJoin_local(b, 3000) }
+func BenchmarkExecJoin_local_3200(b *testing.B) { benchmarkExecJoin_local(b, 3200) }
+func BenchmarkExecJoin_local_3400(b *testing.B) { benchmarkExecJoin_local(b, 3400) }
+func BenchmarkExecJoin_local_3600(b *testing.B) { benchmarkExecJoin_local(b, 3600) }
+func BenchmarkExecJoin_local_3800(b *testing.B) { benchmarkExecJoin_local(b, 3800) }
+func BenchmarkExecJoin_local_4000(b *testing.B) { benchmarkExecJoin_local(b, 4000) }
+func BenchmarkExecJoin_local_4200(b *testing.B) { benchmarkExecJoin_local(b, 4200) }
+func BenchmarkExecJoin_local_4400(b *testing.B) { benchmarkExecJoin_local(b, 4400) }
+func BenchmarkExecJoin_local_4600(b *testing.B) { benchmarkExecJoin_local(b, 4600) }
+func BenchmarkExecJoin_local_4800(b *testing.B) { benchmarkExecJoin_local(b, 4800) }
+func BenchmarkExecJoin_local_5000(b *testing.B) { benchmarkExecJoin_local(b, 5000) }

server/pkg/dal/exec_link_left.go

@@ -0,0 +1,505 @@
+package dal
+
+import (
+	"context"
+	"fmt"
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"io"
+	"sort"
+	"strconv"
+)
+
+type (
+	linkLeft struct {
+		def    Link
+		filter internalFilter
+
+		leftSource  Iterator
+		rightSource Iterator
+		err         error
+		scanRow     *Row
+		planned     bool
+		filtered    bool
+
+		rightSortAttrs []string
+
+		rowTester tester
+
+		// Buffer to keep track of pulled left rows
+		leftRows   []*Row
+		relIndex   *relIndex
+		keepLeft   bool
+		leftIndex  int
+		rightIndex int
+
+		// Some helper fields for temporary data
+		leftRow       *Row
+		rightRow      *Row
+		relScanBuffer *relIndexBuffer
+	}
+)
+
+func (xs *linkLeft) init(ctx context.Context, linkPredType Type, rightSort []string) (err error) {
+	// @note the index is keeping track of the right source attributes so we can
+	//       simplify the sorting logic.
+	xs.relIndex, err = newRelIndex(linkPredType, rightSort...)
+	if err != nil {
+		return
+	}
+
+	xs.rightSortAttrs = rightSort
+
+	xs.rowTester, err = prepareGenericRowTester(xs.filter)
+	if err != nil {
+		return
+	}
+
+	return xs.applyPlan(ctx)
+}
+
+func (xs *linkLeft) Next(ctx context.Context) (more bool) {
+	xs.err = xs.applyPlan(ctx)
+	if xs.err != nil {
+		return false
+	}
+
+	more, xs.err = xs.next(ctx)
+	return
+}
+
+func (xs *linkLeft) More(limit uint, v ValueGetter) (err error) {
+	xs.filter.cursor, err = xs.ForwardCursor(v)
+	if err != nil {
+		return
+	}
+
+	// Redo row tester
+	xs.rowTester, err = prepareGenericRowTester(xs.filter)
+	if err != nil {
+		return
+	}
+
+	// Redo the state
+	// @todo adjust based on aggregation plan; reuse buffered, etc.
+	xs.relIndex.Clear()
+	xs.leftRows = make([]*Row, 0, 128)
+	xs.scanRow = nil
+	xs.planned = false
+	xs.keepLeft = false
+	xs.leftIndex = 0
+	xs.rightIndex = 0
+
+	return
+}
+
+func (xs *linkLeft) Err() error { return xs.err }
+
+func (xs *linkLeft) Scan(s ValueSetter) (err error) {
+	for k, cc := range xs.scanRow.CountValues() {
+		for i := uint(0); i < cc; i++ {
+			// @note internal row won't raise errors so we can safely omit them
+			v, _ := xs.scanRow.GetValue(k, i)
+			err = s.SetValue(k, i, v)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	return
+}
+
+func (xs *linkLeft) Close() (err error) {
+	if xs == nil {
+		return
+	}
+
+	cc := []io.Closer{
+		xs.leftSource,
+		xs.rightSource,
+	}
+
+	for _, c := range cc {
+		if c != nil {
+			err = c.Close()
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return
+}
+
+func (xs *linkLeft) BackCursor(v ValueGetter) (pc *filter.PagingCursor, err error) {
+	g := &rowLink{
+		a: xs.leftRow,
+		b: v,
+	}
+	pc, err = filter.PagingCursorFrom(xs.filter.OrderBy(), g, xs.collectPrimaryAttributes()...)
+	if err != nil {
+		return nil, err
+	}
+
+	pc.ROrder = true
+	pc.LThen = xs.filter.OrderBy().Reversed()
+
+	return
+}
+
+func (xs *linkLeft) ForwardCursor(v ValueGetter) (pc *filter.PagingCursor, err error) {
+	g := &rowLink{
+		a: xs.leftRow,
+		b: v,
+	}
+	pc, err = filter.PagingCursorFrom(xs.filter.OrderBy(), g, xs.collectPrimaryAttributes()...)
+	if err != nil {
+		return nil, err
+	}
+
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Utility methods
+
+// next prepares the next scan row based on the defined link plan
+func (xs *linkLeft) next(ctx context.Context) (more bool, err error) {
+	more, err = xs.pullNext(ctx)
+	if !more || err != nil {
+		return more, err
+	}
+
+	return xs.nextBuffered()
+}
+
+// pullNext pulls additional data so we can produce more
+//
+// This step may be omitted based on the join plan.
+func (xs *linkLeft) pullNext(ctx context.Context) (more bool, err error) {
+	// Pull next chunk from source if not entirely buffered
+	if xs.def.plan.partialScan {
+		return false, fmt.Errorf("partialScan join plan strategy not implemented")
+	}
+
+	// Check if buffer has more
+	// We need to check for gt because the right bits may still be relevant
+	if xs.leftIndex > len(xs.leftRows) {
+		return false, nil
+	}
+	return true, nil
+}
+
+// nextBuffered prepares the next scan row from the buffers
+func (xs *linkLeft) nextBuffered() (more bool, err error) {
+	// keepLeft indicates if we should keep the left row and move onto the next
+	// in the right buffers.
+	//
+	// If we're keeping it, take the next row from the other side, else take the
+	// next left one and reset right counters.
+
+	var (
+		ok bool
+	)
+
+	for {
+		if !xs.keepLeft {
+			// Go for the next left row
+			if xs.leftIndex >= len(xs.leftRows) {
+				return
+			}
+			xs.leftRow = xs.leftRows[xs.leftIndex]
+			xs.leftIndex++
+
+			// Go for the next right buffer
+			xs.relScanBuffer, ok, err = xs.getRelatedBuffer(xs.leftRow)
+			if !ok || err != nil {
+				return
+			}
+			// if len(rel) > 1 {
+			// 	// @todo implement this; not entirely sure how it should be so I'll block it for now
+			// 	// @todo move this check futher up
+			// 	return false, fmt.Errorf("multi-value link predicates not supported")
+			// }
+			// xs.relScanBuffer = rel[0]
+			xs.rightIndex = 0
+
+			xs.scanRow = xs.leftRow
+			xs.keepLeft = true
+			return true, nil
+		}
+
+		// Related buffer done
+		if xs.rightIndex >= len(xs.relScanBuffer.rows) {
+			xs.keepLeft = false
+			continue
+		}
+
+		xs.rightRow = xs.relScanBuffer.rows[xs.rightIndex]
+		xs.rightIndex++
+		xs.scanRow = xs.rightRow
+
+		// Sys defined value to indicate the source
+		// @todo consider using some wrap struct instead
+		xs.scanRow.SetValue(LinkRefIdent, 0, xs.def.RelRight)
+		break
+	}
+	return true, nil
+}
+
+// getRelatedBuffer returns all of the right rows corresponding to the given left row
+func (xs *linkLeft) getRelatedBuffer(l *Row) (out *relIndexBuffer, ok bool, err error) {
+
+	// @todo mv link predicate attrs
+	v, _ := l.GetValue(xs.def.On.Left, 0)
+	out, ok = xs.relIndex.Get(v)
+	return
+}
+
+// applyPlan runs plan specific logic to prepare the state
+func (xs *linkLeft) applyPlan(ctx context.Context) (err error) {
+	if xs.planned || xs.err != nil {
+		return
+	}
+
+	xs.planned = true
+	switch {
+	case !xs.def.plan.partialScan:
+		return xs.pullEntireSource(ctx)
+	}
+
+	return
+}
+
+// pullEntireSource pulls both sources into memory and indexes them for later use
+func (xs *linkLeft) pullEntireSource(ctx context.Context) (err error) {
+	// This bit does the filtering so just mark it of as such here
+	xs.filtered = true
+
+	// First the right source
+	err = xs.pullEntireRightSource(ctx)
+	if err != nil {
+		return
+	}
+
+	// Next the left source
+	err = xs.pullEntireLeftSource(ctx)
+	if err != nil {
+		return
+	}
+
+	// Sort if needed
+	return xs.sortLeftRows()
+}
+
+// pullEntireRightSource pulls and indexes all of the right bits
+func (xs *linkLeft) pullEntireRightSource(ctx context.Context) (err error) {
+	// Drain the source
+	for xs.rightSource.Next(ctx) {
+		r := &Row{
+			counters: make(map[string]uint),
+			values:   make(valueSet),
+		}
+
+		err = xs.rightSource.Scan(r)
+		if err != nil {
+			return
+		}
+
+		err = xs.indexRightRow(r)
+		if err != nil {
+			return
+		}
+	}
+	return xs.rightSource.Err()
+}
+
+// pullEntireLeftSource pulls and indexes all of the left bits
+func (xs *linkLeft) pullEntireLeftSource(ctx context.Context) (err error) {
+	var (
+		rel *relIndexBuffer
+		ok  bool
+	)
+
+	// Drain the source
+	for xs.leftSource.Next(ctx) {
+		l := &Row{
+			counters: make(map[string]uint),
+			values:   make(valueSet),
+		}
+
+		err = xs.leftSource.Scan(l)
+		if err != nil {
+			return
+		}
+
+		rel, ok, err = xs.getRelatedBuffer(l)
+		if err != nil {
+			return
+		}
+		if !ok {
+			continue
+		}
+
+		k, err := xs.keep(ctx, l, rel)
+		if err != nil {
+			return err
+		}
+		if !k {
+			continue
+		}
+
+		xs.leftRows = append(xs.leftRows, l)
+	}
+	return xs.leftSource.Err()
+}
+
+// indexRightRow pushes the provided row onto the rel index
+// @todo consider moving most of this logic to the relIndex struct.
+func (xs *linkLeft) indexRightRow(r *Row) (err error) {
+	// @todo mv link predicate attrs; should be prevented higher up for now
+	v, err := r.GetValue(xs.def.On.Right, 0)
+	if err != nil {
+		return err
+	}
+
+	xs.relIndex.Add(v, r)
+	return
+}
+
+// sortLeftRows sorts the left rows into the correct order
+//
+// Algorithm outline:
+// Compare the two left rows based on the defined sort order.
+// If the two left rows match up, or we're primarily sorting using right rows,
+// check the min/max stats values produced by the index buffer.
+//
+// If some min/max of a chunk is greater/lesser then the other, no row in the other
+// can appear before/after the prior.
+func (xs *linkLeft) sortLeftRows() (err error) {
+	var (
+		leftRelBufferA *relIndexBuffer
+		leftRelBufferB *relIndexBuffer
+
+		a, b any
+	)
+
+	// Use stable sort just so we don't needlesly messup the initial order if we decide
+	// to preserve sorts produced by further steps.
+	sort.SliceStable(xs.leftRows, func(i, j int) bool {
+		if err != nil {
+			return false
+		}
+
+		for _, s := range xs.filter.OrderBy() {
+			// Prepare the data
+			leftRowA := xs.leftRows[i]
+			leftRowB := xs.leftRows[j]
+
+			leftRowA = convertStringToIntForColumn(s.Column, leftRowA)
+			leftRowB = convertStringToIntForColumn(s.Column, leftRowB)
+
+			leftRelBufferA, _, err = xs.getRelatedBuffer(leftRowA)
+			if err != nil {
+				return false
+			}
+			leftRelBufferB, _, err = xs.getRelatedBuffer(leftRowB)
+			if err != nil {
+				return false
+			}
+
+			if !xs.isRightSortAttr(s.Column) {
+				// This bit here orders based on the left attributes
+				less, skip := evalCmpResult(compareGetters(leftRowA, leftRowB, leftRowA.counters, leftRowB.counters, s.Column), s)
+				if !skip {
+					return less
+				}
+			} else {
+				// This bit here orders based on the right attributes
+				// Check the stats of the buffer; make sure to adjust based on direction
+
+				// Use chunk's values
+				if !s.Descending {
+					a = leftRelBufferA.min[s.Column]
+					b = leftRelBufferB.min[s.Column]
+				} else {
+					a = leftRelBufferA.max[s.Column]
+					b = leftRelBufferB.max[s.Column]
+				}
+
+				less, skip := evalCmpResult(compareValues(a, b), s)
+				if !skip {
+					return less
+				}
+			}
+		}
+
+		return false
+	})
+
+	return
+}
+
+func (xs *linkLeft) isRightSortAttr(ident string) bool {
+	for _, i := range xs.rightSortAttrs {
+		if i == ident {
+			return true
+		}
+	}
+	return false
+}
+
+// keep checks if the row should be kept or discarded
+//
+// Link's keep is a bit more complicated and it looks at the related buffer as well.
+func (xs *linkLeft) keep(ctx context.Context, left *Row, buffer *relIndexBuffer) (keep bool, err error) {
+	// If no buffer, we won't keep -- left inner join like behavior
+	if buffer == nil {
+		return false, nil
+	}
+	// No tester include all ok rows
+	if xs.rowTester == nil {
+		return true, nil
+	}
+
+	ch := &rowLink{a: left}
+	for _, ch.b = range buffer.rows {
+		k, err := xs.rowTester.Test(ctx, ch)
+		if err != nil {
+			return false, err
+		}
+		if !k {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func (xs *linkLeft) collectPrimaryAttributes() (out []string) {
+	out = make([]string, 0, 2)
+	for _, m := range xs.def.OutLeftAttributes {
+		if m.Properties().IsPrimary {
+			out = append(out, m.Identifier())
+		}
+	}
+
+	for _, m := range xs.def.OutRightAttributes {
+		if m.Properties().IsPrimary {
+			out = append(out, m.Identifier())
+		}
+	}
+
+	return
+}
+
+// convertStringToIntForColumn converts the string value that have numeric data in the specified column
+// and updates the Row value datatype to int, if the conversion is successful
+func convertStringToIntForColumn(column string, row *Row) *Row {
+	if strValue, ok := row.values[column][0].(string); ok {
+		if rowVal, err := strconv.Atoi(strValue); err == nil {
+			row.values[column][0] = rowVal
+		}
+	}
+	return row
+}

server/pkg/dal/exec_link_left_bench_test.go

@@ -0,0 +1,119 @@
+package dal
+
+import (
+	"context"
+	"math/rand"
+	"testing"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/stretchr/testify/require"
+)
+
+func benchmarkExecLink_left(b *testing.B, n int) {
+	ctx := context.Background()
+	lattrs := []simpleAttribute{
+		{ident: "l_k"},
+		{ident: "l_v1"},
+		{ident: "l_v2"},
+	}
+	fattrs := []simpleAttribute{
+		{ident: "f_k"},
+		{ident: "f_ref"},
+		{ident: "f_v1"},
+		{ident: "f_v2"},
+	}
+
+	la := []simpleAttribute{
+		{ident: "l_k", t: TypeID{}},
+		{ident: "l_v1"},
+		{ident: "l_v2"},
+	}
+	fa := []simpleAttribute{
+		{ident: "f_k", t: TypeID{}},
+		{ident: "f_ref", t: TypeID{}},
+		{ident: "f_v1"},
+		{ident: "f_v2"},
+	}
+
+	// Inmem buffer for example
+	l := InMemoryBuffer()
+	f := InMemoryBuffer()
+	for i := 0; i < n; i++ {
+		require.NoError(b, l.Add(ctx, simpleRow{"l_k": i + 1, "l_v1": "a", "l_v2": rand.Intn(200)}))
+		require.NoError(b, f.Add(ctx, simpleRow{"f_k": i + 1, "f_ref": i + 1, "f_v1": "a", "f_v2": rand.Intn(200)}))
+	}
+
+	b.ResetTimer()
+
+	for n := 0; n < b.N; n++ {
+		def := Link{
+			Ident:              "link",
+			OutLeftAttributes:  saToMapping(lattrs...),
+			OutRightAttributes: saToMapping(fattrs...),
+			LeftAttributes:     saToMapping(la...),
+			RightAttributes:    saToMapping(fa...),
+			filter:             internalFilter{orderBy: filter.SortExprSet{{Column: "f_k"}}},
+			On:                 LinkPredicate{Left: "l_k", Right: "f_ref"},
+		}
+
+		def.iterator(ctx, l, f)
+
+		l.Seek(ctx, 0)
+		f.Seek(ctx, 0)
+	}
+}
+
+// goos: linux
+// goarch: amd64
+// pkg: github.com/cortezaproject/corteza/server/pkg/dal
+// cpu: Intel(R) Core(TM) i7-8750H CPU @ 2.20GHz
+// BenchmarkExecLink_left_200-12     	    1874	    633602 ns/op
+// BenchmarkExecLink_left_400-12     	     910	   1369591 ns/op
+// BenchmarkExecLink_left_600-12     	     590	   2162512 ns/op
+// BenchmarkExecLink_left_800-12     	     451	   2725109 ns/op
+// BenchmarkExecLink_left_1000-12    	     358	   3568684 ns/op
+// BenchmarkExecLink_left_1200-12    	     260	   4059573 ns/op
+// BenchmarkExecLink_left_1400-12    	     250	   4742408 ns/op
+// BenchmarkExecLink_left_1600-12    	     219	   5383900 ns/op
+// BenchmarkExecLink_left_1800-12    	     181	   6880839 ns/op
+// BenchmarkExecLink_left_2000-12    	     165	   6821432 ns/op
+// BenchmarkExecLink_left_2200-12    	     134	   8386884 ns/op
+// BenchmarkExecLink_left_2400-12    	     142	   8471661 ns/op
+// BenchmarkExecLink_left_2600-12    	     135	   8702328 ns/op
+// BenchmarkExecLink_left_2800-12    	     126	   9375744 ns/op
+// BenchmarkExecLink_left_3000-12    	     114	  10156339 ns/op
+// BenchmarkExecLink_left_3200-12    	      96	  10682149 ns/op
+// BenchmarkExecLink_left_3400-12    	     104	  11361907 ns/op
+// BenchmarkExecLink_left_3600-12    	      80	  12616645 ns/op
+// BenchmarkExecLink_left_3800-12    	      73	  13761230 ns/op
+// BenchmarkExecLink_left_4000-12    	      81	  13795284 ns/op
+// BenchmarkExecLink_left_4200-12    	      69	  15898381 ns/op
+// BenchmarkExecLink_left_4400-12    	      72	  15162550 ns/op
+// BenchmarkExecLink_left_4600-12    	      74	  15650527 ns/op
+// BenchmarkExecLink_left_4800-12    	      70	  19573447 ns/op
+// BenchmarkExecLink_left_5000-12    	      69	  17214960 ns/op
+func BenchmarkExecLink_left_200(b *testing.B)  { benchmarkExecLink_left(b, 200) }
+func BenchmarkExecLink_left_400(b *testing.B)  { benchmarkExecLink_left(b, 400) }
+func BenchmarkExecLink_left_600(b *testing.B)  { benchmarkExecLink_left(b, 600) }
+func BenchmarkExecLink_left_800(b *testing.B)  { benchmarkExecLink_left(b, 800) }
+func BenchmarkExecLink_left_1000(b *testing.B) { benchmarkExecLink_left(b, 1000) }
+func BenchmarkExecLink_left_1200(b *testing.B) { benchmarkExecLink_left(b, 1200) }
+func BenchmarkExecLink_left_1400(b *testing.B) { benchmarkExecLink_left(b, 1400) }
+func BenchmarkExecLink_left_1600(b *testing.B) { benchmarkExecLink_left(b, 1600) }
+func BenchmarkExecLink_left_1800(b *testing.B) { benchmarkExecLink_left(b, 1800) }
+func BenchmarkExecLink_left_2000(b *testing.B) { benchmarkExecLink_left(b, 2000) }
+func BenchmarkExecLink_left_2200(b *testing.B) { benchmarkExecLink_left(b, 2200) }
+func BenchmarkExecLink_left_2400(b *testing.B) { benchmarkExecLink_left(b, 2400) }
+func BenchmarkExecLink_left_2600(b *testing.B) { benchmarkExecLink_left(b, 2600) }
+func BenchmarkExecLink_left_2800(b *testing.B) { benchmarkExecLink_left(b, 2800) }
+func BenchmarkExecLink_left_3000(b *testing.B) { benchmarkExecLink_left(b, 3000) }
+func BenchmarkExecLink_left_3200(b *testing.B) { benchmarkExecLink_left(b, 3200) }
+func BenchmarkExecLink_left_3400(b *testing.B) { benchmarkExecLink_left(b, 3400) }
+func BenchmarkExecLink_left_3600(b *testing.B) { benchmarkExecLink_left(b, 3600) }
+func BenchmarkExecLink_left_3800(b *testing.B) { benchmarkExecLink_left(b, 3800) }
+func BenchmarkExecLink_left_4000(b *testing.B) { benchmarkExecLink_left(b, 4000) }
+func BenchmarkExecLink_left_4200(b *testing.B) { benchmarkExecLink_left(b, 4200) }
+func BenchmarkExecLink_left_4400(b *testing.B) { benchmarkExecLink_left(b, 4400) }
+func BenchmarkExecLink_left_4600(b *testing.B) { benchmarkExecLink_left(b, 4600) }
+func BenchmarkExecLink_left_4800(b *testing.B) { benchmarkExecLink_left(b, 4800) }
+func BenchmarkExecLink_left_5000(b *testing.B) { benchmarkExecLink_left(b, 5000) }

server/pkg/dal/filter.go

@@ -0,0 +1,189 @@
+package dal
+
+import (
+	"fmt"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/cortezaproject/corteza/server/pkg/ql"
+)
+
+type (
+	// internalFilter is a wrap to simplify interactions for cases where we might
+	// want to interact with the received filters; i.e. when optimizing pipeline steps
+	internalFilter struct {
+		constraints      map[string][]any
+		stateConstraints map[string]filter.State
+		metaConstraints  map[string]any
+		expression       string
+		expParsed        *ql.ASTNode
+		orderBy          filter.SortExprSet
+		limit            uint
+		cursor           *filter.PagingCursor
+	}
+
+	parsedFilter interface {
+		ExpressionParsed() *ql.ASTNode
+	}
+)
+
+func (f internalFilter) Constraints() map[string][]any             { return f.constraints }
+func (f internalFilter) StateConstraints() map[string]filter.State { return f.stateConstraints }
+func (f internalFilter) MetaConstraints() map[string]any           { return f.metaConstraints }
+func (f internalFilter) Expression() string                        { return f.expression }
+func (f internalFilter) ExpressionParsed() *ql.ASTNode             { return f.expParsed }
+func (f internalFilter) OrderBy() filter.SortExprSet               { return f.orderBy }
+func (f internalFilter) Limit() uint                               { return f.limit }
+func (f internalFilter) Cursor() *filter.PagingCursor              { return f.cursor }
+
+// toInternalFilter converts filter.Filter to internalFilter for easier manipulation
+//
+// nil filters are returned as zeroed-out internalFilters.
+func toInternalFilter(f filter.Filter) (out internalFilter, err error) {
+	if f == nil {
+		return
+	}
+
+	// Base
+	out = internalFilter{
+		constraints:      f.Constraints(),
+		stateConstraints: f.StateConstraints(),
+		expression:       f.Expression(),
+		orderBy:          f.OrderBy(),
+		limit:            f.Limit(),
+		cursor:           f.Cursor(),
+	}
+
+	pf, ok := f.(parsedFilter)
+	if ok {
+		out.expParsed = pf.ExpressionParsed()
+
+		// In case the filter was already provided, we need to make sure the idents
+		// are wrapped
+		out.expParsed.Traverse(func(a *ql.ASTNode) (bool, *ql.ASTNode, error) {
+			a.Symbol = wrapNestedGvalIdent(a.Symbol)
+			return true, a, nil
+		})
+	}
+	if out.expParsed != nil {
+		// We can't mix the two so just clear it out to avoid confusion
+		out.expression = ""
+	}
+
+	// Parse expression for later use
+	if out.expression != "" {
+		out.expParsed, err = newQlParser().Parse(out.expression)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func FilterFromExpr(n *ql.ASTNode) internalFilter {
+	// @todo consider adding string expr for consistency
+	return internalFilter{
+		expParsed: n,
+	}
+}
+
+// MergeFilters returns a new filter based on a overwritten by b
+func (a internalFilter) MergeFilters(b filter.Filter) (c internalFilter, err error) {
+	// In case we got a generic b filter, convert it to the internal one for easier handling
+	aux, ok := b.(internalFilter)
+	if !ok {
+		var err error
+		aux, err = toInternalFilter(b)
+		if err != nil {
+			return c, err
+		}
+	}
+
+	return a.mergeFilters(aux), nil
+}
+
+func (a internalFilter) mergeFilters(b internalFilter) (c internalFilter) {
+
+	c = a
+
+	// expression
+	if b.expParsed != nil {
+		if c.expParsed == nil {
+			c.expParsed = b.expParsed
+			c.expression = b.expression
+		} else {
+			parsedA := a.expParsed
+			rawA := a.expression
+			if parsedA.Ref != "group" {
+				parsedA = &ql.ASTNode{
+					Ref: "group",
+					Args: []*ql.ASTNode{
+						parsedA,
+					},
+				}
+				rawA = fmt.Sprintf("(%s)", rawA)
+			}
+
+			parsedB := b.expParsed
+			rawB := b.expression
+			if parsedB.Ref != "group" {
+				parsedB = &ql.ASTNode{
+					Ref: "group",
+					Args: []*ql.ASTNode{
+						parsedB,
+					},
+				}
+				rawB = fmt.Sprintf("(%s)", rawB)
+			}
+
+			c.expParsed = &ql.ASTNode{
+				Ref:  "and",
+				Args: ql.ASTNodeSet{parsedA, parsedB},
+			}
+			c.expression = fmt.Sprintf("%s && %s", rawA, rawB)
+		}
+
+	}
+
+	// constraints
+	if b.constraints != nil {
+		if c.constraints == nil {
+			c.constraints = make(map[string][]any)
+		}
+
+		for k, v := range b.constraints {
+			c.constraints[k] = append(c.constraints[k], v...)
+		}
+	}
+
+	// state constraints
+	if b.stateConstraints != nil {
+		if c.stateConstraints == nil {
+			c.stateConstraints = make(map[string]filter.State)
+		}
+		for k, v := range b.stateConstraints {
+			c.stateConstraints[k] = v
+		}
+	}
+
+	// order by
+	if b.orderBy != nil {
+		c.orderBy = append(c.orderBy, b.orderBy...)
+	}
+
+	// cursor
+	// always use the latest paging cursor
+	c.cursor = b.cursor
+
+	return
+}
+
+// empty reports if the filter is completely empty or not
+func (f internalFilter) empty() bool {
+	return f.constraints == nil &&
+		f.stateConstraints == nil &&
+		f.expression == "" &&
+		f.orderBy == nil &&
+		f.limit == 0 &&
+		f.cursor == nil
+}

server/pkg/dal/filter_test.go

@@ -0,0 +1,268 @@
+package dal
+
+import (
+	"testing"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/stretchr/testify/require"
+)
+
+func TestInternalFilterMergeFilters_expression(t *testing.T) {
+	t.Run("b no expr", func(t *testing.T) {
+		a, err := toInternalFilter(internalFilter{expression: "a == b"})
+		require.NoError(t, err)
+		b, err := toInternalFilter(internalFilter{expression: ""})
+		require.NoError(t, err)
+
+		c := a.mergeFilters(b)
+		require.Equal(t, "a == b", c.expression)
+		require.Equal(t, "eq(a, b)", c.expParsed.String())
+	})
+
+	t.Run("a no expr", func(t *testing.T) {
+		a, err := toInternalFilter(internalFilter{expression: ""})
+		require.NoError(t, err)
+		b, err := toInternalFilter(internalFilter{expression: "a == b"})
+		require.NoError(t, err)
+
+		c := a.mergeFilters(b)
+		require.Equal(t, "a == b", c.expression)
+		require.Equal(t, "eq(a, b)", c.expParsed.String())
+	})
+
+	t.Run("both missing", func(t *testing.T) {
+		a, err := toInternalFilter(internalFilter{expression: ""})
+		require.NoError(t, err)
+		b, err := toInternalFilter(internalFilter{expression: ""})
+		require.NoError(t, err)
+
+		c := a.mergeFilters(b)
+		require.Equal(t, "", c.expression)
+		require.Equal(t, "<nil>", c.expParsed.String())
+	})
+
+	t.Run("combine", func(t *testing.T) {
+		a, err := toInternalFilter(internalFilter{expression: "a == b"})
+		require.NoError(t, err)
+		b, err := toInternalFilter(internalFilter{expression: "c == d"})
+		require.NoError(t, err)
+
+		c := a.mergeFilters(b)
+		require.Equal(t, "(a == b) && (c == d)", c.expression)
+		require.Equal(t, "and(group(eq(a, b)), group(eq(c, d)))", c.expParsed.String())
+	})
+}
+
+func TestInternalFilterMergeFilters_constraints(t *testing.T) {
+	t.Run("b no constr", func(t *testing.T) {
+		a := internalFilter{constraints: map[string][]any{"a": {"b"}}}
+		b := internalFilter{constraints: nil}
+
+		c := a.mergeFilters(b)
+		require.Equal(t, map[string][]any{"a": {"b"}}, c.constraints)
+	})
+
+	t.Run("a no constr", func(t *testing.T) {
+		a := internalFilter{constraints: nil}
+		b := internalFilter{constraints: map[string][]any{"a": {"b"}}}
+
+		c := a.mergeFilters(b)
+		require.Equal(t, map[string][]any{"a": {"b"}}, c.constraints)
+	})
+
+	t.Run("both missing", func(t *testing.T) {
+		a := internalFilter{constraints: nil}
+		b := internalFilter{constraints: nil}
+
+		c := a.mergeFilters(b)
+		// Use this assertion because equal asserts type also
+		require.Nil(t, c.constraints)
+	})
+
+	t.Run("combine", func(t *testing.T) {
+		a := internalFilter{constraints: map[string][]any{"a": {"b"}}}
+		b := internalFilter{constraints: map[string][]any{"a": {"c"}, "d": {"e"}}}
+
+		c := a.mergeFilters(b)
+		require.Equal(t, map[string][]any{"a": {"b", "c"}, "d": {"e"}}, c.constraints)
+	})
+}
+
+func TestInternalFilterMergeFilters_stateConstraints(t *testing.T) {
+	t.Run("b no constr", func(t *testing.T) {
+		a := internalFilter{stateConstraints: map[string]filter.State{"a": filter.StateInclusive}}
+		b := internalFilter{stateConstraints: nil}
+
+		c := a.mergeFilters(b)
+		require.Equal(t, map[string]filter.State{"a": filter.StateInclusive}, c.stateConstraints)
+	})
+
+	t.Run("a no constr", func(t *testing.T) {
+		a := internalFilter{stateConstraints: nil}
+		b := internalFilter{stateConstraints: map[string]filter.State{"a": filter.StateInclusive}}
+
+		c := a.mergeFilters(b)
+		require.Equal(t, map[string]filter.State{"a": filter.StateInclusive}, c.stateConstraints)
+	})
+
+	t.Run("both missing", func(t *testing.T) {
+		a := internalFilter{stateConstraints: nil}
+		b := internalFilter{stateConstraints: nil}
+
+		c := a.mergeFilters(b)
+		// Use this assertion because equal asserts type also
+		require.Nil(t, c.stateConstraints)
+	})
+
+	t.Run("combine", func(t *testing.T) {
+		a := internalFilter{stateConstraints: map[string]filter.State{"a": filter.StateInclusive}}
+		b := internalFilter{stateConstraints: map[string]filter.State{"a": filter.StateExcluded, "d": filter.StateExclusive}}
+
+		c := a.mergeFilters(b)
+		require.Equal(t, map[string]filter.State{"a": filter.StateExcluded, "d": filter.StateExclusive}, c.stateConstraints)
+	})
+}
+
+func TestInternalFilterMergeFilters_order(t *testing.T) {
+	t.Run("b no order", func(t *testing.T) {
+		a := internalFilter{orderBy: filter.SortExprSet{{Column: "a"}}}
+		b := internalFilter{orderBy: nil}
+
+		c := a.mergeFilters(b)
+		require.Equal(t, filter.SortExprSet{{Column: "a"}}, c.orderBy)
+	})
+
+	t.Run("a no order", func(t *testing.T) {
+		a := internalFilter{orderBy: nil}
+		b := internalFilter{orderBy: filter.SortExprSet{{Column: "a"}}}
+
+		c := a.mergeFilters(b)
+		require.Equal(t, filter.SortExprSet{{Column: "a"}}, c.orderBy)
+	})
+
+	t.Run("both missing", func(t *testing.T) {
+		a := internalFilter{orderBy: nil}
+		b := internalFilter{orderBy: nil}
+
+		c := a.mergeFilters(b)
+		// Use this assertion because equal asserts type also
+		require.Nil(t, c.orderBy)
+	})
+
+	t.Run("combine", func(t *testing.T) {
+		a := internalFilter{orderBy: filter.SortExprSet{{Column: "a"}}}
+		b := internalFilter{orderBy: filter.SortExprSet{{Column: "b"}}}
+
+		c := a.mergeFilters(b)
+		require.Equal(t, filter.SortExprSet{{Column: "a"}, {Column: "b"}}, c.orderBy)
+	})
+}
+
+func TestInternalFilterMergeFilters_cursor(t *testing.T) {
+	t.Run("b no cursor", func(t *testing.T) {
+		a := internalFilter{cursor: &filter.PagingCursor{ROrder: true}}
+		b := internalFilter{orderBy: nil}
+
+		c := a.mergeFilters(b)
+		require.Nil(t, c.cursor)
+	})
+
+	t.Run("a no constr", func(t *testing.T) {
+		a := internalFilter{orderBy: nil}
+		b := internalFilter{cursor: &filter.PagingCursor{ROrder: true}}
+
+		c := a.mergeFilters(b)
+		require.Equal(t, &filter.PagingCursor{ROrder: true}, c.cursor)
+	})
+
+	t.Run("both missing", func(t *testing.T) {
+		a := internalFilter{orderBy: nil}
+		b := internalFilter{orderBy: nil}
+
+		c := a.mergeFilters(b)
+		// Use this assertion because equal asserts type also
+		require.Nil(t, c.cursor)
+	})
+
+	t.Run("combine", func(t *testing.T) {
+		a := internalFilter{cursor: &filter.PagingCursor{ROrder: true}}
+		b := internalFilter{cursor: &filter.PagingCursor{ROrder: false}}
+
+		c := a.mergeFilters(b)
+		require.Equal(t, &filter.PagingCursor{ROrder: false}, c.cursor)
+	})
+}
+
+func TestInternalFilterEmpty(t *testing.T) {
+	tcc := []struct {
+		name string
+		f    internalFilter
+		want bool
+	}{
+		{
+			name: "completely empty",
+			f:    internalFilter{},
+			want: true,
+		},
+		{
+			name: "only constraints",
+			f: internalFilter{
+				constraints: map[string][]any{
+					"a": {"b"},
+				},
+			},
+			want: false,
+		},
+		{
+			name: "only state constraints",
+			f: internalFilter{
+				stateConstraints: map[string]filter.State{
+					"a": filter.StateInclusive,
+				},
+			},
+			want: false,
+		},
+		{
+			name: "only expression",
+			f: internalFilter{
+				expression: "true",
+			},
+			want: false,
+		},
+		{
+			name: "only order",
+			f: internalFilter{
+				orderBy: filter.SortExprSet{{Column: "a"}},
+			},
+			want: false,
+		},
+		{
+			name: "only cursor",
+			f: internalFilter{
+				cursor: &filter.PagingCursor{ROrder: true},
+			},
+			want: false,
+		},
+		{
+			name: "all",
+			f: internalFilter{
+				constraints: map[string][]any{
+					"a": {"b"},
+				},
+				stateConstraints: map[string]filter.State{
+					"a": filter.StateInclusive,
+				},
+				orderBy: filter.SortExprSet{{Column: "a"}},
+				cursor:  &filter.PagingCursor{ROrder: true},
+			},
+			want: false,
+		},
+	}
+
+	for _, tc := range tcc {
+		t.Run(tc.name, func(t *testing.T) {
+			got := tc.f.empty()
+			require.Equal(t, tc.want, got)
+		})
+	}
+}

server/pkg/dal/inmem_buffer.go

@@ -0,0 +1,212 @@
+package dal
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+)
+
+type (
+	// inmemBuffer isa simple DAL buffer which holds all of the data in memory
+	//
+	// This buffer should be used for small datasets
+	inmemBuffer struct {
+		rows []ValueGetter
+		ctrs []map[string]uint
+
+		i      int
+		cap    int
+		closed bool
+
+		more bool
+		err  error
+
+		sort filter.SortExprSet
+	}
+)
+
+// InMemoryBuffer initializes a new DAL buffer where the data is kept in memory
+func InMemoryBuffer() *inmemBuffer {
+	return &inmemBuffer{
+		// @note we'll buffer the value counters along side the actual value counter
+		// because constant hashmap initialization is quite memory intensive and introduces
+		// a bit of a bottleneck.
+		//
+		// @todo investigate if this can be optimized at a lower level
+		rows: make([]ValueGetter, 0, 100),
+		ctrs: make([]map[string]uint, 0, 100),
+
+		// i starts off as -1 so we don't need an extra state flag when doing the first Next
+		i:    -1,
+		more: true,
+	}
+}
+
+// InMemoryBufferWith returns a new buffer with the given value getters
+func InMemoryBufferWith(ctx context.Context, vv ...ValueGetter) (Buffer, error) {
+	b := InMemoryBuffer()
+	for _, v := range vv {
+		if err := b.Add(ctx, v); err != nil {
+			return nil, err
+		}
+	}
+	return b, nil
+}
+
+func (b *inmemBuffer) InOrder(ss ...*filter.SortExpr) (err error) {
+	b.sort = ss
+
+	return nil
+}
+
+// Single makes the buffer only keep one element at the time
+// @todo make this option convert the buffer into a circular buffer
+func (b *inmemBuffer) Single() {
+	b.cap = 1
+	b.rows = make([]ValueGetter, 1)
+	b.ctrs = make([]map[string]uint, 1)
+	b.i = -1
+}
+
+func (b *inmemBuffer) Seek(_ context.Context, i int) (err error) {
+	// we go one level further and start at -1
+	b.i = i - 1
+
+	b.more = i < b.Len()
+	return
+}
+
+func (b *inmemBuffer) isSingle() bool {
+	return b.cap == 1
+}
+
+func (b *inmemBuffer) isSorted() bool {
+	return len(b.sort) > 0
+}
+
+func (b *inmemBuffer) Add(ctx context.Context, v ValueGetter) (err error) {
+	if b.i != -1 && b.isSorted() {
+		return fmt.Errorf("cannot buffer items after an access occurred and the buffer is ordered")
+	}
+
+	if b.isSingle() {
+		b.rows[0] = v
+		b.ctrs[0] = v.CountValues()
+	} else {
+		b.rows = append(b.rows, v)
+		b.ctrs = append(b.ctrs, v.CountValues())
+	}
+
+	b.more = true
+	return
+}
+
+func (b *inmemBuffer) Next(ctx context.Context) bool {
+	if !b.more {
+		return false
+	}
+	if b.closed {
+		panic("cannot call Next on closed buffer")
+	}
+
+	// @todo use something like a b-tree to sort while inserting
+	if b.i == -1 && len(b.sort) > 0 {
+		sort.Sort(b)
+	}
+
+	b.i++
+	if b.isSingle() {
+		b.i = b.i % b.cap
+	}
+
+	b.more = !b.isSingle() && b.i+1 < b.Len()
+
+	return b.i < b.Len()
+}
+
+func (b *inmemBuffer) More(uint, ValueGetter) error {
+	return nil
+}
+
+func (b *inmemBuffer) Err() error {
+	return b.err
+}
+
+func (b *inmemBuffer) Scan(s ValueSetter) (err error) {
+	if b.closed {
+		panic("cannot call Scan on closed buffer")
+	}
+
+	r, ctrs, err := b.rowAt(b.i)
+	if err != nil {
+		return
+	}
+
+	var v any
+	for name, count := range ctrs {
+		for i := uint(0); i < count; i++ {
+			v, err = r.GetValue(name, i)
+			if err != nil {
+				return
+			}
+			err = s.SetValue(name, i, v)
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+// @todo should this get a context for things like IO ops
+func (b *inmemBuffer) Close() error {
+	b.ctrs = nil
+	b.rows = nil
+	b.closed = true
+
+	return nil
+}
+
+func (b *inmemBuffer) BackCursor(ValueGetter) (*filter.PagingCursor, error) {
+	return nil, fmt.Errorf("not supported")
+}
+
+func (b *inmemBuffer) ForwardCursor(ValueGetter) (*filter.PagingCursor, error) {
+	return nil, fmt.Errorf("not supported")
+}
+
+// rowAt returns the row at a given index
+func (b *inmemBuffer) rowAt(i int) (ValueGetter, map[string]uint, error) {
+	if i >= b.Len() {
+		return nil, nil, nil
+	}
+
+	return b.rows[i], b.ctrs[i], nil
+}
+
+// sort.Interface methods
+
+func (b *inmemBuffer) Len() int {
+	return len(b.rows)
+}
+
+func (b *inmemBuffer) Less(i, j int) bool {
+	ra, _, err := b.rowAt(i)
+	if err != nil {
+		panic(err)
+	}
+
+	rb, _, err := b.rowAt(j)
+	if err != nil {
+		panic(err)
+	}
+
+	return makeRowComparator(b.sort...)(ra, rb)
+}
+
+func (b *inmemBuffer) Swap(i, j int) {
+	b.rows[i], b.rows[j] = b.rows[j], b.rows[i]
+	b.ctrs[i], b.ctrs[j] = b.ctrs[j], b.ctrs[i]
+}

server/pkg/dal/inmem_buffer_test.go

@@ -0,0 +1,298 @@
+package dal
+
+import (
+	"context"
+	"testing"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/stretchr/testify/require"
+)
+
+func TestInmemBuffer_rw(t *testing.T) {
+	gBuff := InMemoryBuffer()
+	ctx := context.Background()
+
+	tcc := []struct {
+		name string
+		prep func() *inmemBuffer
+		in   []simpleRow
+		test func(t *testing.T, in []simpleRow, buff *inmemBuffer)
+	}{{
+		name: "fresh insert",
+		prep: func() *inmemBuffer {
+			return gBuff
+		},
+		in: []simpleRow{
+			{
+				"r1_k1": "r1_k1",
+				"r1_k2": "r1_k2",
+				"r1_k3": "r1_k3",
+			}},
+		test: func(t *testing.T, in []simpleRow, buff *inmemBuffer) {
+			r := make(simpleRow)
+
+			require.True(t, buff.Next(ctx))
+			require.NoError(t, buff.Scan(r))
+			require.Equal(t, in[0], r)
+
+			require.False(t, buff.Next(ctx))
+		},
+	}, {
+		name: "insert into existing",
+		prep: func() *inmemBuffer {
+			return gBuff
+		},
+		in: []simpleRow{
+			{
+				"r2_k1": "r2_k1",
+				"r2_k2": "r2_k2",
+				"r2_k3": "r2_k3",
+			}},
+		test: func(t *testing.T, in []simpleRow, buff *inmemBuffer) {
+			r := make(simpleRow)
+
+			require.True(t, buff.Next(ctx))
+			require.NoError(t, buff.Scan(r))
+			require.Equal(t, in[0], r)
+
+			require.False(t, buff.Next(ctx))
+		},
+	}}
+
+	for _, c := range tcc {
+		t.Run(c.name, func(t *testing.T) {
+			buff := c.prep()
+			for _, in := range c.in {
+				require.NoError(t, buff.Add(ctx, in))
+			}
+			c.test(t, c.in, buff)
+		})
+	}
+}
+
+func TestInmemBuffer_single_rw(t *testing.T) {
+	mk := func() *inmemBuffer {
+		buff := InMemoryBuffer()
+		buff.Single()
+		return buff
+	}
+	ctx := context.Background()
+
+	tcc := []struct {
+		name string
+		prep func() *inmemBuffer
+		in   []simpleRow
+		test func(t *testing.T, in []simpleRow, buff *inmemBuffer)
+	}{{
+		name: "one",
+		prep: func() *inmemBuffer {
+			return mk()
+		},
+		in: []simpleRow{
+			{
+				"r1_k1": "r1_k1",
+				"r1_k2": "r1_k2",
+				"r1_k3": "r1_k3",
+			}},
+		test: func(t *testing.T, in []simpleRow, buff *inmemBuffer) {
+			r := make(simpleRow)
+
+			require.True(t, buff.Next(ctx))
+			require.NoError(t, buff.Scan(r))
+			require.Equal(t, in[0], r)
+
+			require.False(t, buff.Next(ctx))
+		},
+	}, {
+		name: "two",
+		prep: func() *inmemBuffer {
+			return mk()
+		},
+		in: []simpleRow{
+			{
+				"r1_k1": "r1_k1",
+				"r1_k2": "r1_k2",
+				"r1_k3": "r1_k3",
+			}, {
+				"r2_k1": "r2_k1",
+				"r2_k2": "r2_k2",
+				"r2_k3": "r2_k3",
+			}},
+		test: func(t *testing.T, in []simpleRow, buff *inmemBuffer) {
+			r := make(simpleRow)
+
+			require.True(t, buff.Next(ctx))
+			require.NoError(t, buff.Scan(r))
+			require.Equal(t, in[1], r)
+
+			require.False(t, buff.Next(ctx))
+		},
+	}}
+
+	for _, c := range tcc {
+		t.Run(c.name, func(t *testing.T) {
+			buff := c.prep()
+			for _, in := range c.in {
+				require.NoError(t, buff.Add(ctx, in))
+			}
+			c.test(t, c.in, buff)
+		})
+	}
+}
+
+func TestInmemBuffer_single_writeMidRead(t *testing.T) {
+	ctx := context.Background()
+
+	buff := InMemoryBuffer()
+	buff.Single()
+
+	a := simpleRow{
+		"r1_k1": "r1_k1",
+		"r1_k2": "r1_k2",
+		"r1_k3": "r1_k3",
+	}
+
+	b := simpleRow{
+		"r2_k1": "r2_k1",
+		"r2_k2": "r2_k2",
+		"r2_k3": "r2_k3",
+	}
+
+	c := simpleRow{
+		"r3_k1": "r3_k1",
+		"r3_k2": "r3_k2",
+		"r3_k3": "r3_k3",
+	}
+
+	// Write
+	require.NoError(t, buff.Add(ctx, a))
+
+	// Read
+	tmp := simpleRow{}
+	require.True(t, buff.Next(ctx))
+	require.NoError(t, buff.Err())
+	require.NoError(t, buff.Scan(tmp))
+	require.Equal(t, a, tmp)
+
+	// Next write
+	require.NoError(t, buff.Add(ctx, b))
+	// Next write
+	require.NoError(t, buff.Add(ctx, c))
+
+	// Read
+	tmp = simpleRow{}
+	require.True(t, buff.Next(ctx))
+	require.NoError(t, buff.Err())
+	require.NoError(t, buff.Scan(tmp))
+	require.Equal(t, c, tmp)
+
+	// Read empty
+	require.False(t, buff.Next(ctx))
+	require.NoError(t, buff.Err())
+}
+
+func TestInmemBuffer_inOrder(t *testing.T) {
+	mk := func() *inmemBuffer {
+		buff := InMemoryBuffer()
+		return buff
+	}
+	ctx := context.Background()
+
+	tcc := []struct {
+		name string
+		prep func() *inmemBuffer
+		in   []simpleRow
+		out  []simpleRow
+		sort filter.SortExprSet
+	}{{
+		name: "in order",
+		prep: func() *inmemBuffer {
+			return mk()
+		},
+		in: []simpleRow{
+			{
+				"order": 3,
+			}, {
+				"order": 2,
+			}, {
+				"order": 4,
+			}, {
+				"order": 1,
+			}},
+		out: []simpleRow{
+			{
+				"order": 1,
+			}, {
+				"order": 2,
+			}, {
+				"order": 3,
+			}, {
+				"order": 4,
+			}},
+		sort: filter.SortExprSet{{
+			Column:     "order",
+			Descending: false,
+		}},
+	},
+	}
+
+	for _, c := range tcc {
+		t.Run(c.name, func(t *testing.T) {
+			buff := c.prep()
+			buff.InOrder(c.sort...)
+			for _, in := range c.in {
+				require.NoError(t, buff.Add(ctx, in))
+			}
+
+			out := make([]simpleRow, 0, 4)
+			for buff.Next(ctx) {
+				require.NoError(t, buff.Err())
+
+				r := simpleRow{}
+				require.NoError(t, buff.Scan(r))
+				out = append(out, r)
+			}
+
+			require.Equal(t, c.out, out)
+		})
+	}
+}
+
+func TestInmemBuffer_seek(t *testing.T) {
+	ctx := context.Background()
+
+	buff := InMemoryBuffer()
+
+	a := simpleRow{
+		"k": "k1",
+	}
+	b := simpleRow{
+		"k": "k2",
+	}
+	c := simpleRow{
+		"k": "k3",
+	}
+
+	tmp := simpleRow{}
+
+	// Write
+	require.NoError(t, buff.Add(ctx, a))
+	require.NoError(t, buff.Add(ctx, b))
+	require.NoError(t, buff.Add(ctx, c))
+
+	// Seek to end
+	require.NoError(t, buff.Seek(ctx, 4))
+	require.False(t, buff.Next(ctx))
+
+	// Seek to start
+	require.NoError(t, buff.Seek(ctx, 0))
+	require.True(t, buff.Next(ctx))
+	require.NoError(t, buff.Scan(tmp))
+	require.Equal(t, a, tmp)
+
+	// Seek to middle
+	require.NoError(t, buff.Seek(ctx, 2))
+	require.True(t, buff.Next(ctx))
+	require.NoError(t, buff.Scan(tmp))
+	require.Equal(t, c, tmp)
+}

server/pkg/dal/issues.go

@@ -0,0 +1,163 @@
+package dal
+
+import "go.uber.org/zap"
+
+type (
+	Issue struct {
+		err   error
+		Issue string `json:"issue,omitempty"`
+
+		Kind issueKind      `json:"kind,omitempty"`
+		Meta map[string]any `json:"meta,omitempty"`
+	}
+	issueSet []Issue
+
+	issueHelper struct {
+		// these two will be used to help clear out unneeded errors
+		connections []uint64
+		models      []uint64
+
+		connectionIssues dalIssueIndex
+		modelIssues      dalIssueIndex
+	}
+
+	issueKind     string
+	dalIssueIndex map[uint64]issueSet
+)
+
+const (
+	connectionIssue issueKind = "connection"
+	modelIssue      issueKind = "model"
+)
+
+func newIssueHelper() *issueHelper {
+	return &issueHelper{
+		connectionIssues: make(dalIssueIndex),
+		modelIssues:      make(dalIssueIndex),
+	}
+}
+
+func (svc *service) SearchConnectionIssues(connectionID uint64) (out []Issue) {
+	return svc.connectionIssues[connectionID]
+}
+
+func (svc *service) SearchModelIssues(resourceID uint64) (out []Issue) {
+	return svc.modelIssues[resourceID]
+}
+
+func (svc *service) SearchResourceIssues(resourceType, resource string) (out []Issue) {
+	var m *Model
+	for _, ax := range svc.models {
+		m = ax.FindByResourceIdent(resourceType, resource)
+	}
+
+	if m == nil {
+		return
+	}
+
+	return svc.modelIssues[m.ResourceID]
+}
+
+func (svc *service) hasConnectionIssues(connectionID uint64) bool {
+	return len(svc.SearchConnectionIssues(connectionID)) > 0
+}
+
+func (svc *service) hasModelIssues(modelID uint64) bool {
+	return len(svc.SearchModelIssues(modelID)) > 0
+}
+
+func (svc *service) updateIssues(issues *issueHelper) {
+	for _, connectionID := range issues.connections {
+		delete(svc.connectionIssues, connectionID)
+	}
+	for connectionID, issues := range issues.connectionIssues {
+		svc.connectionIssues[connectionID] = issues
+	}
+
+	for _, modelID := range issues.models {
+		delete(svc.modelIssues, modelID)
+	}
+	for modelID, issues := range issues.modelIssues {
+		svc.modelIssues[modelID] = issues
+	}
+}
+
+func (svc *service) clearModelIssues() {
+	svc.modelIssues = make(dalIssueIndex)
+}
+
+func (rd *issueHelper) addConnection(connectionID uint64) *issueHelper {
+	rd.connections = append(rd.connections, connectionID)
+	return rd
+}
+
+func (rd *issueHelper) addModel(modelID uint64) *issueHelper {
+	rd.models = append(rd.models, modelID)
+	return rd
+}
+
+func (rd *issueHelper) addConnectionIssue(connectionID uint64, i Issue) {
+	i.Kind = connectionIssue
+	i.Issue = i.err.Error()
+	rd.connectionIssues[connectionID] = append(rd.connectionIssues[connectionID], i)
+}
+
+func (rd *issueHelper) addModelIssue(resourceID uint64, i Issue) {
+	i.Kind = modelIssue
+	i.Issue = i.err.Error()
+	rd.modelIssues[resourceID] = append(rd.modelIssues[resourceID], i)
+}
+
+func (rd *issueHelper) hasConnectionIssues() bool {
+	return len(rd.connectionIssues) > 0
+}
+
+func (rd *issueHelper) hasModelIssues() bool {
+	return len(rd.modelIssues) > 0
+}
+
+func (a *issueHelper) mergeWith(b *issueHelper) {
+	if b == nil {
+		return
+	}
+
+	for connectionID, issues := range b.connectionIssues {
+		a.connectionIssues[connectionID] = append(a.connectionIssues[connectionID], issues...)
+	}
+
+	for modelID, issues := range b.modelIssues {
+		a.modelIssues[modelID] = append(a.modelIssues[modelID], issues...)
+	}
+}
+
+// Op check utils
+func (svc *service) canOpData(ref ModelRef) (err error) {
+	if svc.hasConnectionIssues(ref.ConnectionID) {
+		for _, i := range svc.connectionIssues[ref.ConnectionID] {
+			svc.logger.Debug(
+				"can not perform data operation due to connection issue: "+i.err.Error(),
+				zap.Any("ref", ref.ResourceID),
+			)
+		}
+
+		return errRecordOpProblematicConnection(ref.ConnectionID)
+	}
+
+	var mod = svc.FindModelByRef(ref)
+	if mod == nil {
+		return errModelNotFound(ref.ResourceID)
+	}
+
+	if svc.hasModelIssues(mod.ResourceID) {
+		for _, i := range svc.modelIssues[mod.ResourceID] {
+			svc.logger.Debug(
+				"can not perform data operation due to model issue: "+i.err.Error(),
+				zap.Any("ref", ref.ResourceID),
+			)
+		}
+
+		return errRecordOpProblematicModel(mod.ResourceID)
+	}
+
+	return nil
+}

server/pkg/dal/iterator.go

@@ -0,0 +1,119 @@
+package dal
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+)
+
+type (
+	// Iterator provides an interface for loading data from the underlying store
+	Iterator interface {
+		Next(ctx context.Context) bool
+		More(uint, ValueGetter) error
+		Err() error
+		Scan(ValueSetter) error
+		Close() error
+
+		BackCursor(ValueGetter) (*filter.PagingCursor, error)
+		ForwardCursor(ValueGetter) (*filter.PagingCursor, error)
+
+		// // -1 means unknown
+		// Total() int
+		// Cursor() any
+		// // ... do we need anything else here?
+	}
+
+	iterator interface {
+		Preload(context.Context, uint, *filter.PagingCursor) error
+		Sorting() filter.SortExprSet
+	}
+)
+
+// IteratorEncodeJSON helper function that encodes each item from the iterator as JSON
+// and writes it to th given io.Writer.
+//
+// target initialization function is intentionally used to avoid use of reflection
+func IteratorEncodeJSON(ctx context.Context, w io.Writer, iter Iterator, initTarget func() ValueSetter) (err error) {
+	var (
+		target   ValueSetter
+		firstOut = false
+	)
+
+	for iter.Next(ctx) {
+		if err = iter.Err(); err != nil {
+			return
+		}
+
+		if firstOut {
+			if _, err = w.Write([]byte(`,`)); err != nil {
+				return
+			}
+		}
+
+		firstOut = true
+
+		target = initTarget()
+
+		if err = iter.Scan(target); err != nil {
+			return
+		}
+
+		err = json.NewEncoder(w).Encode(target)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// PreLoadCursor into iterator and check it exist then return the cursor
+// @todo this should be move to the Iterator
+func PreLoadCursor(ctx context.Context, iter Iterator, limit uint, reverse bool, r ValueGetter, fx func(Iterator) (bool, error)) (out *filter.PagingCursor, err error) {
+	makeCursor := func() (*filter.PagingCursor, error) {
+		if reverse {
+			return iter.BackCursor(r)
+		} else {
+			return iter.ForwardCursor(r)
+		}
+	}
+
+	out, err = makeCursor()
+	if err != nil {
+		return
+	}
+
+	err = iter.(iterator).Preload(ctx, limit, out)
+	if err != nil {
+		return nil, nil
+	}
+
+	for {
+		if !iter.Next(ctx) {
+			out = nil
+			return
+		}
+
+		ok, err := fx(iter)
+		if err != nil {
+			return nil, err
+		}
+
+		if ok {
+			return out, err
+
+			// // @todo Skip the things we don't have access to; could cause some edge cases so probably not
+			// // It adds some performance since we skip unneeded stuff but could some records change in the mean time?
+			// // return makeCursor()
+		}
+	}
+}
+
+// IteratorSorting return iterator sorting
+// @todo this should be move to the Iterator
+func IteratorSorting(iter Iterator) filter.SortExprSet {
+	return iter.(iterator).Sorting()
+}

server/pkg/dal/main_test.go

@@ -0,0 +1,113 @@
+package dal
+
+import (
+	"context"
+	"testing"
+)
+
+type (
+	simpleAttribute struct {
+		ident      string
+		source     string
+		expr       string
+		primary    bool
+		multivalue bool
+		t          Type
+	}
+
+	simpleRow map[string]any
+)
+
+func saToMapping(sa ...simpleAttribute) []AttributeMapping {
+	out := make([]AttributeMapping, 0, len(sa))
+	for _, a := range sa {
+		out = append(out, a)
+	}
+
+	return out
+}
+
+func saToAggAttr(sa ...simpleAttribute) []AggregateAttr {
+	out := make([]AggregateAttr, 0, len(sa))
+	for _, a := range sa {
+		aux := AggregateAttr{
+			RawExpr:    a.expr,
+			Identifier: a.ident,
+			Type:       a.t,
+		}
+		if aux.RawExpr == "" {
+			aux.RawExpr = a.source
+		}
+		if aux.RawExpr == "" {
+			aux.RawExpr = a.ident
+		}
+		out = append(out, aux)
+	}
+
+	return out
+}
+
+func (sa simpleAttribute) Properties() MapProperties {
+	return MapProperties{
+		IsPrimary:    sa.primary,
+		Type:         sa.t,
+		IsMultivalue: sa.multivalue,
+	}
+}
+
+func (sa simpleAttribute) Identifier() (ident string) {
+	return sa.ident
+}
+
+func (sa simpleAttribute) Source() (expr string) {
+	if sa.source != "" {
+		return sa.source
+	}
+
+	return sa.ident
+}
+
+func (sa simpleAttribute) Expression() (expr string) {
+	if sa.expr != "" {
+		return sa.expr
+	}
+
+	if sa.source != "" {
+		return sa.source
+	}
+
+	return sa.ident
+}
+
+func bootstrapAggregate(rootTest *testing.T, run func(context.Context, *testing.T, *Aggregate, Buffer)) {
+	bootstrapAggregateNoOpt(rootTest, run)
+}
+
+func bootstrapAggregateNoOpt(rootTest *testing.T, run func(context.Context, *testing.T, *Aggregate, Buffer)) {
+	rootTest.Run("no optimization", func(t *testing.T) {
+		ctx := context.Background()
+		buff := InMemoryBuffer()
+		agg := &Aggregate{}
+
+		run(ctx, t, agg, buff)
+	})
+}
+
+func (r simpleRow) CountValues() map[string]uint {
+	out := make(map[string]uint)
+
+	for k := range r {
+		out[k]++
+	}
+
+	return out
+}
+
+func (r simpleRow) GetValue(k string, place uint) (any, error) {
+	return r[k], nil
+}
+
+func (r simpleRow) SetValue(k string, place uint, v any) error {
+	r[k] = v
+	return nil
+}

server/pkg/dal/model.go

@@ -0,0 +1,578 @@
+package dal
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/handle"
+	"github.com/modern-go/reflect2"
+)
+
+type (
+	// ModelRef is used to retrieve a model from the DAL based on given params
+	ModelRef struct {
+		ConnectionID uint64
+
+		ResourceID uint64
+
+		ResourceType string
+		Resource     string
+
+		Refs map[string]any
+	}
+
+	// Model describes the underlying data and its shape
+	Model struct {
+		ConnectionID uint64 `json:"connectionID"`
+		Ident        string `json:"ident"`
+		Label        string `json:"label"`
+
+		Resource     string `json:"resource"`
+		ResourceID   uint64 `json:"resourceID"`
+		ResourceType string `json:"resourceType"`
+
+		// Refs is an arbitrary map to identify a model
+		// @todo consider reworking this; I'm not the biggest fan
+		Refs map[string]any `json:"refs"`
+
+		SensitivityLevelID uint64 `json:"sensitivityLevelID"`
+
+		Attributes AttributeSet `json:"attributes"`
+
+		Constraints map[string][]any `json:"constraints"`
+		Indexes     IndexSet         `json:"indexes"`
+	}
+
+	ModelSet []*Model
+
+	// Attribute describes a specific value of the dataset
+	Attribute struct {
+		Ident string `json:"ident"`
+		Label string `json:"label"`
+
+		SensitivityLevelID uint64 `json:"sensitivityLevelID"`
+
+		MultiValue bool `json:"multiValue"`
+
+		PrimaryKey bool `json:"primaryKey"`
+
+		// If attribute has SoftDeleteFlag=true we use it
+		// when filtering out deleted items
+		SoftDeleteFlag bool `json:"softDeleteFlag"`
+
+		// System indicates the attribute was defined by the system
+		System bool `json:"system"`
+
+		// Is attribute sortable?
+		// Note: all primary keys are sortable
+		Sortable bool `json:"sortable"`
+
+		// Can attribute be used in query expression?
+		Filterable bool `json:"filterable"`
+
+		// Store describes the strategy the underlying storage system should
+		// apply to the underlying value
+		Store Codec `json:"store"`
+
+		// Type describes what the value represents and how it should be
+		// encoded/decoded
+		Type Type `json:"type"`
+	}
+
+	// auxAttribute is a helper struct used for marshaling/unmarshaling
+	//
+	// This is required since some fields are interfaces
+	auxAttribute struct {
+		Ident              string `json:"ident"`
+		Label              string `json:"label"`
+		SensitivityLevelID uint64 `json:"sensitivityLevelID"`
+		MultiValue         bool   `json:"multiValue"`
+		PrimaryKey         bool   `json:"primaryKey"`
+		SoftDeleteFlag     bool   `json:"softDeleteFlag"`
+		System             bool   `json:"system"`
+		Sortable           bool   `json:"sortable"`
+		Filterable         bool   `json:"filterable"`
+
+		Store *auxAttributeStore `json:"store"`
+		Type  *auxAttributeType  `json:"type"`
+	}
+
+	// auxAttributeStore is a helper struct used for marshaling/unmarshaling
+	//
+	// This is required since some fields are interfaces
+	auxAttributeStore struct {
+		Type string `json:"type"`
+
+		Plain              *CodecPlain              `json:"plain,omitempty"`
+		RecordValueSetJSON *CodecRecordValueSetJSON `json:"recordValueSetJSON,omitempty"`
+		Alias              *CodecAlias              `json:"alias,omitempty"`
+	}
+
+	// auxAttributeType is a helper struct used for marshaling/unmarshaling
+	//
+	// This is required since some fields are interfaces
+	auxAttributeType struct {
+		Type string `json:"type"`
+
+		ID        *TypeID        `json:"id,omitempty"`
+		Ref       *TypeRef       `json:"ref,omitempty"`
+		Timestamp *TypeTimestamp `json:"timestamp,omitempty"`
+		Time      *TypeTime      `json:"time,omitempty"`
+		Date      *TypeDate      `json:"date,omitempty"`
+		Number    *TypeNumber    `json:"number,omitempty"`
+		Text      *TypeText      `json:"text,omitempty"`
+		Boolean   *TypeBoolean   `json:"boolean,omitempty"`
+		Enum      *TypeEnum      `json:"enum,omitempty"`
+		Geometry  *TypeGeometry  `json:"geometry,omitempty"`
+		JSON      *TypeJSON      `json:"jSON,omitempty"`
+		Blob      *TypeBlob      `json:"blob,omitempty"`
+		UUID      *TypeUUID      `json:"uuid,omitempty"`
+	}
+
+	// auxStoreCodec is a helper struct used for marshaling/unmarshaling
+	//
+	// This is required since some fields are interfaces
+	auxStoreCodec struct {
+		Type string `json:"type"`
+
+		CodecPlain              *CodecPlain              `json:"codecPlain"`
+		CodecRecordValueSetJSON *CodecRecordValueSetJSON `json:"codecRecordValueSetJSON"`
+		CodecAlias              *CodecAlias              `json:"codecAlias"`
+	}
+
+	AttributeSet []*Attribute
+
+	Index struct {
+		Ident  string
+		Type   string
+		Unique bool
+
+		Fields []*IndexField
+
+		Predicate string
+	}
+
+	IndexField struct {
+		AttributeIdent string
+		Modifiers      []IndexFieldModifier
+		Sort           IndexFieldSort
+		Nulls          IndexFieldNulls
+	}
+
+	IndexSet []*Index
+
+	IndexFieldModifier string
+	IndexFieldSort     int
+	IndexFieldNulls    int
+)
+
+const (
+	IndexFieldSortDesc IndexFieldSort = -1
+	IndexFieldSortAsc  IndexFieldSort = 1
+
+	IndexFieldNullsLast  IndexFieldNulls = -1
+	IndexFieldNullsFirst IndexFieldNulls = 1
+
+	IndexFieldModifierLower = "LOWERCASE"
+)
+
+func PrimaryAttribute(ident string, codec Codec) *Attribute {
+	out := FullAttribute(ident, TypeID{}, codec)
+	out.Type = &TypeID{}
+	out.PrimaryKey = true
+	return out
+}
+
+func FullAttribute(ident string, at Type, codec Codec) *Attribute {
+	return &Attribute{
+		Ident:      ident,
+		Label:      ident,
+		Sortable:   true,
+		Filterable: true,
+		Store:      codec,
+		Type:       at,
+	}
+}
+
+func (a *Attribute) WithSoftDelete() *Attribute {
+	a.SoftDeleteFlag = true
+	return a
+}
+
+func (a *Attribute) WithMultiValue() *Attribute {
+	a.MultiValue = true
+	return a
+}
+
+func (a *Attribute) StoreIdent() string {
+	switch s := a.Store.(type) {
+	case *CodecRecordValueSetJSON:
+		return s.Ident
+
+	case *CodecAlias:
+		return s.Ident
+
+	default:
+		return a.Ident
+
+	}
+}
+
+// Compare the two attributes
+func (a *Attribute) Compare(b *Attribute) bool {
+	if a == nil || b == nil {
+		return a == b
+	}
+
+	out := true
+	out = out && a.Ident == b.Ident
+	out = out && a.Label == b.Label
+	out = out && a.SensitivityLevelID == b.SensitivityLevelID
+	out = out && a.MultiValue == b.MultiValue
+	out = out && a.PrimaryKey == b.PrimaryKey
+	out = out && a.SoftDeleteFlag == b.SoftDeleteFlag
+	out = out && a.System == b.System
+	out = out && a.Sortable == b.Sortable
+	out = out && a.Filterable == b.Filterable
+	if a.Store == nil || b.Store == nil {
+		out = out && a.Store == b.Store
+	} else {
+		out = out && a.Store.Type() == b.Store.Type()
+	}
+	if a.Type == nil || b.Type == nil {
+		out = out && a.Type == b.Type
+	} else {
+		out = out && a.Type.Type() == b.Type.Type()
+	}
+
+	return out
+}
+
+func (mm ModelSet) FindByResourceID(resourceID uint64) *Model {
+	for _, m := range mm {
+		if m.ResourceID == resourceID {
+			return m
+		}
+	}
+	return nil
+}
+
+func (mm ModelSet) FindByResourceIdent(resourceType, resourceIdent string) *Model {
+	for _, m := range mm {
+		if m.ResourceType != resourceType {
+			continue
+		}
+
+		if m.Resource != resourceIdent {
+			continue
+		}
+
+		return m
+	}
+	return nil
+}
+
+func (mm ModelSet) FindByIdent(ident string) *Model {
+	for _, m := range mm {
+		if m.Ident == ident {
+			return m
+		}
+	}
+
+	return nil
+}
+
+// FindByRefs returns the first Model that matches the given refs
+func (mm ModelSet) FindByRefs(refs map[string]any) *Model {
+	for _, model := range mm {
+		for k, v := range refs {
+			ref, ok := model.Refs[k]
+			if !ok {
+				goto skip
+			}
+			if v != ref {
+				goto skip
+			}
+		}
+
+		return model
+
+	skip:
+	}
+	return nil
+}
+
+// FilterByReferenced returns all of the models that reference b
+func (aa ModelSet) FilterByReferenced(b *Model) (out ModelSet) {
+	for _, aModel := range aa {
+		if aModel.Resource == b.Resource {
+			continue
+		}
+
+		for _, aAttribute := range aModel.Attributes {
+			switch casted := aAttribute.Type.(type) {
+			case *TypeRef:
+				if casted.RefModel.Resource == b.Resource {
+					out = append(out, aModel)
+				}
+			}
+		}
+	}
+
+	return
+}
+
+func (m Model) ToFilter() ModelRef {
+	return ModelRef{
+		ConnectionID: m.ConnectionID,
+
+		ResourceID: m.ResourceID,
+
+		ResourceType: m.ResourceType,
+		Resource:     m.Resource,
+	}
+}
+
+// HasAttribute returns true when the model includes the specified attribute
+func (m Model) HasAttribute(ident string) bool {
+	return m.Attributes.FindByIdent(ident) != nil
+}
+
+func (aa AttributeSet) FindByIdent(ident string) *Attribute {
+	for _, a := range aa {
+		if strings.EqualFold(a.Ident, ident) {
+			return a
+		}
+	}
+
+	return nil
+}
+
+func (aa AttributeSet) FindByStoreIdent(ident string) *Attribute {
+	for _, a := range aa {
+		if strings.EqualFold(a.StoreIdent(), ident) {
+			return a
+		}
+	}
+
+	return nil
+}
+
+// Validate performs a base model validation before it is passed down
+func (m Model) Validate() error {
+	if m.Resource == "" {
+		return fmt.Errorf("resource not defined")
+	}
+
+	seen := make(map[string]bool)
+	for _, attr := range m.Attributes {
+		if attr.Ident == "" {
+			return fmt.Errorf("invalid attribute ident: ident must not be empty")
+		}
+
+		if !handle.IsValid(attr.Ident) {
+			return fmt.Errorf("invalid attribute ident: %s is not a valid handle", attr.Ident)
+		}
+
+		if seen[attr.Ident] {
+			return fmt.Errorf("invalid attribute %s: duplicate attributes are not allowed", attr.Ident)
+		}
+		seen[attr.Ident] = true
+
+		if reflect2.IsNil(attr.Type) {
+			return fmt.Errorf("attribute does not define a type: %s", attr.Ident)
+		}
+	}
+
+	return nil
+}
+
+// Compare the two models
+//
+// This only checks model metadata, the attributes are excluded.
+func (a Model) Compare(b Model) bool {
+	out := true
+
+	out = out && a.ConnectionID == b.ConnectionID
+	out = out && a.Ident == b.Ident
+	out = out && a.Label == b.Label
+	out = out && a.Resource == b.Resource
+	out = out && a.ResourceID == b.ResourceID
+	out = out && a.ResourceType == b.ResourceType
+	out = out && a.SensitivityLevelID == b.SensitivityLevelID
+
+	return out
+}
+
+func (a *Attribute) MarshalJSON() ([]byte, error) {
+	aux := &auxAttribute{
+		Ident:              a.Ident,
+		Label:              a.Label,
+		SensitivityLevelID: a.SensitivityLevelID,
+		MultiValue:         a.MultiValue,
+		PrimaryKey:         a.PrimaryKey,
+		SoftDeleteFlag:     a.SoftDeleteFlag,
+		System:             a.System,
+		Sortable:           a.Sortable,
+		Filterable:         a.Filterable,
+
+		Store: &auxAttributeStore{},
+		Type:  &auxAttributeType{},
+	}
+
+	switch s := a.Store.(type) {
+	case *CodecPlain:
+		aux.Store.Type = "plain"
+		aux.Store.Plain = s
+
+	case *CodecRecordValueSetJSON:
+		aux.Store.Type = "recordValueSetJSON"
+		aux.Store.RecordValueSetJSON = s
+
+	case *CodecAlias:
+		aux.Store.Type = "alias"
+		aux.Store.Alias = s
+
+	default:
+		return nil, fmt.Errorf("unknown store codec type: %T", s)
+	}
+
+	switch t := a.Type.(type) {
+	case *TypeID:
+		aux.Type.Type = "ID"
+		aux.Type.ID = t
+
+	case *TypeRef:
+		aux.Type.Type = "Ref"
+		aux.Type.Ref = t
+
+	case *TypeTimestamp:
+		aux.Type.Type = "Timestamp"
+		aux.Type.Timestamp = t
+
+	case *TypeTime:
+		aux.Type.Type = "Time"
+		aux.Type.Time = t
+
+	case *TypeDate:
+		aux.Type.Type = "Date"
+		aux.Type.Date = t
+
+	case *TypeNumber:
+		aux.Type.Type = "Number"
+		aux.Type.Number = t
+
+	case *TypeText:
+		aux.Type.Type = "Text"
+		aux.Type.Text = t
+
+	case *TypeBoolean:
+		aux.Type.Type = "Boolean"
+		aux.Type.Boolean = t
+
+	case *TypeEnum:
+		aux.Type.Type = "Enum"
+		aux.Type.Enum = t
+
+	case *TypeGeometry:
+		aux.Type.Type = "Geometry"
+		aux.Type.Geometry = t
+
+	case *TypeJSON:
+		aux.Type.Type = "JSON"
+		aux.Type.JSON = t
+
+	case *TypeBlob:
+		aux.Type.Type = "Blob"
+		aux.Type.Blob = t
+
+	case *TypeUUID:
+		aux.Type.Type = "UUID"
+		aux.Type.UUID = t
+
+	default:
+		return nil, fmt.Errorf("unknown attribute type type: %T", t)
+	}
+
+	return json.Marshal(aux)
+}
+
+func (a *Attribute) UnmarshalJSON(data []byte) (err error) {
+	aux := &auxAttribute{}
+	err = json.Unmarshal(data, &aux)
+	if err != nil {
+		return err
+	}
+
+	if a == nil {
+		*a = Attribute{}
+	}
+
+	a.Ident = aux.Ident
+	a.Label = aux.Label
+	a.SensitivityLevelID = aux.SensitivityLevelID
+	a.MultiValue = aux.MultiValue
+	a.PrimaryKey = aux.PrimaryKey
+	a.SoftDeleteFlag = aux.SoftDeleteFlag
+	a.System = aux.System
+	a.Sortable = aux.Sortable
+	a.Filterable = aux.Filterable
+
+	switch aux.Store.Type {
+	case "plain":
+		a.Store = aux.Store.Plain
+
+	case "recordValueSetJSON":
+		a.Store = aux.Store.RecordValueSetJSON
+
+	case "alias":
+		a.Store = aux.Store.Alias
+	}
+
+	switch aux.Type.Type {
+	case "ID":
+		a.Type = aux.Type.ID
+
+	case "Ref":
+		a.Type = aux.Type.Ref
+
+	case "Timestamp":
+		a.Type = aux.Type.Timestamp
+
+	case "Time":
+		a.Type = aux.Type.Time
+
+	case "Date":
+		a.Type = aux.Type.Date
+
+	case "Number":
+		a.Type = aux.Type.Number
+
+	case "Text":
+		a.Type = aux.Type.Text
+
+	case "Boolean":
+		a.Type = aux.Type.Boolean
+
+	case "Enum":
+		a.Type = aux.Type.Enum
+
+	case "Geometry":
+		a.Type = aux.Type.Geometry
+
+	case "JSON":
+		a.Type = aux.Type.JSON
+
+	case "Blob":
+		a.Type = aux.Type.Blob
+
+	case "UUID":
+		a.Type = aux.Type.UUID
+
+	default:
+		return fmt.Errorf("unknown attribute type type: %s", aux.Type.Type)
+	}
+
+	return
+}

server/pkg/dal/model_test.go

@@ -0,0 +1,80 @@
+package dal
+
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestModelFindByRefs(t *testing.T) {
+	tcc := []struct {
+		name  string
+		in    ModelSet
+		refs  map[string]interface{}
+		found bool
+	}{
+		{
+			name: "one ref found",
+			in: ModelSet{
+				{Refs: map[string]any{"a": 1}},
+			},
+			refs:  map[string]any{"a": 1},
+			found: true,
+		},
+		{
+			name: "one ref not found",
+			in: ModelSet{
+				{Refs: map[string]any{"a": 1}},
+			},
+			refs:  map[string]any{"b": 1},
+			found: false,
+		},
+
+		{
+			name: "n refs found",
+			in: ModelSet{
+				{Refs: map[string]any{"a": 1, "b": 1, "c": 1}},
+			},
+			refs:  map[string]any{"a": 1, "b": 1},
+			found: true,
+		},
+		{
+			name: "n refs not found",
+			in: ModelSet{
+				{Refs: map[string]any{"a": 1, "b": 1}},
+			},
+			refs:  map[string]any{"a": 1, "b": 2},
+			found: false,
+		},
+	}
+
+	for _, tc := range tcc {
+		t.Run(tc.name, func(t *testing.T) {
+			m := tc.in.FindByRefs(tc.refs)
+			if tc.found {
+				require.NotNil(t, m)
+			} else {
+				require.Nil(t, m)
+			}
+		})
+	}
+}
+
+func TestAttributeMarshling(t *testing.T) {
+	a := &Attribute{
+		Ident: "foo",
+		Type:  &TypeBlob{},
+		Store: &CodecPlain{},
+	}
+
+	bb, err := json.Marshal(a)
+	require.NoError(t, err)
+
+	b := &Attribute{}
+	err = json.Unmarshal(bb, &b)
+	require.NoError(t, err)
+
+	require.True(t, reflect.DeepEqual(a, b))
+}

server/pkg/dal/normalizer.go

@@ -0,0 +1,54 @@
+package dal
+
+import "strings"
+
+// NormalizeAttrNames normalizes system identifiers to attribute identifiers
+// @todo this is temporary until we re-visit field referencing
+//
+// The normalization step must exist since legacy versions allowed multiple
+// identifier variations for the same system field, such as created_at and createdAt.
+func NormalizeAttrNames(name string) string {
+	switch strings.ToLower(name) {
+	case
+		"recordid",
+		"record_id",
+		"id":
+		return "ID"
+
+	case "moduleid",
+		"module_id",
+		"module":
+		return "moduleID"
+
+	case "owned_by",
+		"ownedby":
+		return "ownedBy"
+
+	case "created_by",
+		"createdby":
+		return "createdBy"
+
+	case "updated_by",
+		"updatedby":
+		return "updatedBy"
+
+	case "deleted_by",
+		"deletedby":
+		return "deletedBy"
+
+	case "created_at",
+		"createdat":
+		return "createdAt"
+
+	case "updated_at",
+		"updatedat":
+		return "updatedAt"
+
+	case "deleted_at",
+		"deletedat":
+		return "deletedAt"
+	}
+
+	return name
+
+}

server/pkg/dal/operations.go

@@ -0,0 +1,155 @@
+package dal
+
+type (
+	Operation    string
+	OperationSet []Operation
+)
+
+const (
+	Create  Operation = "corteza::dal:operation:create"
+	Update  Operation = "corteza::dal:operation:update"
+	Delete  Operation = "corteza::dal:operation:delete"
+	Search  Operation = "corteza::dal:operation:search"
+	Lookup  Operation = "corteza::dal:operation:lookup"
+	Paging  Operation = "corteza::dal:operation:paging"
+	Sorting Operation = "corteza::dal:operation:sorting"
+)
+
+var (
+	full = OperationSet{
+		Create,
+		Update,
+		Delete,
+		Search,
+		Lookup,
+		Paging,
+		Sorting,
+	}
+
+	createOperations = OperationSet{
+		Create,
+	}
+
+	updateOperations = OperationSet{
+		Update,
+	}
+
+	deleteOperations = OperationSet{
+		Update,
+	}
+
+	searchOperations = OperationSet{
+		Search,
+		Paging,
+		Sorting,
+	}
+
+	lookupOperations = OperationSet{
+		Lookup,
+	}
+)
+
+// FullOperations returns all base system defined operations
+func FullOperations() (cc OperationSet) {
+	// Doing an union just to make a fresh copy
+	return full.Union(nil)
+}
+
+// CreateOperations returns only requested operations used for Create operations
+func CreateOperations(requested ...Operation) (required OperationSet) {
+	return common(createOperations, requested)
+}
+
+// UpdateOperations returns only requested operations used for Update operations
+func UpdateOperations(requested ...Operation) (required OperationSet) {
+	return common(updateOperations, requested)
+}
+
+// DeleteOperations returns only requested operations used for delete operations
+func DeleteOperations(requested ...Operation) (required OperationSet) {
+	return common(deleteOperations, requested)
+}
+
+// SearchOperations returns only requested operations used for Search operations
+func SearchOperations(requested ...Operation) (required OperationSet) {
+	return common(searchOperations, requested)
+}
+
+// LookupOperations returns only requested operations used for Search operations
+func LookupOperations(requested ...Operation) (required OperationSet) {
+	return common(lookupOperations, requested)
+}
+
+func common(aa, bb OperationSet) OperationSet {
+	return aa.Intersect(bb)
+}
+
+// ---
+
+// IsSuperset is inverse IsSubset
+//
+// IsSuperset checks if all bb operations are inside aa
+func (aa OperationSet) IsSuperset(bb ...Operation) bool {
+	return OperationSet(bb).IsSubset(aa...)
+}
+
+// IsSubset checks if all aa operations are inside bb
+func (aa OperationSet) IsSubset(bb ...Operation) bool {
+	if len(aa) > len(bb) {
+		return false
+	}
+
+	// When A is subset of B, the difference between the two must be 0
+	return len(aa.Diff(bb)) == 0
+}
+
+// Intersect returns the intersection between the two sets
+func (aa OperationSet) Intersect(bb OperationSet) (cc OperationSet) {
+	cc = make(OperationSet, 0, len(aa))
+	for _, a := range aa {
+		for _, b := range bb {
+			if a == b {
+				cc = append(cc, a)
+				break
+			}
+		}
+	}
+
+	return
+}
+
+// Intersect returns the union between the two sets
+//
+// Duplicates are omitted
+func (aa OperationSet) Union(bb OperationSet) (cc OperationSet) {
+	ix := make(map[Operation]bool)
+	for _, c := range append(aa, bb...) {
+		if !ix[c] {
+			ix[c] = true
+			cc = append(cc, c)
+		}
+	}
+	return
+}
+
+// Diff calculates the difference between the two operation sets
+//
+// The diff uses aa as base
+func (aa OperationSet) Diff(bb OperationSet) (cc OperationSet) {
+	for _, a := range aa {
+		found := false
+		for _, b := range bb {
+			found = a == b
+			if found {
+				break
+			}
+		}
+
+		if found {
+			continue
+		}
+		cc = append(cc, a)
+	}
+
+	return
+}

server/pkg/dal/operations_test.go

@@ -0,0 +1,127 @@
+package dal
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestCommonOperations(t *testing.T) {
+	cases := []struct {
+		name string
+		aa   OperationSet
+		bb   OperationSet
+		cc   OperationSet
+	}{{
+		name: "regular",
+		aa: OperationSet{
+			Create,
+			Update,
+		},
+		bb: OperationSet{
+			Update,
+			Delete,
+		},
+		cc: OperationSet{
+			Update,
+		},
+	}, {
+		name: "no commons",
+		aa: OperationSet{
+			Create,
+		},
+		bb: OperationSet{
+			Delete,
+		},
+		cc: OperationSet{},
+	}, {
+		name: "aa empty",
+		aa:   nil,
+		bb: OperationSet{
+			Update,
+			Delete,
+		},
+		cc: OperationSet{},
+	}, {
+		name: "bb empty",
+		aa: OperationSet{
+			Create,
+			Update,
+		},
+		bb: nil,
+		cc: OperationSet{},
+	}}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			cc := common(c.aa, c.bb)
+			require.Equal(t, c.cc, cc)
+		})
+	}
+}
+
+func TestOperationChecking(t *testing.T) {
+	cases := []struct {
+		name    string
+		support OperationSet
+		require OperationSet
+		out     bool
+	}{{
+		name: "passing: complete match",
+		support: OperationSet{
+			Create,
+			Update,
+		},
+		require: OperationSet{
+			Create,
+			Update,
+		},
+		out: true,
+	}, {
+		name: "passing: supports more then required",
+		support: OperationSet{
+			Create,
+			Update,
+			Delete,
+		},
+		require: OperationSet{
+			Create,
+			Update,
+		},
+		out: true,
+	}, {
+		name: "passing: no required",
+		support: OperationSet{
+			Create,
+			Update,
+			Delete,
+		},
+		require: OperationSet{},
+		out:     true,
+	}, {
+		name:    "passing: no required nor supported",
+		support: OperationSet{},
+		require: OperationSet{},
+		out:     true,
+	}, {
+		name: "failing: missing support",
+		support: OperationSet{
+			Create,
+			Update,
+		},
+		require: OperationSet{
+			Delete,
+		},
+		out: false,
+	}}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			out := c.support.IsSuperset(c.require...)
+			require.Equal(t, c.out, out)
+
+			out = c.require.IsSubset(c.support...)
+			require.Equal(t, c.out, out)
+		})
+	}
+}

server/pkg/dal/optimization.go

@@ -0,0 +1,148 @@
+package dal
+
+import "fmt"
+
+type (
+	// opCost provides a general idea of expensive an operation is for a
+	// specific pipeline step.
+	//
+	// dsSize provides a general idea of how large an underlaying dataset is.
+	opCost int
+	dsSize int
+
+	OpAnalysis struct {
+		ScanCost   opCost
+		SearchCost opCost
+		FilterCost opCost
+		SortCost   opCost
+
+		OutputSize dsSize
+	}
+
+	ppStepWrap struct {
+		step PipelineStep
+
+		parent *ppStepWrap
+		child  []*ppStepWrap
+	}
+)
+
+var (
+	pipelineOptimizers = []func(Pipeline) (Pipeline, error){
+		pipelineClobberSteps,
+	}
+)
+
+const (
+	// operation computation indicators
+	CostUnknown opCost = iota
+	CostFree
+	CostCheep
+	CostAcceptable
+	CostExpensive
+	CostInfinite
+)
+
+const (
+	// dataset size indicators
+	SizeUnknown dsSize = iota
+	SizeTiny
+	SizeSmall
+	SizeMedium
+	SizeLarge
+)
+
+const (
+	OpAnalysisIterate   string = "iterate"
+	OpAnalysisAggregate string = "aggregate"
+	OpAnalysisJoin      string = "join"
+)
+
+// wrapPpSteps wraps the pipeline steps in a more processing friendly format
+// and returns a slice of leave nodes
+//
+// @todo the pipeline representation might change which will make this obsolete
+func wrapPpSteps(pp Pipeline) (leaves []*ppStepWrap) {
+	ix := make(map[PipelineStep]*ppStepWrap)
+
+	for _, p := range pp {
+		ix[p] = &ppStepWrap{step: p}
+	}
+
+	for _, p := range pp {
+		switch c := p.(type) {
+		case *Aggregate:
+			ix[p].child = append(ix[p].child, ix[c.rel])
+			ix[c.rel].parent = ix[p]
+
+		case *Join:
+			ix[p].child = append(ix[p].child, ix[c.relLeft])
+			ix[c.relLeft].parent = ix[p]
+
+			ix[p].child = append(ix[p].child, ix[c.relRight])
+			ix[c.relRight].parent = ix[p]
+
+		case *Link:
+			ix[p].child = append(ix[p].child, ix[c.relLeft])
+			ix[c.relLeft].parent = ix[p]
+
+			ix[p].child = append(ix[p].child, ix[c.relRight])
+			ix[c.relRight].parent = ix[p]
+
+		case *Datasource:
+			continue
+
+		default:
+			panic(fmt.Errorf("impossible state: unknown pipeline step type %v", p))
+		}
+	}
+
+	for _, a := range ix {
+		if len(a.child) == 0 {
+			leaves = append(leaves, a)
+		}
+	}
+
+	return
+}
+
+// unwrapPpSteps unwraps the wrapped pipeline step into the classic Pipeline
+//
+// @todo the pipeline representation might change which will make this obsolete
+func unwrapPpSteps(n *ppStepWrap, seen map[*ppStepWrap]bool) (out Pipeline) {
+	for i, c := range n.child {
+		switch s := n.step.(type) {
+		case *Aggregate:
+			s.rel = c.step
+			s.RelSource = c.step.Identifier()
+		case *Join:
+			if i == 0 {
+				s.relLeft = c.step
+				s.RelLeft = c.step.Identifier()
+			} else {
+				s.relRight = c.step
+				s.RelRight = c.step.Identifier()
+			}
+		case *Link:
+			if i == 0 {
+				s.relLeft = c.step
+				s.RelLeft = c.step.Identifier()
+			} else {
+				s.relRight = c.step
+				s.RelRight = c.step.Identifier()
+			}
+		}
+	}
+
+	// @todo potentially cancancel sooner
+	if !seen[n] {
+		out = append(out, n.step)
+	}
+	seen[n] = true
+
+	if n.parent != nil {
+		out = append(unwrapPpSteps(n.parent, seen), out...)
+	}
+
+	return
+}

server/pkg/dal/optimization_clobber.go

@@ -0,0 +1,69 @@
+package dal
+
+// pipelineClobberSteps tries to reduce the amount of pipeline steps by offloading
+// higher operations to the lower levels
+//
+// As an example; the aggregation can be offloaded to the database for faster
+// execution.
+func pipelineClobberSteps(in Pipeline) (Pipeline, error) {
+	if len(in) <= 1 {
+		// Can't optimize further :upsidedownface:
+		return in, nil
+	}
+
+	// Outline
+	// - get a nicer pipeline representation
+	// 	 @todo make the pipeline nicer in the first place
+	// - traverse from the lief nodes up; try to clobber if the lief node allows it
+	//
+	// The clobbering for a branch ends when there is a node that can't be clobbered.
+	// The progression ends because application level nodes can't be offloaded to.
+	ll := wrapPpSteps(in)
+	for _, l := range ll {
+		for {
+			// When there is no parent, we can't progress further
+			if l.parent == nil {
+				break
+			}
+
+			// if step can't clobber, skip
+			cs, ok := l.step.(clobberableStep)
+			if !ok {
+				break
+			}
+
+			// if child fails to clobber parent, skip to the next child
+			// @note for now we can end the clobbering if any of the steps
+			//       can't be clobbered as all of the application defined steps
+			//       are focused on the single op. and can't do anything else.
+			if !cs.clobber(l.parent.step) {
+				break
+			}
+
+			// if clobbered successfully, update references
+			if l.parent != nil && l.parent.parent != nil {
+				// - update child ref of the parent's parent
+				for i, c := range l.parent.parent.child {
+					if c == l.parent {
+						l.parent.parent.child[i] = l
+					}
+				}
+			}
+
+			l.parent = l.parent.parent
+
+			// @todo for now, clobbering ends after one successfull instance; this is due
+			//       to the current DB implementation doesn't allow nested things.
+			break
+		}
+	}
+
+	// convert back to the pipeline representation; update deps in the process
+	out := make(Pipeline, 0, len(in))
+	seen := make(map[*ppStepWrap]bool, len(in))
+	for _, l := range ll {
+		out = append(out, unwrapPpSteps(l, seen)...)
+	}
+
+	return out, nil
+}

server/pkg/dal/optimization_clobber_test.go

@@ -0,0 +1,179 @@
+package dal
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func makeAnalysisDsAggregate() map[string]OpAnalysis {
+	return map[string]OpAnalysis{
+		OpAnalysisAggregate: {},
+	}
+}
+
+func TestClobberStep(t *testing.T) {
+	t.Run("no steps", func(t *testing.T) {
+		out, err := pipelineClobberSteps(nil)
+
+		require.NoError(t, err)
+		require.Nil(t, out)
+	})
+
+	t.Run("one step no optimize", func(t *testing.T) {
+		out, err := pipelineClobberSteps(Pipeline{&Datasource{}})
+
+		require.NoError(t, err)
+		require.Len(t, out, 1)
+	})
+
+	t.Run("agg ds", func(t *testing.T) {
+		ds := &Datasource{
+			Ident:    "ds_1",
+			analysis: makeAnalysisDsAggregate(),
+		}
+		agg := &Aggregate{
+			Ident:     "agg_1",
+			RelSource: "ds_1",
+			rel:       ds,
+		}
+
+		out, err := pipelineClobberSteps(Pipeline{agg, ds})
+
+		require.NoError(t, err)
+		require.Len(t, out, 1)
+		c := out[0].(*Datasource)
+		require.Len(t, c.clobbered, 1)
+	})
+
+	t.Run("agg agg ds", func(t *testing.T) {
+		// @note for now we're only offloading one aggregation
+		ds := &Datasource{
+			Ident:    "ds_1",
+			analysis: makeAnalysisDsAggregate(),
+		}
+		agg1 := &Aggregate{
+			Ident:     "agg_1",
+			RelSource: "ds_1",
+			rel:       ds,
+		}
+		agg2 := &Aggregate{
+			Ident:     "agg_2",
+			RelSource: "agg_1",
+			rel:       agg1,
+		}
+
+		out, err := pipelineClobberSteps(Pipeline{agg2, agg1, ds})
+
+		require.NoError(t, err)
+		require.Len(t, out, 2)
+		c := out[1].(*Datasource)
+		require.Len(t, c.clobbered, 1)
+	})
+
+	t.Run("join ds ds", func(t *testing.T) {
+		ds1 := &Datasource{
+			Ident:    "ds_1",
+			analysis: makeAnalysisDsAggregate(),
+		}
+		ds2 := &Datasource{
+			Ident:    "ds_2",
+			analysis: makeAnalysisDsAggregate(),
+		}
+
+		join := &Join{
+			Ident:    "join_1",
+			RelLeft:  "ds_1",
+			RelRight: "ds_2",
+
+			relLeft:  ds1,
+			relRight: ds2,
+		}
+
+		out, err := pipelineClobberSteps(Pipeline{join, ds1, ds2})
+
+		require.NoError(t, err)
+		require.Len(t, out, 3)
+	})
+
+	t.Run("join join ds ds ds", func(t *testing.T) {
+		ds1 := &Datasource{
+			Ident:    "ds_1",
+			analysis: makeAnalysisDsAggregate(),
+		}
+		ds2 := &Datasource{
+			Ident:    "ds_2",
+			analysis: makeAnalysisDsAggregate(),
+		}
+		ds3 := &Datasource{
+			Ident:    "ds_3",
+			analysis: makeAnalysisDsAggregate(),
+		}
+
+		join1 := &Join{
+			Ident:    "join_1",
+			RelLeft:  "ds_1",
+			RelRight: "ds_2",
+
+			relLeft:  ds1,
+			relRight: ds2,
+		}
+		join2 := &Join{
+			Ident:    "join_2",
+			RelLeft:  "join_1",
+			RelRight: "ds_3",
+
+			relLeft:  join1,
+			relRight: ds3,
+		}
+
+		out, err := pipelineClobberSteps(Pipeline{join2, join1, ds1, ds2, ds3})
+
+		require.NoError(t, err)
+		require.Len(t, out, 5)
+	})
+
+	t.Run("join join agg ds ds ds", func(t *testing.T) {
+		ds1 := &Datasource{
+			Ident:    "ds_1",
+			analysis: makeAnalysisDsAggregate(),
+		}
+		agg := &Aggregate{
+			Ident:     "agg_1",
+			RelSource: "ds_1",
+			rel:       ds1,
+		}
+		ds2 := &Datasource{
+			Ident:    "ds_2",
+			analysis: makeAnalysisDsAggregate(),
+		}
+		ds3 := &Datasource{
+			Ident:    "ds_3",
+			analysis: makeAnalysisDsAggregate(),
+		}
+
+		join1 := &Join{
+			Ident:    "join_1",
+			RelLeft:  "agg_1",
+			RelRight: "ds_2",
+
+			relLeft:  agg,
+			relRight: ds2,
+		}
+		join2 := &Join{
+			Ident:    "join_2",
+			RelLeft:  "join_1",
+			RelRight: "ds_3",
+
+			relLeft:  join1,
+			relRight: ds3,
+		}
+
+		out, err := pipelineClobberSteps(Pipeline{join2, join1, agg, ds1, ds2, ds3})
+
+		require.NoError(t, err)
+		require.Len(t, out, 5)
+
+		require.Len(t, ds1.clobbered, 1)
+	})
+}

server/pkg/dal/pipeline.go

@@ -0,0 +1,162 @@
+package dal
+
+import (
+	"context"
+	"fmt"
+)
+
+type (
+	// Pipeline defines a series of steps performed over the data
+	//
+	// General outline of how a pipeline is used:
+	// 1. Populate the pipeline with steps to define the desired outcome
+	// 2. Analyze the pipeline to collect statistics and assist optimizers
+	// 3. Run structure optimization which may reorder/drop steps
+	// 4. Run step optimization which may re-configure individual steps such as
+	//    push parts of a filter to lower level, request specific sort order, ...
+	// 5. Use the pipeline as an iterator to pull the data from.
+	Pipeline []PipelineStep
+
+	// PipelineStep defines an operation performed over the data in the pipeline
+	PipelineStep interface {
+		Identifier() string
+		Sources() []string
+		Attributes() [][]AttributeMapping
+
+		Analyze(ctx context.Context) error
+		Analysis() map[string]OpAnalysis
+	}
+
+	// clobberableStep can have other steps offload their work to it.
+	// This is primarily used to offload the work to the database.
+	clobberableStep interface {
+		clobber(PipelineStep) bool
+	}
+
+	// Attribute mapping outlines specific attributes within a pipeline
+	// @todo reconsider this interface; potentially remove it or split it up
+	AttributeMapping interface {
+		Identifier() string
+		Expression() (expression string)
+		Source() (ident string)
+		Properties() MapProperties
+	}
+
+	// MapProperties describe the attribute such as it's type and constraints
+	MapProperties struct {
+		Label               string
+		IsPrimary           bool
+		IsSystem            bool
+		IsFilterable        bool
+		IsSortable          bool
+		IsMultivalue        bool
+		MultivalueDelimiter string
+		Nullable            bool
+		Type                Type
+	}
+)
+
+// LinkSteps links related steps into a tree structure
+//
+// @todo make it return a new slice and not mutate the original
+func (pp Pipeline) LinkSteps() (err error) {
+	// map steps by identifiers
+	steps := make(map[string]PipelineStep)
+	for _, s := range pp {
+		steps[s.Identifier()] = s
+	}
+
+	// Link up...
+	err = func() (err error) {
+		for _, s := range pp {
+			switch rs := s.(type) {
+			case *Aggregate:
+				rs.rel = steps[rs.RelSource]
+				if rs.rel == nil {
+					return fmt.Errorf("aggregate: missing source relation %s", rs.RelSource)
+				}
+
+			case *Join:
+				rs.relLeft = steps[rs.RelLeft]
+				rs.relRight = steps[rs.RelRight]
+				if rs.relLeft == nil {
+					return fmt.Errorf("join: missing left relation %s", rs.relLeft)
+				}
+				if rs.relRight == nil {
+					return fmt.Errorf("join: missing right relation %s", rs.relRight)
+				}
+
+			case *Link:
+				rs.relLeft = steps[rs.RelLeft]
+				rs.relRight = steps[rs.RelRight]
+				if rs.relLeft == nil {
+					return fmt.Errorf("link: missing left relation %s", rs.relLeft)
+				}
+				if rs.relRight == nil {
+					return fmt.Errorf("link: missing right relation %s", rs.relRight)
+				}
+			}
+		}
+		return
+	}()
+	if err != nil {
+		return fmt.Errorf("unable to link steps: %w", err)
+	}
+
+	return nil
+}
+
+func (pp Pipeline) root() PipelineStep {
+	ix := make(map[string]PipelineStep)
+
+	for _, s := range pp {
+		ix[s.Identifier()] = s
+	}
+
+	for _, s := range pp {
+		for _, src := range s.Sources() {
+			delete(ix, src)
+		}
+	}
+
+	for _, s := range ix {
+		return ix[s.Identifier()]
+	}
+
+	panic("impossible state: no root-level pipeline step")
+}
+
+// Slice returns a new pipeline with all steps of the subtree with ident as root
+func (pp Pipeline) Slice(ident string) (out Pipeline) {
+	// Find root
+	var r PipelineStep
+	for _, p := range pp {
+		if p.Identifier() == ident {
+			r = p
+			break
+		}
+	}
+
+	return pp.slice(r)
+}
+
+// slice is the recursive counterpart for the Slice method
+func (pp Pipeline) slice(s PipelineStep) (out Pipeline) {
+	out = append(out, s)
+
+	switch n := s.(type) {
+	case *Datasource:
+		return
+
+	case *Aggregate:
+		return append(out, pp.slice(n.rel)...)
+
+	case *Join:
+		return append(out, append(pp.slice(n.relLeft), pp.slice(n.relRight)...)...)
+
+	case *Link:
+		return append(out, append(pp.slice(n.relLeft), pp.slice(n.relRight)...)...)
+	}
+
+	return
+}

server/pkg/dal/rel_index.go

@@ -0,0 +1,146 @@
+package dal
+
+import (
+	"fmt"
+
+	"github.com/spf13/cast"
+)
+
+type (
+	// relIndex is a generic struct for indexing data which join/link can use
+	//
+	// The current index implementation utilizes a series of hashmaps based on what
+	// type of values we're indexing on.
+	// As we currently only support single value predicate, the hashmaps proved
+	// to be a bit more efficient then b-trees (other consideration) when testing
+	// on larger datasets.
+	//
+	// @todo when we support multiple join predicates, this should probably change
+	// into a b-tree as it might be a bit faster thennested hashmaps.
+	//
+	// @todo do some benchmarks in regards to using generics for key
+	relIndex struct {
+		track []string
+
+		keyType Type
+
+		ints    map[int64]*relIndexBuffer
+		strings map[string]*relIndexBuffer
+		ids     map[uint64]*relIndexBuffer
+	}
+)
+
+// newRelIndex initializes a new relIndex with the specified tracked attributes
+// @todo benchmark with generics as key
+func newRelIndex(t Type, track ...string) (out *relIndex, err error) {
+	out = &relIndex{
+		track:   track,
+		keyType: t,
+	}
+
+	return out, out.initHashmaps()
+}
+
+// Add adds a new row to the index under the specified key
+func (ri *relIndex) Add(k any, r *Row) {
+	switch ri.keyType.(type) {
+	case TypeNumber, *TypeNumber:
+		ri.addInt(cast.ToInt64(k), r)
+		return
+
+	case TypeText, *TypeText:
+		ri.addString(cast.ToString(k), r)
+		return
+
+	case TypeID, *TypeID,
+		TypeRef, *TypeRef:
+		ri.addID(cast.ToUint64(k), r)
+		return
+	}
+
+	// @note this is validated when initializing
+	panic(fmt.Sprintf("cannot use type %s as index key", ri.keyType.Type()))
+}
+
+func (ri *relIndex) Get(k any) (out *relIndexBuffer, ok bool) {
+	switch ri.keyType.(type) {
+	case TypeNumber, *TypeNumber:
+		return ri.getInt(cast.ToInt64(k))
+
+	case TypeText, *TypeText:
+		return ri.getString(cast.ToString(k))
+
+	case TypeID, *TypeID,
+		TypeRef, *TypeRef:
+		return ri.getID(cast.ToUint64(k))
+	}
+
+	// @note this is validated when initializing
+	panic(fmt.Sprintf("cannot use type %s as index key", ri.keyType.Type()))
+}
+
+// Clear clears out the index omitting the need to reinitialize
+func (ri *relIndex) Clear() (err error) {
+	return ri.initHashmaps()
+}
+
+func (ri *relIndex) initHashmaps() (err error) {
+	// @note initHashmaps initializes only the ones we need to save up on space
+	switch ri.keyType.(type) {
+	case TypeNumber, *TypeNumber:
+		ri.ints = make(map[int64]*relIndexBuffer, 512)
+		return
+
+	case TypeText, *TypeText:
+		ri.strings = make(map[string]*relIndexBuffer, 512)
+		return
+
+	case TypeID, *TypeID,
+		TypeRef, *TypeRef:
+		ri.ids = make(map[uint64]*relIndexBuffer, 512)
+		return
+	}
+	return fmt.Errorf("cannot use type %s as index key", ri.keyType.Type())
+}
+
+func (ri *relIndex) addInt(k int64, r *Row) {
+	c, ok := ri.getInt(k)
+	if !ok {
+		c = newRelIndexBuffer(ri.track...)
+		ri.ints[k] = c
+	}
+	c.add(r)
+}
+
+func (ri *relIndex) getInt(k int64) (out *relIndexBuffer, ok bool) {
+	out, ok = ri.ints[k]
+	return
+}
+
+func (ri *relIndex) addString(k string, r *Row) {
+	c, ok := ri.getString(k)
+	if !ok {
+		c = newRelIndexBuffer(ri.track...)
+		ri.strings[k] = c
+	}
+	c.add(r)
+}
+
+func (ri *relIndex) getString(k string) (out *relIndexBuffer, ok bool) {
+	out, ok = ri.strings[k]
+	return
+}
+
+func (ri *relIndex) addID(k uint64, r *Row) {
+	c, ok := ri.getID(k)
+	if !ok {
+		c = newRelIndexBuffer(ri.track...)
+		ri.ids[k] = c
+	}
+	c.add(r)
+}
+
+func (ri *relIndex) getID(k uint64) (out *relIndexBuffer, ok bool) {
+	out, ok = ri.ids[k]
+	return
+}

server/pkg/dal/rel_index_buffer.go

@@ -0,0 +1,71 @@
+package dal
+
+type (
+	relIndexBuffer struct {
+		// attribute statistics to track
+		//
+		// These stats can be used when performing operations like sorting and
+		// binary searching for specific values.
+		track []string
+
+		min map[string]any
+		max map[string]any
+
+		// @todo make b-tree for sorting? will probably be small so simple slice sort
+		// should be ok ig...
+		//
+		// Probably don't need to sort yet
+		rows []*Row
+	}
+)
+
+// newRelIndexBuffer initializes a new relIndexBuffer tracking the given attributes
+func newRelIndexBuffer(tt ...string) *relIndexBuffer {
+	return &relIndexBuffer{
+		min:   make(map[string]any),
+		max:   make(map[string]any),
+		track: tt,
+	}
+}
+
+// add adds a new *row to the buffer
+func (lc *relIndexBuffer) add(r *Row) {
+	if len(lc.rows) == 0 {
+		for _, ix := range lc.track {
+			lc.min[ix] = r.values[ix][0]
+			lc.max[ix] = r.values[ix][0]
+		}
+
+		lc.rows = append(lc.rows, r)
+		return
+	}
+
+	lc.updMin(r)
+	lc.updMax(r)
+
+	lc.rows = append(lc.rows, r)
+}
+
+// updMin updates the min stat
+func (lc *relIndexBuffer) updMin(r *Row) {
+	for _, ix := range lc.track {
+		for i := uint(0); i < r.CountValues()[ix]; i++ {
+			v := r.values[ix][i]
+			if compareValues(v, lc.min[ix]) < 0 {
+				lc.min[ix] = v
+			}
+		}
+	}
+}
+
+// updMax updates the max stat
+func (lc *relIndexBuffer) updMax(r *Row) {
+	for _, ix := range lc.track {
+		for i := uint(0); i < r.CountValues()[ix]; i++ {
+			v := r.values[ix][i]
+			if compareValues(v, lc.max[ix]) > 0 {
+				lc.max[ix] = v
+			}
+		}
+	}
+}

server/pkg/dal/rel_index_buffer_test.go

@@ -0,0 +1,31 @@
+package dal
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestRelIndexBuffer(t *testing.T) {
+	cc := newRelIndexBuffer("a", "b")
+
+	cc.add((&Row{}).WithValue("a", 0, 1).WithValue("b", 0, "a"))
+
+	require.Len(t, cc.rows, 1)
+	require.Equal(t, 1, cc.min["a"])
+	require.Equal(t, 1, cc.max["a"])
+	require.Equal(t, "a", cc.min["b"])
+	require.Equal(t, "a", cc.max["b"])
+
+	cc.add((&Row{}).WithValue("a", 0, -1).WithValue("b", 0, "a"))
+	require.Equal(t, -1, cc.min["a"])
+	require.Equal(t, 1, cc.max["a"])
+	require.Equal(t, "a", cc.min["b"])
+	require.Equal(t, "a", cc.max["b"])
+
+	cc.add((&Row{}).WithValue("a", 0, 2).WithValue("b", 0, "aa"))
+	require.Equal(t, -1, cc.min["a"])
+	require.Equal(t, 2, cc.max["a"])
+	require.Equal(t, "a", cc.min["b"])
+	require.Equal(t, "aa", cc.max["b"])
+}

server/pkg/dal/runner.go

@@ -0,0 +1,19 @@
+package dal
+
+import (
+	"context"
+)
+
+type (
+	tester interface {
+		// Test returns a boolean output based on the expression, mostly used for filters
+		// @todo remove the error and rely on validator so make sure everything is valid
+		Test(ctx context.Context, params any) (bool, error)
+	}
+
+	evaluator interface {
+		// Eval returns some value based on the expression, mostly used for attribute eval
+		// @todo remove the error and rely on validator so make sure eve
+		Eval(ctx context.Context, params any) (any, error)
+	}
+)

server/pkg/dal/runner_gval.go

@@ -0,0 +1,472 @@
+package dal
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"strings"
+
+	"github.com/PaesslerAG/gval"
+	"github.com/cortezaproject/corteza/server/pkg/expr"
+	"github.com/cortezaproject/corteza/server/pkg/gvalfnc"
+	"github.com/cortezaproject/corteza/server/pkg/ql"
+)
+
+type (
+	runnerGval struct {
+		eval gval.Evaluable
+	}
+	converterGval struct {
+		parser *ql.Parser
+	}
+	exprHandlerGval struct {
+		Handler func(...string) string
+
+		// @todo temporarily added to support the current aggregate attribute type calculation
+		OutType        Type
+		OutTypeUnknown bool
+	}
+)
+
+var (
+	globalGvalConverter converterGval
+)
+
+var (
+	refToGvalExp = map[string]*exprHandlerGval{
+		// operators
+		"not": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("(! %s)", args[0])
+			},
+			OutType: &TypeBoolean{},
+		},
+
+		// - bool
+		"and": {
+			Handler: func(args ...string) string {
+				return strings.Join(args, " && ")
+			},
+			OutType: &TypeBoolean{},
+		},
+		"or": {
+			Handler: func(args ...string) string {
+				return strings.Join(args, " || ")
+			},
+			OutType: &TypeBoolean{},
+		},
+		"xor": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("(%s != %s)", args[0], args[1])
+			},
+			OutType: &TypeBoolean{},
+		},
+
+		// - comp.
+		"eq": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("%s == %s", args[0], args[1])
+			},
+			OutType: &TypeBoolean{},
+		},
+		"ne": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("%s != %s", args[0], args[1])
+			},
+			OutType: &TypeBoolean{},
+		},
+		"lt": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("%s < %s", args[0], args[1])
+			},
+			OutType: &TypeBoolean{},
+		},
+		"le": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("%s <= %s", args[0], args[1])
+			},
+			OutType: &TypeBoolean{},
+		},
+		"gt": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("%s > %s", args[0], args[1])
+			},
+			OutType: &TypeBoolean{},
+		},
+		"ge": {
+			// Handler: makeGenericCompHandler(">="),
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("%s >= %s", args[0], args[1])
+			},
+			OutType: &TypeBoolean{},
+		},
+
+		// - math
+		"add": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("%s + %s", args[0], args[1])
+			},
+			OutTypeUnknown: true,
+		},
+		"sub": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("%s - %s", args[0], args[1])
+			},
+			OutTypeUnknown: true,
+		},
+		"mult": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("%s * %s", args[0], args[1])
+			},
+			OutTypeUnknown: true,
+		},
+		"div": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("%s / %s", args[0], args[1])
+			},
+			OutTypeUnknown: true,
+		},
+
+		"in": {
+			Handler: func(args ...string) string {
+				// The arguments must be reversed!!
+				return fmt.Sprintf("has(%s, %s)", args[1], args[0])
+			},
+			OutTypeUnknown: true,
+		},
+		"nin": {
+			Handler: func(args ...string) string {
+				// The arguments must be reversed!!
+				return fmt.Sprintf("!has(%s, %s)", args[1], args[0])
+			},
+			OutTypeUnknown: true,
+		},
+
+		// - strings
+		"concat": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("concat(%s)", strings.Join(args, ", "))
+			},
+			OutType: &TypeText{},
+		},
+		"instr": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("instr(%s, %s)", args[0], args[1])
+			},
+			OutType: &TypeNumber{},
+		},
+
+		// @todo implement; the commented versions are not good enough
+		// "like": {
+		// 	Handler: func(args ...string) string {
+		// 		// @todo better regex construction
+		// 		nn := strings.Replace(strings.Trim(args[1], "\""), "%", ".*", -1)
+		// 		nn = strings.Replace(nn, "_", ".[1]", -1)
+
+		// 		return fmt.Sprintf("%s =~ ^%s$", args[0], nn)
+		// 	},
+		// },
+		// "nlike": {
+		// 	Handler: func(args ...string) string {
+		// 		// @todo better regex construction
+		// 		nn := strings.Replace(args[1], "%", ".*", -1)
+		// 		nn = strings.Replace(nn, "_", ".[1]", -1)
+
+		// 		return fmt.Sprintf("!(%s =~ ^%s$)", args[0], nn)
+		// 	},
+		// },
+
+		// "is": {
+		// 	Handler: func(args ...string) string {
+		// 		return fmt.Sprintf("%s == %s", args[0], args[1])
+		// 	}
+		// }
+
+		// - filtering
+		"now": {
+			Handler: func(args ...string) string {
+				return "now()"
+			},
+			OutType: &TypeTimestamp{},
+		},
+		"quarter": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("quarter(%s)", args[0])
+			},
+			OutType: &TypeNumber{},
+		},
+		"year": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("year(%s)", args[0])
+			},
+			OutType: &TypeNumber{},
+		},
+		"month": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("month(%s)", args[0])
+			},
+			OutType: &TypeNumber{},
+		},
+		"date": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("date(%s)", args[0])
+			},
+			OutType: &TypeDate{},
+		},
+		"day": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("day(%s)", args[0])
+			},
+			OutType: &TypeNumber{},
+		},
+		"week": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("week(%s)", args[0])
+			},
+			OutType: &TypeNumber{},
+		},
+		"date_format": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("strftime(%s, %s)", args[0], args[1])
+			},
+			OutType: &TypeText{},
+		},
+
+		// generic stuff
+		"null": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("isNil(%s)", args[0])
+			},
+			OutType: &TypeBoolean{},
+		},
+		"nnull": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("!isNil(%s)", args[0])
+			},
+			OutType: &TypeBoolean{},
+		},
+		"exists": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("!isNil(%s)", args[0])
+			},
+			OutType: &TypeBoolean{},
+		},
+
+		// - typecast
+		"float": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("float(%s)", args[0])
+			},
+			OutType: &TypeNumber{},
+		},
+		"int": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("int(%s)", args[0])
+			},
+			OutType: &TypeNumber{},
+		},
+		"string": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("string(%s)", args[0])
+			},
+			OutType: &TypeText{},
+		},
+
+		"group": {
+			Handler: func(args ...string) string {
+				return fmt.Sprintf("(%s)", args[0])
+			},
+			OutTypeUnknown: true,
+		},
+	}
+)
+
+// newConverterGval initializes a new gval exp. converter
+func newConverterGval(ii ...ql.IdentHandler) converterGval {
+	if globalGvalConverter.parser == nil {
+		globalGvalConverter = converterGval{
+			parser: newQlParser(append(
+				ii,
+				// This should always happen for gval expressions
+				func(i ql.Ident) (ql.Ident, error) {
+					i.Value = wrapNestedGvalIdent(i.Value)
+					return i, nil
+				},
+			)...),
+		}
+	}
+
+	return globalGvalConverter
+}
+
+// newRunnerGval initializes a new gval exp. runner from the provided expression
+func newRunnerGval(expr string) (out *runnerGval, err error) {
+	out = &runnerGval{}
+
+	c := newConverterGval()
+	n, err := c.Parse(expr)
+	if err != nil {
+		return nil, err
+	}
+
+	expr, err = c.Convert(n)
+	if err != nil {
+		return nil, err
+	}
+
+	out.eval, err = newGval(expr)
+	return
+}
+
+// newRunnerGvalParsed initializes a new gval exp. runner from the pre-parsed expression
+func newRunnerGvalParsed(n *ql.ASTNode) (out *runnerGval, err error) {
+	out = &runnerGval{}
+	c := newConverterGval()
+
+	expr, err := c.Convert(n)
+	if err != nil {
+		return
+	}
+
+	out.eval, err = newGval(expr)
+	return
+}
+
+// newGval initializes a new gval evaluatable for the provided expression
+//
+// The eval. includes a small subset of supported expr. functions which may be
+// used in the pipeline.
+//
+// @note the subset is limited to simplify the (eventual) offloading to the DB.
+//       At some point, more functions will be supported, and the ones which can't
+//       be offloaded will be performed in some exec. step.
+func newGval(e string) (gval.Evaluable, error) {
+	return gval.Full(
+		// Extra functions we'll need
+		// @note don't bring in all of the expr. pkg functions as we'll need to
+		//       support these on the DB as well
+		gval.Function("now", gvalfnc.Now),
+		gval.Function("quarter", gvalfnc.Quarter),
+		gval.Function("year", gvalfnc.Year),
+		gval.Function("month", gvalfnc.Month),
+		gval.Function("strftime", gvalfnc.StrfTime),
+		gval.Function("date", gvalfnc.Date),
+		gval.Function("day", gvalfnc.Day),
+		gval.Function("isNil", gvalfnc.IsNil),
+		gval.Function("float", gvalfnc.CastFloat),
+		gval.Function("int", gvalfnc.CastInt),
+		gval.Function("string", gvalfnc.CastString),
+		gval.Function("concat", gvalfnc.ConcatStrings),
+		gval.Function("has", arrHas),
+	).NewEvaluable(e)
+}
+
+func newQlParser(onIdent ...ql.IdentHandler) *ql.Parser {
+	pp := ql.NewParser()
+	pp.OnIdent = func(ident ql.Ident) (_ ql.Ident, err error) {
+		ident.Value = NormalizeAttrNames(ident.Value)
+
+		for _, oi := range onIdent {
+			ident, err = oi(ident)
+			if err != nil {
+				return
+			}
+		}
+
+		return ident, err
+	}
+
+	return pp
+}
+
+func (e *runnerGval) Test(ctx context.Context, rows any) (bool, error) {
+	o, err := e.eval(ctx, rows)
+	if err != nil {
+		return false, err
+	}
+
+	v, ok := o.(bool)
+	return ok && v, nil
+}
+
+func (e *runnerGval) Eval(ctx context.Context, rows any) (any, error) {
+	o, err := e.eval(ctx, rows)
+	if err != nil {
+		return nil, err
+	}
+
+	return o, nil
+}
+
+// Parse parses the QL expression into QL ASTNodes
+func (c converterGval) Parse(expr string) (*ql.ASTNode, error) {
+	return c.parser.Parse(expr)
+}
+
+// Convert converts the given nodes into a GVal expression
+// @todo add more validation so we can potentially omit exec. error checks
+func (c converterGval) Convert(n *ql.ASTNode) (expr string, err error) {
+	return c.convert(n)
+}
+
+func (c converterGval) convert(n *ql.ASTNode) (_ string, err error) {
+	switch {
+	case n.Symbol != "":
+		return n.Symbol, nil
+	case n.Value != nil:
+		// @todo I don't think this is quite ok, but it works for now so it'll do for now
+		if n.Value.V.Type() == "String" {
+			return fmt.Sprintf("\"%v\"", n.Value.V.Get()), nil
+		}
+		return fmt.Sprintf("%v", n.Value.V.Get()), nil
+	}
+
+	args := make([]string, len(n.Args))
+	for i, a := range n.Args {
+		args[i], err = c.convert(a)
+		if err != nil {
+			return
+		}
+	}
+
+	return c.refHandler(n, args...)
+}
+
+func (c converterGval) refHandler(n *ql.ASTNode, args ...string) (out string, err error) {
+	r := strings.ToLower(n.Ref)
+	if refToGvalExp[r] == nil {
+		return "", fmt.Errorf("unknown ref %q", n.Ref)
+	}
+	return refToGvalExp[r].Handler(args...), nil
+}
+
+// arrHas is a helper to assure the expr.Has always gets an array (or map) as
+// the first argument
+//
+// @todo this is needed because how the ValueGetters returns multi-value fields so
+//       an edge case where a field would have [a] but here, it would be presented
+//       as a.
+//       This would become obsolete when we address the actual issue.
+func arrHas(arr interface{}, vv ...interface{}) (b bool, err error) {
+	arr = expr.UntypedValue(arr)
+
+	if isMap(arr) {
+		return expr.Has(arr, vv...)
+	}
+
+	var (
+		c = reflect.ValueOf(arr)
+	)
+
+	switch c.Kind() {
+	case reflect.Slice:
+		return expr.Has(arr, vv...)
+
+	default:
+		return expr.Has([]any{arr}, vv...)
+	}
+}
+
+func isMap(v interface{}) bool {
+	return reflect.TypeOf(v).Kind() == reflect.Map
+}

server/pkg/dal/runner_gval_test.go

@@ -0,0 +1,181 @@
+package dal
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestRowEvaluatorTest(t *testing.T) {
+	tc := []struct {
+		name string
+		expr string
+		in   *Row
+		out  bool
+	}{{
+		name: "single row ok",
+		expr: `test == 10`,
+		in:   (&Row{}).WithValue("test", 0, 10),
+		out:  true,
+	}, {
+		name: "single row nok",
+		expr: `test == 11`,
+		in:   (&Row{}).WithValue("test", 0, 10),
+		out:  false,
+	}}
+
+	ctx := context.Background()
+
+	for _, c := range tc {
+		t.Run(c.name, func(t *testing.T) {
+			evl, err := newRunnerGval(c.expr)
+			require.NoError(t, err)
+			tt, err := evl.Test(ctx, c.in)
+			require.NoError(t, err)
+			require.Equal(t, c.out, tt)
+		})
+	}
+}
+
+func TestHandlers(t *testing.T) {
+	yr := time.Now().Year()
+	mnt := int(time.Now().Month())
+	qtr := mnt / 4
+	dy := time.Now().Day()
+
+	// @todo some of the tests were omitted because they would be too flaky
+	//       or can not yet pass (the gval nil handlers primarily)
+	tcc := []struct {
+		name string
+		expr string
+		in   simpleRow
+		out  any
+	}{
+		{
+			name: "not",
+			expr: `!true`,
+			in:   simpleRow{},
+			out:  false,
+		},
+		{
+			name: "and",
+			expr: `true && false`,
+			in:   simpleRow{},
+			out:  false,
+		},
+		{
+			name: "or",
+			expr: `false || true`,
+			in:   simpleRow{},
+			out:  true,
+		},
+		{
+			name: "xor",
+			expr: `'a' xor 'b'`,
+			in:   simpleRow{},
+			out:  true,
+		},
+		{
+			name: "eq",
+			expr: `1 == 2`,
+			in:   simpleRow{},
+			out:  false,
+		},
+		{
+			name: "ne",
+			expr: `3 != 2`,
+			in:   simpleRow{},
+			out:  true,
+		},
+		{
+			name: "lt",
+			expr: `1 < 2`,
+			in:   simpleRow{},
+			out:  true,
+		},
+		{
+			name: "le",
+			expr: `1 <= 2`,
+			in:   simpleRow{},
+			out:  true,
+		},
+		{
+			name: "gt",
+			expr: `2 > 1`,
+			in:   simpleRow{},
+			out:  true,
+		},
+		{
+			name: "ge",
+			expr: `1 >= 2`,
+			in:   simpleRow{},
+			out:  false,
+		},
+		{
+			name: "add",
+			expr: `1+3`,
+			in:   simpleRow{},
+			out:  float64(4),
+		},
+		{
+			name: "sub",
+			expr: `3-5`,
+			in:   simpleRow{},
+			out:  float64(-2),
+		},
+		{
+			name: "mult",
+			expr: `1*2`,
+			in:   simpleRow{},
+			out:  float64(2),
+		},
+		{
+			name: "div",
+			expr: `2/2`,
+			in:   simpleRow{},
+			out:  float64(1),
+		},
+		{
+			name: "concat",
+			expr: `concat('a', 'b')`,
+			in:   simpleRow{},
+			out:  "ab",
+		},
+		{
+			name: "quarter",
+			expr: `quarter(now())`,
+			in:   simpleRow{},
+			out:  qtr,
+		},
+		{
+			name: "year",
+			expr: `year(now())`,
+			in:   simpleRow{},
+			out:  yr,
+		},
+		{
+			name: "month",
+			expr: `month(now())`,
+			in:   simpleRow{},
+			out:  mnt,
+		},
+		{
+			name: "day",
+			expr: `day(now())`,
+			in:   simpleRow{},
+			out:  dy,
+		},
+	}
+
+	for _, tc := range tcc {
+		t.Run(tc.name, func(t *testing.T) {
+			evl, err := newRunnerGval(tc.expr)
+			require.NoError(t, err)
+			out, err := evl.Eval(context.Background(), tc.in)
+			require.NoError(t, err)
+			require.Equal(t, tc.out, out)
+		})
+	}
+}

server/pkg/dal/sensitivity_level.go

@@ -0,0 +1,109 @@
+package dal
+
+import "sort"
+
+type (
+	SensitivityLevel struct {
+		Handle string
+		ID     uint64
+		Level  int
+	}
+	SensitivityLevelSet []SensitivityLevel
+
+	sensitivityLevelIndex struct {
+		set SensitivityLevelSet
+
+		byHandle map[string]int
+		byID     map[uint64]int
+	}
+
+	SensitivityLevelUsage struct {
+		connections []map[string]any
+		modules     []map[string]any
+		fields      []map[string]any
+	}
+)
+
+func SensitivityLevelIndex(levels ...SensitivityLevel) *sensitivityLevelIndex {
+	out := &sensitivityLevelIndex{
+		set:      make(SensitivityLevelSet, len(levels)),
+		byHandle: make(map[string]int),
+		byID:     make(map[uint64]int),
+	}
+
+	for i, l := range levels {
+		out.set[i] = l
+		out.byHandle[l.Handle] = i
+		out.byID[l.ID] = i
+	}
+
+	return out
+}
+
+func (sli sensitivityLevelIndex) with(levels ...SensitivityLevel) *sensitivityLevelIndex {
+	slvls := append(sli.set, levels...)
+	sort.Sort(slvls)
+
+	return SensitivityLevelIndex(slvls...)
+}
+
+func (sli sensitivityLevelIndex) without(levels ...SensitivityLevel) *sensitivityLevelIndex {
+	nn := make(SensitivityLevelSet, 0, len(sli.set))
+
+	remIndex := SensitivityLevelIndex(levels...)
+
+	for _, existing := range sli.set {
+		if !remIndex.includes(existing.ID) {
+			nn = append(nn, existing)
+		}
+	}
+
+	sort.Sort(nn)
+
+	return SensitivityLevelIndex(nn...)
+}
+
+func (sli sensitivityLevelIndex) includes(l uint64) (ok bool) {
+	if l == 0 {
+		return true
+	}
+
+	if sli.byID == nil {
+		return false
+	}
+
+	_, ok = sli.byID[l]
+	return
+}
+
+func (sli sensitivityLevelIndex) isSubset(a, b uint64) (ok bool) {
+	// Edgecases
+	// If A is zero theneverything is possible
+	if a == 0 {
+		return true
+	}
+	// If B is zero, then A must also be zero
+	if b == 0 {
+		return a == 0
+	}
+
+	var lvlA, lvlB int
+
+	if lvlA, ok = sli.byID[a]; !ok {
+		return false
+	}
+
+	if lvlB, ok = sli.byID[b]; !ok {
+		return false
+	}
+
+	return lvlA <= lvlB
+}
+
+func (a SensitivityLevelSet) Len() int           { return len(a) }
+func (a SensitivityLevelSet) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a SensitivityLevelSet) Less(i, j int) bool { return a[i].Level < a[j].Level }
+
+func (u SensitivityLevelUsage) Empty() bool {
+	return len(u.connections)+len(u.fields)+len(u.modules) == 0
+}

server/pkg/dal/sensitivity_level_test.go

@@ -0,0 +1,95 @@
+package dal
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSensitivityLevelIncluded(t *testing.T) {
+	t.Run("does not include; empty", func(t *testing.T) {
+		ll := sensitivityLevelIndex{}
+
+		require.False(t, ll.includes(1))
+	})
+
+	t.Run("does not include; not in there", func(t *testing.T) {
+		ll := SensitivityLevelIndex(SensitivityLevel{
+			Handle: "a",
+			ID:     0,
+			Level:  1,
+		})
+
+		require.False(t, ll.includes(1))
+	})
+
+	t.Run("includes", func(t *testing.T) {
+		ll := SensitivityLevelIndex(SensitivityLevel{
+			Handle: "a",
+			ID:     1,
+			Level:  1,
+		})
+
+		require.True(t, ll.includes(1))
+	})
+}
+
+func TestSensitivityLevelSubset(t *testing.T) {
+	commonLevels := SensitivityLevelSet{{
+		ID:     1,
+		Handle: "a",
+		Level:  1,
+	}, {
+		ID:     2,
+		Handle: "b",
+		Level:  2,
+	}, {
+		ID:     3,
+		Handle: "c",
+		Level:  3,
+	}}
+
+	cases := []struct {
+		name string
+		a    uint64
+		b    uint64
+		out  bool
+	}{{
+		name: "lower is zero",
+		a:    0,
+		b:    1,
+		out:  true,
+	}, {
+		name: "both zero",
+		a:    0,
+		b:    0,
+		out:  true,
+	}, {
+		name: "upper zero, lower not",
+		a:    1,
+		b:    0,
+		out:  false,
+	}, {
+		name: "lower less",
+		a:    1,
+		b:    2,
+		out:  true,
+	}, {
+		name: "equal",
+		a:    2,
+		b:    2,
+		out:  true,
+	}, {
+		name: "lower higher",
+		a:    3,
+		b:    2,
+		out:  false,
+	}}
+
+	ix := SensitivityLevelIndex(commonLevels...)
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			require.Equal(t, c.out, ix.isSubset(c.a, c.b))
+		})
+	}
+}

server/pkg/dal/utils.go

@@ -0,0 +1,512 @@
+package dal
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/cortezaproject/corteza/server/pkg/ql"
+	"github.com/spf13/cast"
+)
+
+type (
+	// @todo consider reworking this
+	SimpleAttr struct {
+		Ident string
+		Expr  string
+		Src   string
+		Props MapProperties
+	}
+
+	// Row is a generic implementation for ValueGetter and ValueSetter
+	//
+	// Primarily used within DAL pipeline execution steps, but may also be used
+	// outside.
+	Row struct {
+		counters map[string]uint
+		values   valueSet
+
+		// Metadata to make it easier to work with
+		// @todo add when needed
+	}
+
+	valueSet map[string][]any
+)
+
+const (
+	attributeNestingSeparator     = "."
+	attributeNestingGvalSeparator = "___DLTR___"
+)
+
+func (sa SimpleAttr) Identifier() string              { return sa.Ident }
+func (sa SimpleAttr) Expression() (expression string) { return sa.Expr }
+func (sa SimpleAttr) Source() (ident string)          { return sa.Src }
+func (sa SimpleAttr) Properties() MapProperties       { return sa.Props }
+
+// WithValue is a simple helper to construct rows with populated values
+//
+// @note The main use is for tests so restrain from using it in code.
+func (r *Row) WithValue(name string, pos uint, v any) *Row {
+	err := r.SetValue(name, pos, v)
+	if err != nil {
+		panic(err)
+	}
+
+	return r
+}
+
+func (r Row) SelectGVal(ctx context.Context, wk string) (interface{}, error) {
+	// @todo this way of determining multivalue is not entirely ok but we can get away with it
+	//       since we have special functions to work with those.
+	//       We need to somehow inform this bit as to what fields are multi value.
+
+	k := unwrapNestedGvalIdent(wk)
+	cc := r.CountValues()[k]
+	if cc == 0 {
+		return nil, nil
+	}
+
+	if cc == 1 {
+		return r.GetValue(k, 0)
+	}
+
+	out := make([]any, cc)
+	for i := uint(0); i < cc; i++ {
+		out[i], _ = r.GetValue(k, i)
+	}
+	return out, nil
+}
+
+// Reset clears out the row so the same instance can be reused where possible
+//
+// Important: Reset only clears out the counters and does not re-init/clear out
+// the underlaying values. Don't directly iterate over the values, but use the
+// counters.
+func (r *Row) Reset() {
+	for k := range r.counters {
+		r.counters[k] = 0
+	}
+}
+
+func (r *Row) SetValue(name string, pos uint, v any) error {
+	if r.values == nil {
+		r.values = make(valueSet)
+	}
+	if r.counters == nil {
+		r.counters = make(map[string]uint)
+	}
+
+	// Make sure there is space for it
+	// @note benchmarking proves that the rest of the function introduces a lot of memory pressure.
+	//       Investigate options on reworking this/reducing allocations.
+	if int(pos)+1 > len(r.values[name]) {
+		r.values[name] = append(r.values[name], make([]any, (int(pos)+1)-len(r.values[name]))...)
+	}
+
+	r.values[name][pos] = v
+	if pos >= r.counters[name] {
+		r.counters[name]++
+	}
+
+	return nil
+}
+
+func (r *Row) CountValues() map[string]uint {
+	return r.counters
+}
+
+func (r *Row) GetValue(name string, pos uint) (any, error) {
+	if r.values == nil {
+		return nil, nil
+	}
+	if r.counters == nil {
+		return nil, nil
+	}
+	if pos >= r.counters[name] {
+		return nil, nil
+	}
+
+	return r.values[name][pos], nil
+}
+
+func (r *Row) String() string {
+	out := make([]string, 0, 20)
+
+	for k, cc := range r.counters {
+		for i := uint(0); i < cc; i++ {
+			v := r.values[k][i]
+			out = append(out, fmt.Sprintf("%s [%d] %v", k, i, v))
+		}
+	}
+
+	return strings.Join(out, " | ")
+}
+
+// compareGetters compares the two ValueGetters
+// -1: a is less then b
+// 0: a is equal to b
+// 1: a is greater then b
+//
+// Multi value rules:
+// - if a has less items then b, a is less then b (-1)
+// - if a has more items then b, a is more then b (1)
+// - if a and b have the same amount of items; if any of the corresponding values
+//   are different, that outcome is used as the result
+//
+// This function is used to satisfy sort's less function requirement.
+func compareGetters(a, b ValueGetter, ac, bc map[string]uint, attr string) int {
+	// If a has less values then b, then a is less then b
+	if ac[attr] < bc[attr] {
+		return -1
+	} else if ac[attr] > bc[attr] {
+		return 1
+	}
+
+	// If a and b have the same number of values, then we need to compare them
+	for i := uint(0); i < ac[attr]; i++ {
+		va, err := a.GetValue(attr, i)
+		if err != nil {
+			return 1
+		}
+
+		vb, err := b.GetValue(attr, i)
+		if err != nil {
+			return 1
+		}
+
+		// Continue the cmp. until we find two values that are different
+		cmp := compareValues(va, vb)
+		if cmp != 0 {
+			return cmp
+		}
+	}
+
+	// If any value is different from the other, the loop above would end; so
+	// here, we can safely say they are the same
+	return 0
+}
+
+// compareValues compares the two values
+// @todo identify what other types we should support
+// -1: a is less then b
+// 0: a is equal to b
+// 1: a is greater then b
+//
+// @note I considered using GVal here but it introduces more overhead then
+//       what I've conjured here.
+// @todo look into using generics or some wrapping types here
+func compareValues(va, vb any) int {
+	// simple/edge cases
+	if va == vb {
+		return 0
+	}
+	if va == nil {
+		return -1
+	}
+	if vb == nil {
+		return 1
+	}
+
+	// Compare based on type
+	switch ca := va.(type) {
+	case string:
+		cb, err := cast.ToStringE(vb)
+		if err != nil {
+			return -1
+		}
+		if ca < cb {
+			return -1
+		}
+		if ca > cb {
+			return 1
+		}
+
+	case int, int8, int16, int32, int64:
+		// this one can't error since we know it's an ok value
+		xa := cast.ToInt64(va)
+		cb, err := cast.ToInt64E(vb)
+		if err != nil {
+			return -1
+		}
+		if xa < cb {
+			return -1
+		}
+		if xa > cb {
+			return 1
+		}
+
+	case uint, uint8, uint16, uint32, uint64:
+		// this one can't error since we know it's an ok value
+		xa := cast.ToUint64(va)
+		cb, err := cast.ToUint64E(vb)
+		if err != nil {
+			return -1
+		}
+		if xa < cb {
+			return -1
+		}
+		if xa > cb {
+			return 1
+		}
+
+	case float32, float64:
+		// this one can't error since we know it's an ok value
+		xa := cast.ToFloat64(va)
+		cb, err := cast.ToFloat64E(vb)
+		if err != nil {
+			return -1
+		}
+		if xa < cb {
+			return -1
+		}
+		if xa > cb {
+			return 1
+		}
+
+	case time.Time, *time.Time:
+		// this one can't error since we know it's an ok value
+		xa := cast.ToTime(va)
+		cb, err := cast.ToTimeE(vb)
+		if err != nil {
+			return -1
+		}
+		if xa.Before(cb) {
+			return -1
+		}
+		if xa.After(cb) {
+			return 1
+		}
+	}
+
+	panic(fmt.Sprintf("unsupported type for values %v, %v", va, vb))
+}
+
+// constraintsToExpression converts the given constraints map to a ql parsable expression
+func constraintsToExpression(cc map[string][]any) string {
+	out := make([]string, 0, 10)
+
+	for k, vv := range cc {
+		part := make([]string, len(vv))
+		for i, v := range vv {
+			if vs, ok := v.(string); ok {
+				part[i] = fmt.Sprintf(`%s == '%s'`, k, vs)
+			} else {
+				part[i] = fmt.Sprintf("%s == %v", k, v)
+			}
+		}
+
+		pt := strings.Join(part, " || ")
+		if len(cc) > 1 {
+			out = append(out, fmt.Sprintf("(%s)", pt))
+		} else {
+			out = append(out, pt)
+		}
+
+	}
+
+	return strings.Join(out, " && ")
+}
+
+// stateConstraintsToExpression converts the given state expression to a ql parsable expression
+func stateConstraintsToExpression(cc map[string]filter.State) string {
+	out := make([]string, 0, 10)
+
+	for k, s := range cc {
+		// Inclusive one is omitted since the condition always evaluates
+		// as true (field == null || field != null => true)
+		switch s {
+		case filter.StateExcluded:
+			out = append(out, fmt.Sprintf("%s == null", k))
+
+		// Only these ones
+		case filter.StateExclusive:
+			out = append(out, fmt.Sprintf("%s != null", k))
+		}
+	}
+
+	return strings.Join(out, " && ")
+}
+
+// @todo see if the rest of the "conversion" functions should return a QL node
+//       like the cursor one does.
+func prepareGenericRowTester(f internalFilter) (_ tester, err error) {
+	var (
+		parts    = make([]string, 0, 5)
+		pcNode   *ql.ASTNode
+		exprNode *ql.ASTNode
+	)
+
+	{
+		// Convert the regular constraints
+		if cc := f.Constraints(); len(cc) != 0 {
+			parts = append(parts, constraintsToExpression(cc))
+		}
+
+		// Convert state constraints
+		// @todo check if the attributes in the state constraints are nullable.
+		if sc := f.StateConstraints(); len(sc) != 0 {
+			parts = append(parts, stateConstraintsToExpression(sc))
+		}
+
+		exprNode = f.ExpressionParsed()
+		if exprNode == nil {
+			expr := f.Expression()
+			if expr != "" {
+				parts = append(parts, f.Expression())
+			}
+		}
+
+		// Convert the paging cursor
+		if pc := f.Cursor(); pc != nil {
+			pcNode, err = f.Cursor().ToAST(nil, nil)
+			if err != nil {
+				return
+			}
+			pcNode.Traverse(func(a *ql.ASTNode) (bool, *ql.ASTNode, error) {
+				if a.Symbol != "" {
+					a.Symbol = wrapNestedGvalIdent(a.Symbol)
+				}
+				return true, a, nil
+			})
+		}
+	}
+
+	expr := strings.Join(parts, " && ")
+
+	// Everything is empty, not doing anything
+	if len(expr) == 0 && exprNode == nil && pcNode == nil {
+		return nil, nil
+	}
+
+	args := make([]*ql.ASTNode, 0, 5)
+
+	// Paging cursors
+	if pcNode != nil {
+		args = append(args, pcNode)
+	}
+
+	// Parsed filter expression
+	if exprNode != nil {
+		args = append(args, exprNode)
+	}
+
+	// Rest of the generated expression string
+	if len(expr) > 0 {
+		expr, err := newConverterGval().Parse(expr)
+		if err != nil {
+			return nil, err
+		}
+		args = append(args, expr)
+	}
+
+	return newRunnerGvalParsed(
+		&ql.ASTNode{
+			Ref:  "and",
+			Args: args,
+		},
+	)
+}
+
+// makeRowComparator returns a ValueGetter comparator for the given sort expr
+func makeRowComparator(ss ...*filter.SortExpr) func(a, b ValueGetter) bool {
+	return func(a, b ValueGetter) bool {
+		for _, s := range ss {
+			cmp := compareGetters(a, b, a.CountValues(), b.CountValues(), s.Column)
+
+			less, skip := evalCmpResult(cmp, s)
+			if !skip {
+				return less
+			}
+		}
+
+		return false
+	}
+}
+
+func evalCmpResult(cmp int, s *filter.SortExpr) (less, skip bool) {
+	if cmp != 0 {
+		if s.Descending {
+			return cmp > 0, false
+		}
+		return cmp < 0, false
+	}
+
+	return false, true
+}
+
+func indexAttrs(aa ...AttributeMapping) (out map[string]bool) {
+	out = make(map[string]bool, len(aa))
+	indexAttrsInto(out, aa...)
+	return
+}
+
+func indexAttrsInto(dst map[string]bool, aa ...AttributeMapping) {
+	for _, a := range aa {
+		dst[a.Identifier()] = true
+	}
+}
+
+// keysFromExpr returns all of the identifiers used in agg. group expressions
+//
+// The hasConstants return argument is true if any of the expressions returns a
+// constant value, such as year(now()) or 42
+func keysFromExpr(nn ...*ql.ASTNode) (out []string, hasConstants bool) {
+	out = make([]string, 0, (len(nn)+1)*2)
+	auxOut := make(map[string]bool, (len(nn)+1)*2)
+
+	for _, n := range nn {
+		symbols := n.CollectSymbols()
+		if len(symbols) == 0 {
+			hasConstants = true
+		}
+
+		for _, s := range symbols {
+			auxOut[s] = true
+		}
+	}
+
+	for k := range auxOut {
+		out = append(out, k)
+	}
+
+	return
+}
+
+// Assure sort validates that the filter's definition includes all of the primary
+// keys and that the paging cursor's sort is compatible
+func assureSort(f internalFilter, primaries []string) (out internalFilter, err error) {
+	out = f
+
+	// make sure all primary keys are in there
+	for _, p := range primaries {
+		if out.orderBy.Get(p) == nil {
+			out.orderBy = append(out.orderBy, &filter.SortExpr{
+				Column:     p,
+				Descending: out.orderBy.LastDescending(),
+			})
+		}
+	}
+
+	// No cursor, no problem
+	if f.cursor == nil {
+		return
+	}
+
+	// Make sure the cursor can handle this sort def
+	out.orderBy, err = out.cursor.Sort(out.orderBy)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func wrapNestedGvalIdent(ident string) string {
+	return strings.ReplaceAll(ident, attributeNestingSeparator, attributeNestingGvalSeparator)
+}
+
+func unwrapNestedGvalIdent(ident string) string {
+	return strings.ReplaceAll(ident, attributeNestingGvalSeparator, attributeNestingSeparator)
+}

server/pkg/dal/utils_bench_test.go

@@ -0,0 +1,42 @@
+package dal
+
+import (
+	"testing"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+)
+
+// goos: linux
+// goarch: amd64
+// pkg: github.com/cortezaproject/corteza/server/pkg/dal
+// cpu: Intel(R) Core(TM) i7-8750H CPU @ 2.20GHz
+// BenchmarkRowComparator-12    	  161826	      7015 ns/op	    5120 B/op	      40 allocs/op
+// PASS
+func BenchmarkRowComparator(b *testing.B) {
+	cmp := makeRowComparator(
+		&filter.SortExpr{Column: "a", Descending: false},
+		&filter.SortExpr{Column: "b", Descending: true},
+		&filter.SortExpr{Column: "c", Descending: true},
+		&filter.SortExpr{Column: "d", Descending: false},
+	)
+
+	r1 := simpleRow{"a": 10, "b": "aa", "c": 33, "d": 500}
+	r2 := simpleRow{"a": 50, "b": "aaaaa", "c": 31, "d": 10}
+	r3 := simpleRow{"a": 31, "b": "a", "c": 11, "d": 1000}
+	r4 := simpleRow{"a": 42, "b": "", "c": 0, "d": 300}
+	r5 := simpleRow{"a": 22, "b": "aaaaaaaaaa", "c": -25, "d": 21}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		cmp(r1, r2)
+		cmp(r1, r3)
+		cmp(r1, r4)
+		cmp(r1, r5)
+		cmp(r2, r3)
+		cmp(r2, r4)
+		cmp(r2, r5)
+		cmp(r3, r4)
+		cmp(r3, r5)
+		cmp(r4, r5)
+	}
+}

server/pkg/dal/utils_test.go

@@ -0,0 +1,466 @@
+package dal
+
+import (
+	"testing"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCompareValues(t *testing.T) {
+	n := time.Now()
+	m := n.Add(time.Second)
+
+	tcc := []struct {
+		name string
+		a    any
+		b    any
+		out  int
+	}{
+		{
+			name: "two ints; eq",
+			a:    10,
+			b:    10,
+			out:  0,
+		}, {
+			name: "two ints; lt",
+			a:    9,
+			b:    10,
+			out:  -1,
+		}, {
+			name: "two ints; gt",
+			a:    10,
+			b:    9,
+			out:  1,
+		}, {
+			name: "two ints; a nil",
+			a:    nil,
+			b:    10,
+			out:  -1,
+		}, {
+			name: "two ints; b nil",
+			a:    10,
+			b:    nil,
+			out:  1,
+		},
+
+		{
+			name: "two strings; eq",
+			a:    "aa",
+			b:    "aa",
+			out:  0,
+		}, {
+			name: "two strings; lt",
+			a:    "a",
+			b:    "aa",
+			out:  -1,
+		}, {
+			name: "two strings; gt",
+			a:    "aa",
+			b:    "a",
+			out:  1,
+		}, {
+			name: "two strings; a nil",
+			a:    nil,
+			b:    "aa",
+			out:  -1,
+		}, {
+			name: "two strings; b nil",
+			a:    "aa",
+			b:    nil,
+			out:  1,
+		},
+
+		{
+			name: "two uints; eq",
+			a:    uint(10),
+			b:    uint(10),
+			out:  0,
+		}, {
+			name: "two uints; lt",
+			a:    uint(9),
+			b:    uint(10),
+			out:  -1,
+		}, {
+			name: "two uints; gt",
+			a:    uint(10),
+			b:    uint(9),
+			out:  1,
+		}, {
+			name: "two uints; a nil",
+			a:    nil,
+			b:    uint(10),
+			out:  -1,
+		}, {
+			name: "two uints; b nil",
+			a:    uint(10),
+			b:    nil,
+			out:  1,
+		},
+
+		{
+			name: "two floats; eq",
+			a:    float64(10),
+			b:    float64(10),
+			out:  0,
+		}, {
+			name: "two floats; lt",
+			a:    float64(9),
+			b:    float64(10),
+			out:  -1,
+		}, {
+			name: "two floats; gt",
+			a:    float64(10),
+			b:    float64(9),
+			out:  1,
+		}, {
+			name: "two floats; a nil",
+			a:    nil,
+			b:    float64(10),
+			out:  -1,
+		}, {
+			name: "two floats; b nil",
+			a:    float64(10),
+			b:    nil,
+			out:  1,
+		},
+
+		{
+			name: "two times; eq",
+			a:    n,
+			b:    n,
+			out:  0,
+		}, {
+			name: "two times; lt",
+			a:    n,
+			b:    m,
+			out:  -1,
+		}, {
+			name: "two times; gt",
+			a:    m,
+			b:    n,
+			out:  1,
+		}, {
+			name: "two times; a nil",
+			a:    nil,
+			b:    n,
+			out:  -1,
+		}, {
+			name: "two times; b nil",
+			a:    n,
+			b:    nil,
+			out:  1,
+		},
+
+		{
+			name: "two nils",
+			a:    nil,
+			b:    nil,
+			out:  0,
+		},
+	}
+
+	for _, c := range tcc {
+		t.Run(c.name, func(t *testing.T) {
+			require.Equal(t, c.out, compareValues(c.a, c.b))
+		})
+	}
+}
+
+func TestCompareGetters(t *testing.T) {
+	tcc := []struct {
+		name string
+		a    ValueGetter
+		b    ValueGetter
+		attr string
+		out  int
+	}{{
+		name: "single eq",
+		a:    simpleRow{"a": 10},
+		b:    simpleRow{"a": 10},
+		attr: "a",
+		out:  0,
+	}, {
+		name: "single lt",
+		a:    simpleRow{"a": 9},
+		b:    simpleRow{"a": 10},
+		attr: "a",
+		out:  -1,
+	}, {
+		name: "single gt",
+		a:    simpleRow{"a": 10},
+		b:    simpleRow{"a": 9},
+		attr: "a",
+		out:  1,
+	},
+
+		{
+			name: "multi eq both empty",
+			a:    &Row{},
+			b:    &Row{},
+			attr: "a",
+			out:  0,
+		}, {
+			name: "multi eq same values",
+			a:    (&Row{}).WithValue("a", 0, 1),
+			b:    (&Row{}).WithValue("a", 0, 1),
+			attr: "a",
+			out:  0,
+		}, {
+			name: "multi lt a less items",
+			a:    (&Row{}),
+			b:    (&Row{}).WithValue("a", 0, 1),
+			attr: "a",
+			out:  -1,
+		}, {
+			name: "multi lt a item less",
+			a:    (&Row{}).WithValue("a", 0, 0),
+			b:    (&Row{}).WithValue("a", 0, 1),
+			attr: "a",
+			out:  -1,
+		}, {
+			name: "multi gt a more items",
+			a:    (&Row{}).WithValue("a", 0, 1),
+			b:    (&Row{}),
+			attr: "a",
+			out:  1,
+		}, {
+			name: "multi gt a item more",
+			a:    (&Row{}).WithValue("a", 0, 1),
+			b:    (&Row{}).WithValue("a", 0, 0),
+			attr: "a",
+			out:  1,
+		}}
+
+	for _, c := range tcc {
+		t.Run(c.name, func(t *testing.T) {
+			require.Equal(t, c.out, compareGetters(c.a, c.b, c.a.CountValues(), c.b.CountValues(), c.attr))
+		})
+	}
+}
+
+func TestConstraintsToExpression(t *testing.T) {
+	tcc := []struct {
+		name string
+		cc   map[string][]any
+		out  []string
+	}{{
+		name: "one constraint, one value",
+		cc: map[string][]any{
+			"k1": {"v1"},
+		},
+		out: []string{`k1 == 'v1'`},
+	}, {
+		name: "one constraint, multiple values",
+		cc: map[string][]any{
+			"k1": {"v1", 10, true},
+		},
+		out: []string{`k1 == 'v1' || k1 == 10 || k1 == true`},
+	}, {
+		name: "multiple constraints, multiple values",
+		cc: map[string][]any{
+			"k1": {"v1", 10, true},
+			"k2": {"v2", 42, false},
+		},
+		out: []string{
+			`(k1 == 'v1' || k1 == 10 || k1 == true)`,
+			`(k2 == 'v2' || k2 == 42 || k2 == false)`,
+		},
+	}}
+
+	for _, c := range tcc {
+		t.Run(c.name, func(t *testing.T) {
+			// Do it like so since map order is not defined and the test would be flaky
+			got := constraintsToExpression(c.cc)
+			for _, o := range c.out {
+				require.Contains(t, got, o)
+			}
+		})
+	}
+}
+
+func TestStateConstraintsToExpression(t *testing.T) {
+	tcc := []struct {
+		name string
+		cc   map[string]filter.State
+		out  []string
+	}{{
+		name: "exclude these ones",
+		cc: map[string]filter.State{
+			"k1": filter.StateExcluded,
+			"k2": filter.StateExcluded,
+		},
+		out: []string{`k1 == null`, `k2 == null`},
+	}, {
+		name: "only these ones",
+		cc: map[string]filter.State{
+			"k1": filter.StateExclusive,
+			"k2": filter.StateExclusive,
+		},
+		out: []string{`k1 != null`, `k2 != null`},
+	}, {
+		name: "both",
+		cc: map[string]filter.State{
+			"k1": filter.StateExclusive,
+			"k2": filter.StateExclusive,
+		},
+		out: []string{},
+	}, {
+		name: "mix and match",
+		cc: map[string]filter.State{
+			"k1": filter.StateExcluded,
+			"k2": filter.StateInclusive,
+			"k3": filter.StateExclusive,
+		},
+		out: []string{
+			`k1 == null`,
+			`k3 != null`,
+		},
+	}}
+
+	for _, c := range tcc {
+		t.Run(c.name, func(t *testing.T) {
+			// Do it like so since map order is not defined and the test would be flaky
+			got := stateConstraintsToExpression(c.cc)
+			for _, o := range c.out {
+				require.Contains(t, got, o)
+			}
+		})
+	}
+}
+
+func TestRowComparator(t *testing.T) {
+	// @todo add some more extreme cases
+	tcc := []struct {
+		name string
+		a    ValueGetter
+		b    ValueGetter
+		ss   filter.SortExprSet
+		less bool
+	}{
+		// Simple one col cases
+		{
+			name: "single column simple asc less",
+			a:    simpleRow{"a": 10},
+			b:    simpleRow{"a": 20},
+			ss:   filter.SortExprSet{{Column: "a", Descending: false}},
+			less: true,
+		},
+		{
+			name: "single column simple asc more",
+			a:    simpleRow{"a": 20},
+			b:    simpleRow{"a": 10},
+			ss:   filter.SortExprSet{{Column: "a", Descending: false}},
+			less: false,
+		},
+		{
+			name: "single column simple asc equal",
+			a:    simpleRow{"a": 10},
+			b:    simpleRow{"a": 10},
+			ss:   filter.SortExprSet{{Column: "a", Descending: false}},
+			less: false,
+		},
+		{
+			name: "single column simple desc less",
+			a:    simpleRow{"a": 20},
+			b:    simpleRow{"a": 10},
+			ss:   filter.SortExprSet{{Column: "a", Descending: true}},
+			less: true,
+		},
+		{
+			name: "single column simple desc more",
+			a:    simpleRow{"a": 10},
+			b:    simpleRow{"a": 20},
+			ss:   filter.SortExprSet{{Column: "a", Descending: true}},
+			less: false,
+		},
+		{
+			name: "single column simple desc equal",
+			a:    simpleRow{"a": 10},
+			b:    simpleRow{"a": 10},
+			ss:   filter.SortExprSet{{Column: "a", Descending: true}},
+			less: false,
+		},
+
+		// basic 2 col cases
+		{
+			name: "two column asc less first priority",
+			a:    simpleRow{"a": 10, "b": 100},
+			b:    simpleRow{"a": 20, "b": 1},
+			ss:   filter.SortExprSet{{Column: "a", Descending: false}, {Column: "b", Descending: false}},
+			less: true,
+		},
+		{
+			name: "two column asc less first equal",
+			a:    simpleRow{"a": 10, "b": 1},
+			b:    simpleRow{"a": 10, "b": 2},
+			ss:   filter.SortExprSet{{Column: "a", Descending: false}, {Column: "b", Descending: false}},
+			less: true,
+		},
+		{
+			name: "two column asc equal",
+			a:    simpleRow{"a": 10, "b": 1},
+			b:    simpleRow{"a": 10, "b": 1},
+			ss:   filter.SortExprSet{{Column: "a", Descending: false}, {Column: "b", Descending: false}},
+			less: false,
+		},
+		{
+			name: "two column desc less first priority",
+			a:    simpleRow{"a": 20, "b": 1},
+			b:    simpleRow{"a": 10, "b": 100},
+			ss:   filter.SortExprSet{{Column: "a", Descending: true}, {Column: "b", Descending: true}},
+			less: true,
+		},
+		{
+			name: "two column desc less first equal",
+			a:    simpleRow{"a": 10, "b": 2},
+			b:    simpleRow{"a": 10, "b": 1},
+			ss:   filter.SortExprSet{{Column: "a", Descending: true}, {Column: "b", Descending: true}},
+			less: true,
+		},
+		{
+			name: "two column desc equal",
+			a:    simpleRow{"a": 10, "b": 1},
+			b:    simpleRow{"a": 10, "b": 1},
+			ss:   filter.SortExprSet{{Column: "a", Descending: true}, {Column: "b", Descending: true}},
+			less: false,
+		},
+	}
+
+	for _, c := range tcc {
+		t.Run(c.name, func(t *testing.T) {
+
+			less := makeRowComparator(c.ss...)(c.a, c.b)
+			require.Equal(t, c.less, less)
+		})
+	}
+}
+
+func TestRowResetting(t *testing.T) {
+	r := &Row{}
+
+	gv := func(ident string, pos uint) any {
+		v, err := r.GetValue(ident, pos)
+		require.NoError(t, err)
+		return v
+	}
+
+	r.SetValue("a", 0, 1)
+	r.SetValue("a", 0, 2)
+	r.SetValue("a", 1, 1)
+
+	require.Equal(t, 2, gv("a", 0))
+	require.Equal(t, 1, gv("a", 1))
+	require.Equal(t, uint(2), r.counters["a"])
+
+	r.Reset()
+	require.Equal(t, uint(0), r.counters["a"])
+
+	r.SetValue("a", 0, 3)
+	r.SetValue("b", 0, 4)
+	require.Equal(t, 3, gv("a", 0))
+	require.Equal(t, uint(1), r.counters["a"])
+	require.Equal(t, 4, gv("b", 0))
+	require.Equal(t, uint(1), r.counters["b"])
+}