snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/system/reporting/frame.go

@@ -0,0 +1,139 @@
+package reporting
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/cortezaproject/corteza/server/system/types"
+)
+
+type (
+	Frame struct {
+		Name   string `json:"name"`
+		Source string `json:"source"`
+		Ref    string `json:"ref,omitempty"`
+
+		// RefValue is the common value between the two datasources
+		RefValue string `json:"refValue,omitempty"`
+		// RelColumn is what column in the local ds was used
+		RelColumn string `json:"relColumn,omitempty"`
+		// RelSource is the ds that this frame is related to
+		RelSource string `json:"relSource,omitempty"`
+
+		Columns FrameColumnSet `json:"columns"`
+		Rows    []FrameRow     `json:"rows"`
+
+		Paging *filter.Paging          `json:"paging"`
+		Sort   filter.SortExprSet      `json:"sort"`
+		Filter *types.ReportFilterExpr `json:"filter"`
+	}
+
+	FrameDescription struct {
+		Source  string         `json:"source"`
+		Ref     string         `json:"ref,omitempty"`
+		Columns FrameColumnSet `json:"columns"`
+	}
+
+	FrameRow []string
+
+	FrameColumnSet []FrameColumn
+	FrameColumn    struct {
+		Name    string `json:"name"`
+		Label   string `json:"label"`
+		Kind    string `json:"kind"`
+		Primary bool   `json:"primary"`
+		Unique  bool   `json:"unique"`
+		System  bool   `json:"system"`
+
+		Multivalue          bool   `json:"multivalue"`
+		MultivalueDelimiter string `json:"multivalueDelimiter"`
+	}
+
+	FrameDefinitionSet []*FrameDefinition
+	FrameDefinition    struct {
+		Name    string         `json:"name"`
+		Source  string         `json:"source"`
+		Ref     string         `json:"ref"`
+		Columns FrameColumnSet `json:"columns"`
+
+		Filter *types.ReportFilterExpr `json:"filter"`
+		Paging *filter.Paging          `json:"paging"`
+		Sort   filter.SortExprSet      `json:"sort"`
+	}
+)
+
+// @todo nicer formatting and alignment
+func (f Frame) String() string {
+	out := fmt.Sprintf("n: %10s; src: %10s\n", f.Name, f.Source)
+
+	if f.Ref != "" {
+		out += fmt.Sprintf("ref: %10s; col: %10s\n; key: %10s\n", f.Ref, f.RelColumn, f.RefValue)
+	}
+
+	if f.RelSource != "" {
+		out += fmt.Sprintf("Rel source: %10s;\n", f.RelSource)
+	}
+
+	for _, c := range f.Columns {
+		out += fmt.Sprintf("%s<%s>, ", c.Name, c.Kind)
+	}
+	out = strings.TrimRight(out, " ,")
+	out += "\n"
+
+	for i, r := range f.Rows {
+		out += fmt.Sprintf("%d| %s", i+1, strings.Join(r, ", "))
+		out += "\n"
+	}
+
+	if f.Paging != nil {
+		out += "\n"
+		out += fmt.Sprintf("< %s; =%s; > %s", f.Paging.PrevPage.String(), f.Paging.PageCursor.String(), f.Paging.NextPage.String())
+	}
+
+	if len(f.Sort) > 0 {
+		out += "\n"
+		out += f.Sort.String()
+	}
+
+	out += "\n"
+	out += fmt.Sprintf("ix %d; len %d", 0, len(f.Rows))
+
+	return out
+}
+
+func (cc FrameColumnSet) String() string {
+	out := ""
+	for _, c := range cc {
+		out += fmt.Sprintf("%s<%s>, ", c.Name, c.Kind)
+	}
+	return strings.TrimRight(out, " ,")
+}
+
+// OmitSys returns the columns that are not system defined
+func (cc FrameColumnSet) OmitSys() FrameColumnSet {
+	out := make(FrameColumnSet, 0, len(cc))
+	for _, c := range cc {
+		if !c.System {
+			out = append(out, c)
+		}
+	}
+
+	return out
+}
+
+func (r FrameRow) String() string {
+	return strings.Join(r, ", ")
+}
+
+// FilterBySource returns a set of definitions for the requested identifier
+func (dd FrameDefinitionSet) FilterBySource(ident string) FrameDefinitionSet {
+	out := make(FrameDefinitionSet, 0, len(dd))
+	for _, d := range dd {
+		if d.Source == ident {
+			out = append(out, d)
+		}
+	}
+
+	return out
+}

server/system/reporting/frame_builder.go

@@ -0,0 +1,141 @@
+package reporting
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/modern-go/reflect2"
+)
+
+type (
+	// reportFrameBuilder assist in frame construction from iterators
+	//
+	// Primarily simplifies mapping the correct iterator attributes to correct
+	// frame row columns and having them encoded properly.
+	reportFrameBuilder struct {
+		def    *FrameDefinition
+		frame  *Frame
+		ownCol string
+
+		attrMapping map[string]int
+		attrMvDel   map[string]string
+	}
+)
+
+// newReportFrameBuilder initializes a new report frame builder
+func newReportFrameBuilder(def *FrameDefinition) *reportFrameBuilder {
+	// Index cols for easier lookups
+	attrMap := make(map[string]int)
+	mvDelMap := make(map[string]string)
+	for i, c := range def.Columns {
+		attrMap[c.Name] = i
+
+		if c.Multivalue {
+			mvDelMap[c.Name] = c.MultivalueDelimiter
+			if mvDelMap[c.Name] == "" {
+				mvDelMap[c.Name] = "\n"
+			}
+		}
+	}
+
+	out := &reportFrameBuilder{
+		def:         def,
+		attrMapping: attrMap,
+	}
+
+	// Init output frame
+	out.freshFrame()
+	return out
+}
+
+// linked includes additional metadata required by the link step
+func (b *reportFrameBuilder) linked(ownCol, relCol, relSrc string) {
+	b.ownCol = ownCol
+	b.frame.RelColumn = relCol
+	b.frame.RelSource = relSrc
+}
+
+// addRow adds a new dal.Row to the frame
+func (b *reportFrameBuilder) addRow(r *dal.Row) {
+	rrow := make(FrameRow, len(b.def.Columns))
+
+	for k, cc := range r.CountValues() {
+		ix, ok := b.attrMapping[k]
+		if !ok {
+			continue
+		}
+
+		auxRow := make(FrameRow, cc)
+		for i := uint(0); i < cc; i++ {
+			val, _ := r.GetValue(k, i)
+			auxRow[i] = b.stringifyVal(k, val)
+		}
+		rrow[ix] = b.joinMultiVal(k, auxRow)
+	}
+
+	b.frame.Rows = append(b.frame.Rows, rrow)
+
+	if b.ownCol != "" {
+		v, _ := r.GetValue(b.ownCol, 0)
+		b.frame.RefValue = b.stringifyVal(b.ownCol, v)
+	}
+}
+
+// done returns the constructed frame and prepares a new frame with the same
+// metadata as the original one
+func (b *reportFrameBuilder) done() *Frame {
+	out := b.frame
+	b.freshFrame()
+
+	return out
+}
+
+func (b *reportFrameBuilder) stringifyVal(col string, val any) string {
+	// Edgecase for when values are nil since the stringer uses <nil> for those
+	if reflect2.IsNil(val) {
+		return ""
+	}
+
+	// @todo nicer formatting and such? V1 didn't do much different
+	return fmt.Sprintf("%v", val)
+}
+
+func (b *reportFrameBuilder) joinMultiVal(col string, vals []string) string {
+	// @todo b.attrMvDel[col] is never actually set so it should be ALONG with this default
+	del, ok := b.attrMvDel[col]
+	if !ok {
+		del = "\n"
+	}
+	return strings.Join(vals, del)
+}
+
+func (b *reportFrameBuilder) freshFrame() {
+	// reuse the old frame metadata and clears out the rows
+	if b.frame != nil {
+		aux := *b.frame
+		b.frame = &aux
+		b.frame.Rows = []FrameRow{}
+		return
+	}
+
+	b.frame = &Frame{
+		Name:   b.def.Name,
+		Source: b.def.Source,
+		Ref:    b.def.Ref,
+
+		Rows: []FrameRow{},
+
+		Columns: b.def.Columns,
+		Sort:    b.def.Sort,
+		Filter:  b.def.Filter,
+	}
+
+	if b.def.Paging != nil {
+		b.frame.Paging = &filter.Paging{
+			Limit:      b.def.Paging.Limit,
+			PageCursor: b.def.Paging.PageCursor,
+		}
+	}
+}

server/system/reporting/reporting.go

@@ -0,0 +1,741 @@
+// Package reporting implements utilities for working with system reports and
+// DAL pipelines
+//
+// The package exists primarily for making the system service codebase smaller
+// and prepare for reusability with compose record reports.
+package reporting
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/cortezaproject/corteza/server/system/types"
+	"github.com/spf13/cast"
+)
+
+type (
+	// run is a wrapper around the pipeline and corresponding
+	// output frame definitions
+	run struct {
+		Pipeline dal.Pipeline
+		Defs     FrameDefinitionSet
+	}
+
+	modelFinder interface {
+		FindModel(dal.ModelRef) *dal.Model
+	}
+
+	// @todo not a fan of this anymore; consider reworking pkg/dal/service to
+	//       provide a more sequential approach we can use.
+	dryRunner interface {
+		modelFinder
+		Dryrun(context.Context, dal.Pipeline) error
+	}
+)
+
+// Runs create a set of report runs based on step and frame definitions
+func Runs(pr modelFinder, steps types.ReportStepSet, defs FrameDefinitionSet) (out []run, err error) {
+	// Prepare runs based on the provided definitions
+	//
+	// - If consecutive definitions point to the same source with the same name
+	//   consider them to fall under the same workload (the link step)
+	// - else, one def per workload
+	auxDefs := make(FrameDefinitionSet, 0)
+	var aux run
+	for i, def := range defs {
+		if i == 0 {
+			auxDefs = append(auxDefs, def)
+			continue
+		}
+
+		// Definitions fall together
+		if def.Name == defs[i-1].Name && def.Source == defs[i-1].Source {
+			auxDefs = append(auxDefs, def)
+			continue
+		}
+
+		// Make run for the previous definition (exclude current!!)
+		aux, err = makeRun(pr, steps, auxDefs)
+		if err != nil {
+			return
+		}
+		out = append(out, aux)
+
+		// Prepare next definition batch including the current one
+		auxDefs = make(FrameDefinitionSet, 0)
+		auxDefs = append(auxDefs, def)
+	}
+
+	// Handle the ones (potentially) not covered by the above loop
+	if len(auxDefs) > 0 {
+		aux, err = makeRun(pr, steps, auxDefs)
+		if err != nil {
+			return
+		}
+		out = append(out, aux)
+	}
+
+	return
+}
+
+// Frames returns a set of Frame for the given workload & iterator combo
+func Frames(ctx context.Context, iter dal.Iterator, r run) (ff []*Frame, err error) {
+	// Preprocessing on the workload's frame definitions; assure all
+	// columns/metdata are there to avoid nonesense later down the line
+	updateDefinitionColumns(r)
+
+	// @todo perhaps need to change; for now only this scenario introduces multiple
+	//       frame defs per workload
+	if len(r.Defs) > 1 {
+		return stepLinkFrames(ctx, iter, r)
+	}
+
+	return stepFrames(ctx, iter, r)
+}
+
+// Describe returns a set of frame descriptions based on the given pipeline
+func Describe(ctx context.Context, rr dryRunner, ss types.ReportStepSet, sources []string) (out []FrameDescription, err error) {
+	// Make a run for the whole thing
+	pp, err := makePipeline(rr, ss, nil)
+	if err != nil {
+		return
+	}
+
+	var aux []FrameDescription
+	for _, src := range sources {
+		// Use the requested source as root
+		sub := pp.Slice(src)
+		err = rr.Dryrun(ctx, sub)
+		if err != nil {
+			return
+		}
+
+		s := sub[0]
+		aux, err = describePipeline(s, src)
+		if err != nil {
+			return
+		}
+		out = append(out, aux...)
+	}
+
+	return
+}
+
+// stepLinkFrames is dedicated for the link step due to it's unique output
+func stepLinkFrames(ctx context.Context, iter dal.Iterator, r run) (ff []*Frame, err error) {
+	defs := r.Defs
+	// @note this will only be called for the link step so it can freely panic if violated
+	defLink := r.Pipeline[0].(*dal.Link)
+
+	// Unpack frame definitions for the link
+	defLeft, defRight := unpackLinkDefs(defs, r.Pipeline)
+
+	// Init vars to keep track of the progress
+	// @note true is left, false is right
+	counters := make(map[bool]uint)
+
+	builders := make(map[bool]*reportFrameBuilder)
+	builders[true] = newReportFrameBuilder(defLeft)
+	builders[false] = newReportFrameBuilder(defRight)
+	builders[false].linked(defLink.On.Right, defLink.On.Left, defLink.RelLeft)
+
+	limits := make(map[bool]uint)
+	if defLeft.Paging != nil {
+		limits[true] = defLeft.Paging.Limit
+	}
+	if defRight.Paging != nil {
+		limits[false] = defRight.Paging.Limit
+	}
+
+	// Helper to determine if we need a next cursor
+	nextCursor := false
+
+	// Helpers for reading iterators
+	var (
+		ref    string
+		row    = &dal.Row{}
+		doingF = false
+	)
+
+	for iter.Next(ctx) {
+		if limits[true] > 0 && counters[true] >= limits[true] {
+			nextCursor = true
+			break
+		}
+		row.Reset()
+
+		_ = iter.Scan(row)
+
+		// Determine ref and which vars to use
+		aux, _ := row.GetValue(dal.LinkRefIdent, 0)
+		ref = cast.ToString(aux)
+		if ref == "" {
+			ref = defLeft.Ref
+		}
+		left := ref == defLeft.Ref
+
+		// When needed, flush the finished frames to the output
+		if left && doingF {
+			ff = append(ff, builders[false].done())
+			doingF = false
+		} else if !left {
+			doingF = true
+		}
+
+		builders[left].addRow(row)
+		counters[left]++
+	}
+	if err = iter.Err(); err != nil {
+		return
+	}
+
+	// If the loop ended before the limit cut it off, we need to finish the
+	// last right frame as it wasn't yet in the above loop
+	if !nextCursor {
+		if doingF {
+			ff = append(ff, builders[false].done())
+		}
+	}
+
+	// Apply paging cursor to the left frame
+	// @todo consider applying them to the right as well, for now, no
+	if nextCursor {
+		if builders[true].frame.Paging == nil {
+			builders[true].frame.Paging = &filter.Paging{}
+		}
+		builders[true].frame.Paging.NextPage, err = iter.ForwardCursor(row)
+		if err != nil {
+			return
+		}
+	}
+
+	// Complete the output with the left frame
+	// @note the left frame goes to the start and the right frames are in the same order
+	//       as the related rows from the left frame.
+	return append([]*Frame{builders[true].done()}, ff...), nil
+}
+
+// unpackLinkDefs returns the left and the right frame definition disregarding
+// the order of the definitions in the input
+func unpackLinkDefs(defs FrameDefinitionSet, pp dal.Pipeline) (left, right *FrameDefinition) {
+	r := pp[0]
+
+	l := r.(*dal.Link)
+
+	find := func(defs FrameDefinitionSet, ref string) *FrameDefinition {
+		for _, def := range defs {
+			if def.Ref == ref {
+				return def
+			}
+		}
+		return nil
+	}
+
+	return find(defs, l.RelLeft), find(defs, l.RelRight)
+}
+
+// stepFrames is a generic iter to Frame converter
+func stepFrames(ctx context.Context, iter dal.Iterator, r run) (ff []*Frame, err error) {
+	defs := r.Defs
+
+	// @note only the link step takes multiple defs and that one is not covered
+	//       by this function
+	if len(defs) != 1 {
+		panic(fmt.Sprintf("impossible state: expecting one frame definition, got %d", len(defs)))
+	}
+	def := defs[0]
+
+	// Init vars to keep track of the progress
+	limit := uint(0)
+	counter := uint(0)
+	builder := newReportFrameBuilder(def)
+	if def.Paging != nil {
+		limit = def.Paging.Limit
+	}
+
+	// Helper to determine if we need a next cursor
+	nextCursor := false
+
+	// Helpers for reading iterators
+	row := &dal.Row{}
+
+	for iter.Next(ctx) {
+		if limit > 0 && counter >= limit {
+			nextCursor = true
+			break
+		}
+		row.Reset()
+
+		_ = iter.Scan(row)
+		builder.addRow(row)
+		counter++
+	}
+	if err = iter.Err(); err != nil {
+		return
+	}
+
+	// Apply paging cursor to the frame
+	if nextCursor {
+		if builder.frame.Paging == nil {
+			builder.frame.Paging = &filter.Paging{}
+		}
+		builder.frame.Paging.NextPage, err = iter.ForwardCursor(row)
+		if err != nil {
+			return
+		}
+	}
+
+	return append(ff, builder.done()), nil
+}
+
+func describePipeline(s dal.PipelineStep, src string) (out []FrameDescription, err error) {
+	aa := s.Attributes()
+
+	out = make([]FrameDescription, len(aa))
+	for i, a := range aa {
+		out[i].Source = src
+		out[i].Columns = mappingToFrameCols(a)
+	}
+
+	// @note this case is only possible for the link step; expand when/if needed
+	if len(out) == 2 {
+		l := s.(*dal.Link)
+		out[0].Ref = l.RelLeft
+		out[1].Ref = l.RelRight
+	}
+
+	return
+}
+
+func makeRun(pr modelFinder, ss types.ReportStepSet, defs FrameDefinitionSet) (out run, err error) {
+	var pp dal.Pipeline
+	pp, err = makePipeline(pr, ss, defs)
+	if err != nil {
+		return
+	}
+
+	out.Defs = defs
+	out.Pipeline = pp.Slice(defs[0].Source)
+	return
+}
+
+// makePipeline converts the given report steps into the DAL pipeline
+func makePipeline(mf modelFinder, ss types.ReportStepSet, defs FrameDefinitionSet) (pp dal.Pipeline, err error) {
+	for _, step := range ss {
+		switch {
+		case step.Load != nil:
+			aux, err := convStepLoad(mf, *step.Load, defs.FilterBySource(step.Load.Name))
+			if err != nil {
+				return nil, err
+			}
+			pp = append(pp, aux)
+
+		case step.Aggregate != nil:
+			aux, err := convStepAggregate(*step.Aggregate, defs.FilterBySource(step.Aggregate.Name))
+			if err != nil {
+				return nil, err
+			}
+			pp = append(pp, aux)
+
+		case step.Join != nil:
+			aux, err := convStepJoin(*step.Join, defs.FilterBySource(step.Join.Name))
+			if err != nil {
+				return nil, err
+			}
+			pp = append(pp, aux)
+
+		case step.Link != nil:
+			aux, err := convStepLink(*step.Link, defs.FilterBySource(step.Link.Name))
+			if err != nil {
+				return nil, err
+			}
+			pp = append(pp, aux)
+
+		default:
+			// this should never happen
+			panic(fmt.Errorf("unknown step type: %v", step.Kind))
+		}
+	}
+
+	return pp, pp.LinkSteps()
+}
+
+// mappingToFrameCols converts pipeline AttributeMapping to FrameColumnSet
+func mappingToFrameCols(mm []dal.AttributeMapping) FrameColumnSet {
+	out := make(FrameColumnSet, 0, len(mm))
+
+	for _, m := range mm {
+		out = append(out, mappingToFrameCol(m))
+	}
+
+	return out
+}
+
+// @note current implementation a bit _rushed_ since I'll probably rethink
+//       how the pipeline handles attributes -- will revisit then.
+func mappingToFrameCol(m dal.AttributeMapping) FrameColumn {
+	p := m.Properties()
+
+	const (
+		// Coppied around to reduce imports
+		emailLength = 254
+		urlLength   = 2048
+
+		attachmentResType = "corteza::system:attachment"
+		userResType       = "corteza::system:user"
+		moduleResType     = "corteza::compose:module"
+	)
+
+	l := m.Properties().Label
+	if l == "" {
+		l = m.Identifier()
+	}
+	out := FrameColumn{
+		Name:  m.Identifier(),
+		Label: l,
+		Kind:  "String",
+
+		Primary:             p.IsPrimary,
+		System:              p.IsSystem,
+		Multivalue:          p.IsMultivalue,
+		MultivalueDelimiter: p.MultivalueDelimiter,
+	}
+	if out.MultivalueDelimiter == "" {
+		out.MultivalueDelimiter = "\n"
+	}
+
+	switch t := p.Type.(type) {
+	case *dal.TypeBoolean:
+		out.Kind = "Boolean"
+	case *dal.TypeDate, *dal.TypeTime, *dal.TypeTimestamp:
+		out.Kind = "DateTime"
+
+	case *dal.TypeNumber:
+		out.Kind = "Number"
+
+	case *dal.TypeEnum:
+		out.Kind = "Select"
+
+	case *dal.TypeText:
+		// @note temporary solution; we should push some meta along with it
+		if t.Length == emailLength {
+			out.Kind = "Email"
+		} else if t.Length == urlLength {
+			out.Kind = "URL"
+		} else {
+			out.Kind = "String"
+		}
+
+	case *dal.TypeRef:
+		switch t.RefModel.ResourceType {
+		case moduleResType:
+			out.Kind = "Record"
+		case userResType:
+			out.Kind = "User"
+		case attachmentResType:
+			out.Kind = "File"
+		}
+	}
+
+	return out
+}
+
+// Report step -> DAL step conversion
+
+// convStepLoad converts ReportStepLoad to dal.Datasource
+func convStepLoad(pr modelFinder, step types.ReportStepLoad, defs FrameDefinitionSet) (out *dal.Datasource, err error) {
+	// Validation
+	if len(defs) > 1 {
+		err = fmt.Errorf("cannot convert load step: expecting at most one definition, got %d", len(defs))
+		return
+	}
+
+	// Get additional filtering
+	var extf filter.Filter
+	if len(defs) == 1 {
+		extf = filterFromDef(defs[0])
+	}
+
+	// Prepare model ref
+	mfr, err := makeModelRef(step)
+	if err != nil {
+		return
+	}
+
+	// @todo refactor this out after we support other resources with potentially missing soft delete fields
+	f, err := dal.FilterFromExpr(step.Filter.Node()).
+		MergeFilters(filter.Generic(filter.WithStateConstraint("deletedAt", filter.StateExcluded)))
+	if err != nil {
+		return
+	}
+
+	f, err = f.MergeFilters(extf)
+	if err != nil {
+		return
+	}
+
+	// Make pipeline step
+	return &dal.Datasource{
+		Ident:         step.Name,
+		Filter:        f,
+		ModelRef:      mfr,
+		OutAttributes: modelAttributes(pr, step, mfr),
+	}, nil
+}
+
+// convStepAggregate converts ReportStepAggregate to dal.Aggregate
+func convStepAggregate(step types.ReportStepAggregate, defs FrameDefinitionSet) (out *dal.Aggregate, err error) {
+	// Validation
+	if len(defs) > 1 {
+		err = fmt.Errorf("cannot convert aggregate step: expecting at most one definition, got %d", len(defs))
+		return
+	}
+
+	// Get additional filtering
+	var extf filter.Filter
+	if len(defs) == 1 {
+		extf = filterFromDef(defs[0])
+	}
+
+	f, err := dal.FilterFromExpr(step.Filter.Node()).
+		MergeFilters(extf)
+	if err != nil {
+		return
+	}
+
+	ggs := make([]dal.AggregateAttr, 0, len(step.Keys))
+	for _, c := range step.Keys {
+		ggs = append(ggs, dal.AggregateAttr{
+			Key:        true,
+			Identifier: c.Name,
+			Label:      c.Label,
+			Expression: c.Def.Node(),
+		})
+	}
+
+	vvs := make([]dal.AggregateAttr, 0, len(step.Columns))
+	for _, c := range step.Columns {
+		vvs = append(vvs, dal.AggregateAttr{
+			Identifier: c.Name,
+			Label:      c.Label,
+			Expression: c.Def.Node(),
+		})
+	}
+
+	// Make pipeline step
+	out = &dal.Aggregate{
+		Ident:     step.Name,
+		RelSource: step.Source,
+		Filter:    f,
+
+		Group:         ggs,
+		OutAttributes: vvs,
+	}
+	return
+}
+
+// convStepJoin converts ReportStepJoin to dal.Join
+func convStepJoin(step types.ReportStepJoin, defs FrameDefinitionSet) (out *dal.Join, err error) {
+	// Validation
+	if len(defs) > 1 {
+		err = fmt.Errorf("cannot convert join step: expecting at most one definition, got %d", len(defs))
+		return
+	}
+
+	// Get additional filtering
+	var extf filter.Filter
+	if len(defs) == 1 {
+		extf = filterFromDef(defs[0])
+	}
+
+	f, err := dal.FilterFromExpr(step.Filter.Node()).
+		MergeFilters(extf)
+	if err != nil {
+		return
+	}
+
+	// Make pipeline step
+	out = &dal.Join{
+		Ident:    step.Name,
+		RelLeft:  step.LocalSource,
+		RelRight: step.ForeignSource,
+
+		Filter: f,
+
+		On: dal.JoinPredicate{
+			Left:  step.LocalColumn,
+			Right: step.ForeignColumn,
+		},
+	}
+	return
+}
+
+// convStepLink converts ReportStepLink to dal.Link
+func convStepLink(step types.ReportStepLink, defs FrameDefinitionSet) (out *dal.Link, err error) {
+	// Validation
+	if len(defs) > 2 {
+		err = fmt.Errorf("cannot convert join step: expecting at most two definitions, got %d", len(defs))
+		return
+	}
+
+	// @todo additional filtering; will need to split the dal.Link filter into
+	//       left and right for more control and clarity
+
+	var extf filter.Filter
+	if len(defs) > 0 {
+		extf = filterFromDef(defs[0])
+	}
+
+	// Get additional filtering
+	f, err := dal.FilterFromExpr(step.Filter.Node()).
+		MergeFilters(extf)
+	if err != nil {
+		return
+	}
+
+	// @todo temporary constraint
+	if len(defs) == 2 {
+		d := defs[1]
+		if d.Filter != nil && d.Filter.Node() != nil || len(d.Sort) > 0 {
+			return nil, fmt.Errorf("temporary constraint: cannot apply sorting/filtering to foreign frame definition")
+		}
+	}
+
+	// Make pipeline step
+	out = &dal.Link{
+		Ident:    step.Name,
+		RelLeft:  step.LocalSource,
+		RelRight: step.ForeignSource,
+
+		On: dal.LinkPredicate{
+			Left:  step.LocalColumn,
+			Right: step.ForeignColumn,
+		},
+		Filter: f,
+	}
+	return
+}
+
+// updateDefinitionColumns assures run's frame column completeness
+func updateDefinitionColumns(r run) {
+	ppAttrs := r.Pipeline[0].Attributes()
+	for i, def := range r.Defs {
+		if len(def.Columns) > 0 {
+			continue
+		}
+
+		def.Columns = mappingToFrameCols(ppAttrs[i])
+	}
+}
+
+// makeModelRef returns the model ref based on the step load definition
+// @todo should be expanded when we support models that are not compose modules
+func makeModelRef(step types.ReportStepLoad) (out dal.ModelRef, err error) {
+	var (
+		connectionID          uint64
+		moduleID, namespaceID uint64
+		module, namespace     string
+
+		aux any
+		ok  bool
+	)
+
+	if aux, ok = step.Definition["moduleID"]; ok {
+		moduleID = cast.ToUint64(aux)
+	} else if aux, ok = step.Definition["module"]; ok {
+		module = cast.ToString(aux)
+	} else {
+		err = fmt.Errorf("step definition is missing moduleID or module")
+		return
+	}
+
+	if aux, ok = step.Definition["namespaceID"]; ok {
+		namespaceID = cast.ToUint64(aux)
+	} else if aux, ok = step.Definition["namespace"]; ok {
+		namespace = cast.ToString(aux)
+	} else {
+		err = fmt.Errorf("step definition is missing namespaceID or namespace")
+		return
+	}
+
+	// Connection is optional, default is primary connection
+	if aux, ok = step.Definition["connectionID"]; ok {
+		connectionID = cast.ToUint64(aux)
+	}
+
+	out.ConnectionID = connectionID
+	out.Refs = make(map[string]any)
+
+	// Use only one of the two identifier variations with priority to ID
+	if moduleID > 0 {
+		out.Refs["moduleID"] = moduleID
+	} else {
+		out.Refs["module"] = module
+	}
+
+	if namespaceID > 0 {
+		out.Refs["namespaceID"] = namespaceID
+	} else {
+		out.Refs["namespace"] = namespace
+	}
+
+	return
+}
+
+// modelAttributes returns the attributes defined on the referenced model
+func modelAttributes(pr modelFinder, step types.ReportStepLoad, mfr dal.ModelRef) []dal.AttributeMapping {
+	// All of the attributes
+	attrs, err := getModelAttrs(pr, mfr)
+	if err != nil {
+		return nil
+	}
+
+	return attrToMapping(attrs...)
+}
+
+func getModelAttrs(pr modelFinder, mfr dal.ModelRef) (attrs dal.AttributeSet, err error) {
+	m := pr.FindModel(mfr)
+	if m == nil {
+		return nil, fmt.Errorf("model not found: %v", mfr)
+	}
+
+	return m.Attributes, nil
+}
+
+func attrToMapping(aa ...*dal.Attribute) (out []dal.AttributeMapping) {
+	for _, a := range aa {
+		out = append(out, dal.SimpleAttr{
+			Ident: a.Ident,
+			Src:   a.Ident,
+			Props: dal.MapProperties{
+				Label:     a.Label,
+				Type:      a.Type,
+				IsSystem:  a.System,
+				Nullable:  a.Type.IsNullable(),
+				IsPrimary: a.PrimaryKey,
+				// @todo add multi value delimiter; it's currently not set on the
+				//       attribute so we might need to rethink this a bit
+				IsMultivalue: a.MultiValue,
+			},
+		})
+	}
+	return
+}
+
+func filterFromDef(def *FrameDefinition) (out filter.Filter) {
+	aux := filter.Generic(
+		filter.WithExpressionParsed(def.Filter.Node()),
+		filter.WithOrderBy(def.Sort),
+	)
+
+	if def.Paging != nil {
+		aux = aux.With(
+			filter.WithCursor(def.Paging.PageCursor),
+			filter.WithLimit(def.Paging.Limit),
+		)
+	}
+
+	return aux
+}

server/system/reporting/reporting_test.go

@@ -0,0 +1,328 @@
+package reporting
+
+import (
+	"context"
+	"testing"
+
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/cortezaproject/corteza/server/pkg/ql"
+	"github.com/cortezaproject/corteza/server/system/types"
+	"github.com/stretchr/testify/require"
+)
+
+type (
+	mockModelFinder struct{}
+)
+
+func (mockModelFinder) FindModel(dal.ModelRef) *dal.Model {
+	return &dal.Model{
+		Attributes: dal.AttributeSet{{
+			Ident: "attr1",
+			Label: "Attr 1",
+			Type:  dal.TypeText{Length: 100},
+		}, {
+			Ident: "attr2",
+			Label: "Attr 2",
+			Type:  dal.TypeNumber{},
+		}},
+	}
+}
+
+func TestRuns(t *testing.T) {
+	simpleSS := types.ReportStepSet{
+		{Load: &types.ReportStepLoad{
+			Name: "l1",
+			Definition: map[string]interface{}{
+				"module":    "mod",
+				"namespace": "ns",
+			},
+		}},
+	}
+
+	t.Run("no definitions", func(t *testing.T) {
+		rr, err := Runs(mockModelFinder{}, simpleSS, nil)
+		require.NoError(t, err)
+		require.Empty(t, rr)
+	})
+
+	t.Run("one definition", func(t *testing.T) {
+		defs := FrameDefinitionSet{
+			{Name: "d1", Source: "l1"},
+		}
+		rr, err := Runs(mockModelFinder{}, simpleSS, defs)
+		require.NoError(t, err)
+		require.Len(t, rr, 1)
+	})
+
+	t.Run("two different definitions for same source", func(t *testing.T) {
+		defs := FrameDefinitionSet{
+			{Name: "d1", Source: "l1"},
+			{Name: "d2", Source: "l1"},
+		}
+		rr, err := Runs(mockModelFinder{}, simpleSS, defs)
+		require.NoError(t, err)
+		require.Len(t, rr, 2)
+		require.Equal(t, "d1", rr[0].Defs[0].Name)
+		require.Equal(t, "d2", rr[1].Defs[0].Name)
+	})
+
+	t.Run("two defs for link", func(t *testing.T) {
+		defs := FrameDefinitionSet{
+			{Name: "d1", Source: "lnk", Ref: "l1"},
+			{Name: "d1", Source: "lnk", Ref: "l2"},
+		}
+
+		linkSS := types.ReportStepSet{
+			{Load: &types.ReportStepLoad{
+				Name: "l1",
+				Definition: map[string]interface{}{
+					"module":    "mod",
+					"namespace": "ns",
+				},
+			}},
+			{Load: &types.ReportStepLoad{
+				Name: "l2",
+				Definition: map[string]interface{}{
+					"module":    "mod",
+					"namespace": "ns",
+				},
+			}},
+			{Link: &types.ReportStepLink{
+				Name:          "lnk",
+				LocalSource:   "l1",
+				ForeignSource: "l2",
+				LocalColumn:   "lc",
+				ForeignColumn: "fc",
+			}},
+		}
+
+		rr, err := Runs(mockModelFinder{}, linkSS, defs)
+		require.NoError(t, err)
+		require.Len(t, rr, 1)
+		require.Equal(t, "d1", rr[0].Defs[0].Name)
+		require.Equal(t, "l1", rr[0].Defs[0].Ref)
+		require.Equal(t, "d1", rr[0].Defs[1].Name)
+		require.Equal(t, "l2", rr[0].Defs[1].Ref)
+	})
+
+	t.Run("link and regular", func(t *testing.T) {
+		defs := FrameDefinitionSet{
+			{Name: "d1", Source: "lnk", Ref: "l1"},
+			{Name: "d1", Source: "lnk", Ref: "l2"},
+
+			{Name: "d2", Source: "l1"},
+		}
+
+		linkSS := types.ReportStepSet{
+			{Load: &types.ReportStepLoad{
+				Name: "l1",
+				Definition: map[string]interface{}{
+					"module":    "mod",
+					"namespace": "ns",
+				},
+			}},
+			{Load: &types.ReportStepLoad{
+				Name: "l2",
+				Definition: map[string]interface{}{
+					"module":    "mod",
+					"namespace": "ns",
+				},
+			}},
+			{Link: &types.ReportStepLink{
+				Name:          "lnk",
+				LocalSource:   "l1",
+				ForeignSource: "l2",
+				LocalColumn:   "lc",
+				ForeignColumn: "fc",
+			}},
+		}
+
+		rr, err := Runs(mockModelFinder{}, linkSS, defs)
+		require.NoError(t, err)
+		require.Len(t, rr, 2)
+		require.Equal(t, "d1", rr[0].Defs[0].Name)
+		require.Equal(t, "l1", rr[0].Defs[0].Ref)
+		require.Equal(t, "d1", rr[0].Defs[1].Name)
+		require.Equal(t, "l2", rr[0].Defs[1].Ref)
+
+		require.Equal(t, "d2", rr[1].Defs[0].Name)
+	})
+
+	// Uses all possible step definitions in one go
+	t.Run("full definitions", func(t *testing.T) {
+		defs := FrameDefinitionSet{
+			// Load
+			{Name: "d1", Source: "l1"},
+
+			// Aggregate
+			{Name: "d2", Source: "agg1"},
+
+			// Link
+			{Name: "d3", Source: "lnk", Ref: "l1"},
+			{Name: "d3", Source: "lnk", Ref: "l2"},
+
+			// join
+			{Name: "d4", Source: "jn"},
+		}
+
+		rr, err := Runs(
+			mockModelFinder{},
+			types.ReportStepSet{
+				// Loads
+				{Load: &types.ReportStepLoad{
+					Name: "l1",
+					Definition: map[string]interface{}{
+						"module":    "mod",
+						"namespace": "ns",
+					},
+				}},
+				{Load: &types.ReportStepLoad{
+					Name: "l2",
+					Definition: map[string]interface{}{
+						"module":    "mod",
+						"namespace": "ns",
+					},
+				}},
+
+				// Aggregate
+				{Aggregate: &types.ReportStepAggregate{
+					Name:   "agg1",
+					Source: "l1",
+					Keys: types.ReportAggregateColumnSet{{
+						Name: "k1",
+						Def:  &types.ReportFilterExpr{ASTNode: &ql.ASTNode{Raw: "f1"}},
+					}},
+					Columns: types.ReportAggregateColumnSet{{
+						Name: "c1",
+						Def:  &types.ReportFilterExpr{ASTNode: &ql.ASTNode{Raw: "count(f1)"}},
+					}},
+				}},
+
+				// Link
+				{Link: &types.ReportStepLink{
+					Name:          "lnk",
+					LocalSource:   "l1",
+					ForeignSource: "l2",
+					LocalColumn:   "lc",
+					ForeignColumn: "fc",
+				}},
+
+				// Join
+				{Join: &types.ReportStepJoin{
+					Name:          "jn",
+					LocalSource:   "l1",
+					ForeignSource: "l2",
+					LocalColumn:   "lc",
+					ForeignColumn: "fc",
+				}},
+			},
+			defs,
+		)
+		require.NoError(t, err)
+		require.Len(t, rr, 4)
+	})
+}
+
+func TestStepLoadConv(t *testing.T) {
+	t.Run("ref from ID", func(t *testing.T) {
+		c, err := convStepLoad(
+			mockModelFinder{},
+			types.ReportStepLoad{
+				Name: "l1",
+				Definition: map[string]interface{}{
+					"moduleID":     42,
+					"namespaceID":  43,
+					"connectionID": 44,
+				},
+			},
+			nil,
+		)
+
+		require.NoError(t, err)
+		require.NotNil(t, c)
+	})
+
+	t.Run("ref from handles", func(t *testing.T) {
+		c, err := convStepLoad(
+			mockModelFinder{},
+			types.ReportStepLoad{
+				Name: "l1",
+				Definition: map[string]interface{}{
+					"module":       "mod1",
+					"namespace":    "ns1",
+					"connectionID": 44,
+				},
+			},
+			nil,
+		)
+
+		require.NoError(t, err)
+		require.NotNil(t, c)
+	})
+
+	t.Run("missing module ref", func(t *testing.T) {
+		_, err := convStepLoad(
+			mockModelFinder{},
+			types.ReportStepLoad{
+				Name: "l1",
+				Definition: map[string]interface{}{
+					"namespace":    "ns1",
+					"connectionID": 44,
+				},
+			},
+			nil,
+		)
+
+		require.Error(t, err)
+	})
+
+	t.Run("missing namespace ref", func(t *testing.T) {
+		_, err := convStepLoad(
+			mockModelFinder{},
+			types.ReportStepLoad{
+				Name: "l1",
+				Definition: map[string]interface{}{
+					"module":       "mod1",
+					"connectionID": 44,
+				},
+			},
+			nil,
+		)
+
+		require.Error(t, err)
+	})
+}
+
+func TestFrames_regular(t *testing.T) {
+	ctx := context.Background()
+
+	buff := dal.InMemoryBuffer()
+	buff.Add(ctx, (&dal.Row{}).WithValue("id", 0, 1).WithValue("f1", 0, "f1 v1").WithValue("f2", 0, "f2 v1"))
+	buff.Add(ctx, (&dal.Row{}).WithValue("id", 0, 2).WithValue("f1", 0, "f1 v2").WithValue("f2", 0, "f2 v2"))
+	buff.Add(ctx, (&dal.Row{}).WithValue("id", 0, 3).WithValue("f1", 0, "f1 v3").WithValue("f2", 0, "f2 v3"))
+
+	r := run{
+		Pipeline: dal.Pipeline{&dal.Datasource{Ident: "l1"}},
+
+		Defs: FrameDefinitionSet{
+			{
+				Name:   "d1",
+				Source: "l1",
+				Columns: FrameColumnSet{
+					{Name: "id", Label: "id", Kind: "Record"},
+					{Name: "f1", Label: "f1", Kind: "String"},
+				},
+			},
+		},
+	}
+
+	ff, err := Frames(ctx, buff, r)
+	require.NoError(t, err)
+
+	require.Len(t, ff, 1)
+	f := ff[0]
+	require.Len(t, f.Rows, 3)
+	require.Equal(t, FrameRow{"1", "f1 v1"}, f.Rows[0])
+	require.Equal(t, FrameRow{"2", "f1 v2"}, f.Rows[1])
+	require.Equal(t, FrameRow{"3", "f1 v3"}, f.Rows[2])
+}