snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/pkg/dal/exec_link_left.go

@@ -0,0 +1,505 @@
+package dal
+
+import (
+	"context"
+	"fmt"
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"io"
+	"sort"
+	"strconv"
+)
+
+type (
+	linkLeft struct {
+		def    Link
+		filter internalFilter
+
+		leftSource  Iterator
+		rightSource Iterator
+		err         error
+		scanRow     *Row
+		planned     bool
+		filtered    bool
+
+		rightSortAttrs []string
+
+		rowTester tester
+
+		// Buffer to keep track of pulled left rows
+		leftRows   []*Row
+		relIndex   *relIndex
+		keepLeft   bool
+		leftIndex  int
+		rightIndex int
+
+		// Some helper fields for temporary data
+		leftRow       *Row
+		rightRow      *Row
+		relScanBuffer *relIndexBuffer
+	}
+)
+
+func (xs *linkLeft) init(ctx context.Context, linkPredType Type, rightSort []string) (err error) {
+	// @note the index is keeping track of the right source attributes so we can
+	//       simplify the sorting logic.
+	xs.relIndex, err = newRelIndex(linkPredType, rightSort...)
+	if err != nil {
+		return
+	}
+
+	xs.rightSortAttrs = rightSort
+
+	xs.rowTester, err = prepareGenericRowTester(xs.filter)
+	if err != nil {
+		return
+	}
+
+	return xs.applyPlan(ctx)
+}
+
+func (xs *linkLeft) Next(ctx context.Context) (more bool) {
+	xs.err = xs.applyPlan(ctx)
+	if xs.err != nil {
+		return false
+	}
+
+	more, xs.err = xs.next(ctx)
+	return
+}
+
+func (xs *linkLeft) More(limit uint, v ValueGetter) (err error) {
+	xs.filter.cursor, err = xs.ForwardCursor(v)
+	if err != nil {
+		return
+	}
+
+	// Redo row tester
+	xs.rowTester, err = prepareGenericRowTester(xs.filter)
+	if err != nil {
+		return
+	}
+
+	// Redo the state
+	// @todo adjust based on aggregation plan; reuse buffered, etc.
+	xs.relIndex.Clear()
+	xs.leftRows = make([]*Row, 0, 128)
+	xs.scanRow = nil
+	xs.planned = false
+	xs.keepLeft = false
+	xs.leftIndex = 0
+	xs.rightIndex = 0
+
+	return
+}
+
+func (xs *linkLeft) Err() error { return xs.err }
+
+func (xs *linkLeft) Scan(s ValueSetter) (err error) {
+	for k, cc := range xs.scanRow.CountValues() {
+		for i := uint(0); i < cc; i++ {
+			// @note internal row won't raise errors so we can safely omit them
+			v, _ := xs.scanRow.GetValue(k, i)
+			err = s.SetValue(k, i, v)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	return
+}
+
+func (xs *linkLeft) Close() (err error) {
+	if xs == nil {
+		return
+	}
+
+	cc := []io.Closer{
+		xs.leftSource,
+		xs.rightSource,
+	}
+
+	for _, c := range cc {
+		if c != nil {
+			err = c.Close()
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return
+}
+
+func (xs *linkLeft) BackCursor(v ValueGetter) (pc *filter.PagingCursor, err error) {
+	g := &rowLink{
+		a: xs.leftRow,
+		b: v,
+	}
+	pc, err = filter.PagingCursorFrom(xs.filter.OrderBy(), g, xs.collectPrimaryAttributes()...)
+	if err != nil {
+		return nil, err
+	}
+
+	pc.ROrder = true
+	pc.LThen = xs.filter.OrderBy().Reversed()
+
+	return
+}
+
+func (xs *linkLeft) ForwardCursor(v ValueGetter) (pc *filter.PagingCursor, err error) {
+	g := &rowLink{
+		a: xs.leftRow,
+		b: v,
+	}
+	pc, err = filter.PagingCursorFrom(xs.filter.OrderBy(), g, xs.collectPrimaryAttributes()...)
+	if err != nil {
+		return nil, err
+	}
+
+	return
+}
+
+// // // // // // // // // // // // // // // // // // // // // // // // //
+// Utility methods
+
+// next prepares the next scan row based on the defined link plan
+func (xs *linkLeft) next(ctx context.Context) (more bool, err error) {
+	more, err = xs.pullNext(ctx)
+	if !more || err != nil {
+		return more, err
+	}
+
+	return xs.nextBuffered()
+}
+
+// pullNext pulls additional data so we can produce more
+//
+// This step may be omitted based on the join plan.
+func (xs *linkLeft) pullNext(ctx context.Context) (more bool, err error) {
+	// Pull next chunk from source if not entirely buffered
+	if xs.def.plan.partialScan {
+		return false, fmt.Errorf("partialScan join plan strategy not implemented")
+	}
+
+	// Check if buffer has more
+	// We need to check for gt because the right bits may still be relevant
+	if xs.leftIndex > len(xs.leftRows) {
+		return false, nil
+	}
+	return true, nil
+}
+
+// nextBuffered prepares the next scan row from the buffers
+func (xs *linkLeft) nextBuffered() (more bool, err error) {
+	// keepLeft indicates if we should keep the left row and move onto the next
+	// in the right buffers.
+	//
+	// If we're keeping it, take the next row from the other side, else take the
+	// next left one and reset right counters.
+
+	var (
+		ok bool
+	)
+
+	for {
+		if !xs.keepLeft {
+			// Go for the next left row
+			if xs.leftIndex >= len(xs.leftRows) {
+				return
+			}
+			xs.leftRow = xs.leftRows[xs.leftIndex]
+			xs.leftIndex++
+
+			// Go for the next right buffer
+			xs.relScanBuffer, ok, err = xs.getRelatedBuffer(xs.leftRow)
+			if !ok || err != nil {
+				return
+			}
+			// if len(rel) > 1 {
+			// 	// @todo implement this; not entirely sure how it should be so I'll block it for now
+			// 	// @todo move this check futher up
+			// 	return false, fmt.Errorf("multi-value link predicates not supported")
+			// }
+			// xs.relScanBuffer = rel[0]
+			xs.rightIndex = 0
+
+			xs.scanRow = xs.leftRow
+			xs.keepLeft = true
+			return true, nil
+		}
+
+		// Related buffer done
+		if xs.rightIndex >= len(xs.relScanBuffer.rows) {
+			xs.keepLeft = false
+			continue
+		}
+
+		xs.rightRow = xs.relScanBuffer.rows[xs.rightIndex]
+		xs.rightIndex++
+		xs.scanRow = xs.rightRow
+
+		// Sys defined value to indicate the source
+		// @todo consider using some wrap struct instead
+		xs.scanRow.SetValue(LinkRefIdent, 0, xs.def.RelRight)
+		break
+	}
+	return true, nil
+}
+
+// getRelatedBuffer returns all of the right rows corresponding to the given left row
+func (xs *linkLeft) getRelatedBuffer(l *Row) (out *relIndexBuffer, ok bool, err error) {
+
+	// @todo mv link predicate attrs
+	v, _ := l.GetValue(xs.def.On.Left, 0)
+	out, ok = xs.relIndex.Get(v)
+	return
+}
+
+// applyPlan runs plan specific logic to prepare the state
+func (xs *linkLeft) applyPlan(ctx context.Context) (err error) {
+	if xs.planned || xs.err != nil {
+		return
+	}
+
+	xs.planned = true
+	switch {
+	case !xs.def.plan.partialScan:
+		return xs.pullEntireSource(ctx)
+	}
+
+	return
+}
+
+// pullEntireSource pulls both sources into memory and indexes them for later use
+func (xs *linkLeft) pullEntireSource(ctx context.Context) (err error) {
+	// This bit does the filtering so just mark it of as such here
+	xs.filtered = true
+
+	// First the right source
+	err = xs.pullEntireRightSource(ctx)
+	if err != nil {
+		return
+	}
+
+	// Next the left source
+	err = xs.pullEntireLeftSource(ctx)
+	if err != nil {
+		return
+	}
+
+	// Sort if needed
+	return xs.sortLeftRows()
+}
+
+// pullEntireRightSource pulls and indexes all of the right bits
+func (xs *linkLeft) pullEntireRightSource(ctx context.Context) (err error) {
+	// Drain the source
+	for xs.rightSource.Next(ctx) {
+		r := &Row{
+			counters: make(map[string]uint),
+			values:   make(valueSet),
+		}
+
+		err = xs.rightSource.Scan(r)
+		if err != nil {
+			return
+		}
+
+		err = xs.indexRightRow(r)
+		if err != nil {
+			return
+		}
+	}
+	return xs.rightSource.Err()
+}
+
+// pullEntireLeftSource pulls and indexes all of the left bits
+func (xs *linkLeft) pullEntireLeftSource(ctx context.Context) (err error) {
+	var (
+		rel *relIndexBuffer
+		ok  bool
+	)
+
+	// Drain the source
+	for xs.leftSource.Next(ctx) {
+		l := &Row{
+			counters: make(map[string]uint),
+			values:   make(valueSet),
+		}
+
+		err = xs.leftSource.Scan(l)
+		if err != nil {
+			return
+		}
+
+		rel, ok, err = xs.getRelatedBuffer(l)
+		if err != nil {
+			return
+		}
+		if !ok {
+			continue
+		}
+
+		k, err := xs.keep(ctx, l, rel)
+		if err != nil {
+			return err
+		}
+		if !k {
+			continue
+		}
+
+		xs.leftRows = append(xs.leftRows, l)
+	}
+	return xs.leftSource.Err()
+}
+
+// indexRightRow pushes the provided row onto the rel index
+// @todo consider moving most of this logic to the relIndex struct.
+func (xs *linkLeft) indexRightRow(r *Row) (err error) {
+	// @todo mv link predicate attrs; should be prevented higher up for now
+	v, err := r.GetValue(xs.def.On.Right, 0)
+	if err != nil {
+		return err
+	}
+
+	xs.relIndex.Add(v, r)
+	return
+}
+
+// sortLeftRows sorts the left rows into the correct order
+//
+// Algorithm outline:
+// Compare the two left rows based on the defined sort order.
+// If the two left rows match up, or we're primarily sorting using right rows,
+// check the min/max stats values produced by the index buffer.
+//
+// If some min/max of a chunk is greater/lesser then the other, no row in the other
+// can appear before/after the prior.
+func (xs *linkLeft) sortLeftRows() (err error) {
+	var (
+		leftRelBufferA *relIndexBuffer
+		leftRelBufferB *relIndexBuffer
+
+		a, b any
+	)
+
+	// Use stable sort just so we don't needlesly messup the initial order if we decide
+	// to preserve sorts produced by further steps.
+	sort.SliceStable(xs.leftRows, func(i, j int) bool {
+		if err != nil {
+			return false
+		}
+
+		for _, s := range xs.filter.OrderBy() {
+			// Prepare the data
+			leftRowA := xs.leftRows[i]
+			leftRowB := xs.leftRows[j]
+
+			leftRowA = convertStringToIntForColumn(s.Column, leftRowA)
+			leftRowB = convertStringToIntForColumn(s.Column, leftRowB)
+
+			leftRelBufferA, _, err = xs.getRelatedBuffer(leftRowA)
+			if err != nil {
+				return false
+			}
+			leftRelBufferB, _, err = xs.getRelatedBuffer(leftRowB)
+			if err != nil {
+				return false
+			}
+
+			if !xs.isRightSortAttr(s.Column) {
+				// This bit here orders based on the left attributes
+				less, skip := evalCmpResult(compareGetters(leftRowA, leftRowB, leftRowA.counters, leftRowB.counters, s.Column), s)
+				if !skip {
+					return less
+				}
+			} else {
+				// This bit here orders based on the right attributes
+				// Check the stats of the buffer; make sure to adjust based on direction
+
+				// Use chunk's values
+				if !s.Descending {
+					a = leftRelBufferA.min[s.Column]
+					b = leftRelBufferB.min[s.Column]
+				} else {
+					a = leftRelBufferA.max[s.Column]
+					b = leftRelBufferB.max[s.Column]
+				}
+
+				less, skip := evalCmpResult(compareValues(a, b), s)
+				if !skip {
+					return less
+				}
+			}
+		}
+
+		return false
+	})
+
+	return
+}
+
+func (xs *linkLeft) isRightSortAttr(ident string) bool {
+	for _, i := range xs.rightSortAttrs {
+		if i == ident {
+			return true
+		}
+	}
+	return false
+}
+
+// keep checks if the row should be kept or discarded
+//
+// Link's keep is a bit more complicated and it looks at the related buffer as well.
+func (xs *linkLeft) keep(ctx context.Context, left *Row, buffer *relIndexBuffer) (keep bool, err error) {
+	// If no buffer, we won't keep -- left inner join like behavior
+	if buffer == nil {
+		return false, nil
+	}
+	// No tester include all ok rows
+	if xs.rowTester == nil {
+		return true, nil
+	}
+
+	ch := &rowLink{a: left}
+	for _, ch.b = range buffer.rows {
+		k, err := xs.rowTester.Test(ctx, ch)
+		if err != nil {
+			return false, err
+		}
+		if !k {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func (xs *linkLeft) collectPrimaryAttributes() (out []string) {
+	out = make([]string, 0, 2)
+	for _, m := range xs.def.OutLeftAttributes {
+		if m.Properties().IsPrimary {
+			out = append(out, m.Identifier())
+		}
+	}
+
+	for _, m := range xs.def.OutRightAttributes {
+		if m.Properties().IsPrimary {
+			out = append(out, m.Identifier())
+		}
+	}
+
+	return
+}
+
+// convertStringToIntForColumn converts the string value that have numeric data in the specified column
+// and updates the Row value datatype to int, if the conversion is successful
+func convertStringToIntForColumn(column string, row *Row) *Row {
+	if strValue, ok := row.values[column][0].(string); ok {
+		if rowVal, err := strconv.Atoi(strValue); err == nil {
+			row.values[column][0] = rowVal
+		}
+	}
+	return row
+}