upstream-corteza/server/compose/dalutils/records.go

package dalutils

import (
	"context"
	"fmt"
	"math"

	"github.com/cortezaproject/corteza/server/compose/types"
	"github.com/cortezaproject/corteza/server/pkg/dal"
	"github.com/cortezaproject/corteza/server/pkg/filter"
	"github.com/spf13/cast"
)

type (
	creator interface {
		Create(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, vv ...dal.ValueGetter) error
	}

	updater interface {
		Update(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, rr ...dal.ValueGetter) (err error)
	}

	searcher interface {
		Search(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, f filter.Filter) (dal.Iterator, error)
	}

	lookuper interface {
		Lookup(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, lookup dal.ValueGetter, dst dal.ValueSetter) (err error)
	}

	deleter interface {
		Delete(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, pkv ...dal.ValueGetter) (err error)
	}

	counter interface {
		Count(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, f filter.Filter) (uint, error)
	}
)

// ComposeRecordsList iterates over results and collects all available records
func ComposeRecordsList(ctx context.Context, s searcher, mod *types.Module, filter types.RecordFilter) (set types.RecordSet, outFilter types.RecordFilter, err error) {
	iter, err := prepIterator(ctx, s, mod, filter)
	if err != nil {
		return
	}

	set, _, outFilter, err = drainIterator(ctx, iter, mod, filter)
	return
}

func ComposeRecordsListN(ctx context.Context, s searcher, mod *types.Module, filter types.RecordFilter) (set types.RecordSet, summaries map[string]types.RecordSummary, outFilter types.RecordFilter, err error) {
	iter, err := prepIterator(ctx, s, mod, filter)
	if err != nil {
		return
	}

	set, summaries, outFilter, err = drainIterator(ctx, iter, mod, filter)
	return
}

func ComposeRecordsIterator(ctx context.Context, s searcher, mod *types.Module, filter types.RecordFilter) (iter dal.Iterator, outFilter types.RecordFilter, err error) {
	iter, err = prepIterator(ctx, s, mod, filter)
	if err != nil {
		return
	}

	outFilter = filter
	outFilter.Paging = *filter.Paging.Clone()

	return
}

func ComposeRecordsFind(ctx context.Context, l lookuper, mod *types.Module, recordID uint64) (out *types.Record, err error) {
	out = prepareRecordTarget(mod)

	err = l.Lookup(ctx, mod.ModelRef(), recLookupOperations(mod), dal.PKValues{"id": recordID}, out)
	if err != nil {
		return
	}

	return
}

func ComposeRecordsCount(ctx context.Context, c counter, mod *types.Module, filter types.RecordFilter) (cnt uint, err error) {
	constraints := map[string][]interface{}{
		"namespaceID": {mod.NamespaceID},
	}

	if filter.ModuleID != 0 {
		constraints["moduleID"] = []interface{}{filter.ModuleID}
	}

	dalFilter := filter.ToConstraintedFilter(constraints)

	return c.Count(ctx, mod.ModelRef(), recLookupOperations(mod), dalFilter)
}

func ComposeRecordCreate(ctx context.Context, c creator, mod *types.Module, records ...*types.Record) (err error) {
	return c.Create(ctx, mod.ModelRef(), recCreateOperations(mod), recToGetters(records...)...)
}

func ComposeRecordUpdate(ctx context.Context, u updater, mod *types.Module, records ...*types.Record) (err error) {
	return u.Update(ctx, mod.ModelRef(), recUpdateOperations(mod), recToGetters(records...)...)
}

func ComposeRecordSoftDelete(ctx context.Context, u updater, mod *types.Module, records ...*types.Record) (err error) {
	return u.Update(ctx, mod.ModelRef(), recUpdateOperations(mod), recToGetters(records...)...)
}

func ComposeRecordUndelete(ctx context.Context, u updater, mod *types.Module, records ...*types.Record) (err error) {
	return u.Update(ctx, mod.ModelRef(), recUpdateOperations(mod), recToGetters(records...)...)
}

func ComposeRecordDelete(ctx context.Context, d deleter, mod *types.Module, records ...*types.Record) (err error) {
	return d.Delete(ctx, mod.ModelRef(), recDeleteOperations(mod), recToGetters(records...)...)
}

func WalkIterator(ctx context.Context, iter dal.Iterator, mod *types.Module, f func(r *types.Record) error) (err error) {
	for iter.Next(ctx) {
		r := prepareRecordTarget(mod)
		if err = iter.Scan(r); err != nil {
			return
		}

		if err = f(r); err != nil {
			return
		}
	}

	return iter.Err()
}

// // // // // // // // // // // // // // // // // // // // // // // // //
// Utils

func prepFilter(filter types.RecordFilter, mod *types.Module) filter.Filter {
	return filter.ToConstraintedFilter(mod.Config.DAL.Constraints)
}

func prepIterator(ctx context.Context, dal searcher, mod *types.Module, filter types.RecordFilter) (iter dal.Iterator, err error) {
	dalFilter := prepFilter(filter, mod)

	iter, err = dal.Search(ctx, mod.ModelRef(), recSearchOperations(mod, filter), dalFilter)
	return
}

// drains iterator and collects all records
//
// Collection of records is done with respect to check function and limit constraint on record filter
// For any other filter constraint we assume that underlying DAL took care of it
func drainIterator(ctx context.Context, iter dal.Iterator, mod *types.Module, f types.RecordFilter) (set types.RecordSet, summaries map[string]types.RecordSummary, outFilter types.RecordFilter, err error) {
	// close iterator after we've drained it
	defer iter.Close()

	if f.PageCursor != nil {
		if f.IncPageNavigation || f.IncTotal {
			err = fmt.Errorf("not allowed to fetch page navigation or total item count with page cursor")
			return
		}
	}

	var (
		ok         bool
		fetched    uint
		filtered   uint
		lastRecord *types.Record
	)

	// Get the requested number of record
	if f.Limit > 0 {
		set = make(types.RecordSet, 0, f.Limit)
	} else {
		set = make(types.RecordSet, 0, 1000)
	}

	for f.Limit == 0 || uint(len(set)) < f.Limit {
		var firstRecord *types.Record

		// reset counters every drain
		fetched = 0
		filtered = 0

		add := make(types.RecordSet, 0, 12)
		err = WalkIterator(ctx, iter, mod, func(r *types.Record) error {
			lastRecord = r
			if firstRecord == nil {
				firstRecord = r
			}

			// check fetched record
			if f.Check != nil {
				if ok, err = f.Check(r); err != nil {
					return err
				} else if !ok {
					filtered++
					return nil
				}
			}

			fetched++
			add = append(add, r)
			return err
		})

		// if an error occurred inside Next()/WalkIterator,
		// we need to stop draining
		if err != nil {
			return
		}

		// If it's reverse, we need to add extra fetches to the start
		if f.PageCursor != nil && f.PageCursor.ROrder {
			set = append(add, set...)
		} else {
			set = append(set, add...)
		}

		total := fetched + filtered
		if total == 0 || f.Limit == 0 {
			// do not re-fetch if:
			// 1) nothing was fetch in the previous run
			// 2) there was no limit (everything was fetched)
			// 3) there are less total (fetched and filtered) items then value of limit
			break
		}

		// Fetch more records
		setLen := uint(len(set))
		if total > 0 && setLen < f.Limit {
			fetchMore := f.Limit - setLen
			var crsrRec *types.Record

			// request more items
			if f.PageCursor == nil || !f.PageCursor.ROrder {
				crsrRec = lastRecord
			} else {
				crsrRec = firstRecord
			}

			if err = iter.More(fetchMore, crsrRec); err != nil {
				return
			}
		}
	}

	// Get the page nav/total/next-prev cursors
	nav, auxSm, err := generatePageNavigation(ctx, iter, mod, f, set)
	if err != nil {
		return
	}

	summaries = auxSm

	// Make out filter
	outFilter = f
	outFilter.Paging = nav.Paging
	outFilter.Sorting = nav.Sorting

	return
}

// generatePageNavigation generates page navigation for a given record set using an iterator and filter limit.
// If the limit is not defined, the page navigation will consist of only one page without a cursor.
// If the limit is defined and is greater than the total number of records in the set,
// the page navigation will consist of only one page without a cursor.
// If the limit is defined and is less than the total number of records in the set,
// the page navigation will have multiple pages with cursor(s) based on the total number of records and the provided limit.
// @todo revisit and clean up this function properly
func generatePageNavigation(ctx context.Context, iter dal.Iterator, mod *types.Module, p types.RecordFilter, set types.RecordSet) (out types.RecordFilter, summaries map[string]types.RecordSummary, err error) {
	const (
		howMuchMore = 1000
	)
	var (
		ok      bool
		total   = uint(len(set))
		setLen  = len(set)
		counter = 0

		first *types.Record
		last  *types.Record
		page  filter.Page

		pageNavigation = []*filter.Page{
			{
				Page:   1,
				Count:  0,
				Cursor: nil,
			},
		}

		// generatePage generates pageNavigation for given record set
		generatePage = func(last *types.Record) (err error) {
			if !p.IncPageNavigation || p.Limit == 0 || len(pageNavigation) == 0 {
				return
			}

			lastNavPageNo := len(pageNavigation) - 1
			nextPage, err := iter.ForwardCursor(last)
			if err != nil {
				return
			}

			if total < p.Limit {
				pageNavigation[lastNavPageNo].Count = total
			}

			// prepare page
			if total != 0 && (total%p.Limit) == 0 {
				pageNavigation[lastNavPageNo].Count = p.Limit
				page = filter.Page{
					Page:   uint(len(pageNavigation) + 1),
					Count:  p.Limit,
					Cursor: nextPage,
				}
			}

			expectedItemCountUpToPage := uint(lastNavPageNo+1) * p.Limit
			if p.Limit == 1 {
				expectedItemCountUpToPage = uint(lastNavPageNo) * p.Limit
			}

			if expectedItemCountUpToPage < total {
				// push page when limit is matched with the previous page item size
				if pageNavigation[lastNavPageNo].Count == p.Limit {
					pageNavigation = append(pageNavigation, &filter.Page{
						Page:   page.Page,
						Count:  total % p.Limit,
						Cursor: page.Cursor, // prev cursor
					})
				}
			}

			return
		}

		recordChecker = func(i dal.Iterator) (ok bool, err error) {
			if p.Check == nil {
				return true, err
			}

			rc := prepareRecordTarget(mod)
			err = i.Scan(rc)

			if err != nil {
				return
			}

			return p.Check(rc)
		}
	)

	if len(p.Summaries) > 0 {
		summaries = make(map[string]types.RecordSummary, len(p.Summaries))
	}

	if setLen == 0 {
		return
	}

	existing := make(map[any]struct{}, 24)
	looped := false

	procSummary := func(r *types.Record) (err error) {
		for _, smDef := range p.Summaries {
			// Get record value
			var vv []any
			vv, err = r.GetValues(smDef.Field)
			if err != nil {
				return
			}

			bit := summaries[fmt.Sprintf("%s %s", smDef.Name, smDef.Field)]

			// This will be constant so we're good
			bit.Name = smDef.Name

			// Skip empty
			if len(vv) == 0 {
				bit.EmptyCount++
				summaries[fmt.Sprintf("%s %s", smDef.Name, smDef.Field)] = bit
				continue
			}

			bit.NotEmptyCount++

			for _, v := range vv {
				bit.Count++

				if _, ok := existing[v]; !ok {
					existing[v] = struct{}{}
					bit.UniqueCount++
				}
			}

			switch smDef.Name {
			case "min":
				for _, v := range vv {
					if !looped {
						bit.Min = cast.ToFloat64(v)
					} else {
						bit.Min = math.Min(bit.Min, cast.ToFloat64(v))
					}
				}

			case "max":
				for _, v := range vv {
					if !looped {
						bit.Max = cast.ToFloat64(v)
					} else {
						bit.Max = math.Max(bit.Max, cast.ToFloat64(v))
					}
				}

			case "avg":
				for _, v := range vv {
					bit.Sum += cast.ToFloat64(v)
				}

			case "sum":
				for _, v := range vv {
					bit.Sum += cast.ToFloat64(v)
				}

			case "earliest":
				for _, v := range vv {
					if bit.Earliest.IsZero() {
						bit.Earliest = cast.ToTime(v)
					} else {
						aux := cast.ToTime(v)
						if !aux.IsZero() && aux.Before(bit.Earliest) {
							bit.Earliest = aux
						}
					}
				}

			case "latest":
				for _, v := range vv {
					if bit.Latest.IsZero() {
						bit.Latest = cast.ToTime(v)
					} else {
						aux := cast.ToTime(v)
						if !aux.IsZero() && aux.After(bit.Latest) {
							bit.Latest = aux
						}
					}
				}
			}

			looped = true
			summaries[fmt.Sprintf("%s %s", smDef.Name, smDef.Field)] = bit
		}
		return
	}

	// Firstly sort out the current things
	if len(p.Summaries) > 0 {
		for _, r := range set {
			err = procSummary(r)
			if err != nil {
				return
			}
		}
	}

	first = set[0]
	last = set[setLen-1]

	// Limit
	out.Limit = p.Limit

	// Sorting
	out.Sort = dal.IteratorSorting(iter)

	// No need to generate prev/next cursor
	// if limit is not defined and set is empty
	if p.Limit > 0 && len(set) > 0 {
		// PrevPage
		if p.PageCursor != nil {
			out.PrevPage, err = dal.PreLoadCursor(ctx, iter, 100, true, first, recordChecker)
			if err != nil {
				return
			}
		}

		// NextPage
		out.NextPage, err = dal.PreLoadCursor(ctx, iter, 100, false, last, recordChecker)
		if err != nil {
			return
		}
	}

	if p.IncTotal || p.IncPageNavigation || len(p.Summaries) > 0 {
		// For the first page nav
		err = generatePage(last)
		if err != nil {
			return
		}

		for counter == 0 || counter < howMuchMore {
			counter++

			interLoop := 0

			if err = iter.More(howMuchMore, last); err != nil {
				return
			}

			err = WalkIterator(ctx, iter, mod, func(rec *types.Record) error {
				// check fetched record
				if p.Check != nil {
					if ok, err = p.Check(rec); err != nil {
						return err
					} else if !ok {
						return nil
					}
				}

				err = procSummary(rec)
				if err != nil {
					return err
				}

				interLoop++
				total++
				last = rec
				return generatePage(rec)
			})
			if err != nil {
				return
			}

			if interLoop < howMuchMore {
				break
			}
		}
	}

	// Total
	if p.IncTotal {
		out.Total = total
	}

	// Page navigation
	if p.IncPageNavigation {
		// Ensure that the last page count is correct if it's not equal to the limit.
		lastPageCount := pageNavigation[len(pageNavigation)-1].Count
		if lastPageCount > 0 && lastPageCount != p.Limit && lastPageCount != total%p.Limit {
			pageNavigation[len(pageNavigation)-1].Count = total % p.Limit
		}

		if p.Limit == 1 {
			pageNavigation = pageNavigation[:len(pageNavigation)-1]
		}

		out.PageNavigation = pageNavigation
	}

	// Do averages
	if len(p.Summaries) > 0 {
		for n, s := range summaries {
			if s.Name != "avg" {
				continue
			}

			s.Avg = s.Sum / float64(s.Count)
			summaries[n] = s
		}
	}

	return
}

func prepareRecordTarget(module *types.Module) *types.Record {
	// so we can avoid some code later involving (non)partitioned modules :seenoevil:
	r := &types.Record{
		ModuleID:    module.ID,
		NamespaceID: module.NamespaceID,
		Values:      make(types.RecordValueSet, 0, len(module.Fields)),
	}
	r.SetModule(module)

	return r
}

func recToGetters(rr ...*types.Record) (out []dal.ValueGetter) {
	out = make([]dal.ValueGetter, len(rr))

	for i := range rr {
		out[i] = rr[i]
	}

	return
}

func recCreateOperations(m *types.Module) (out dal.OperationSet) {
	return dal.CreateOperations()
}

func recUpdateOperations(m *types.Module) (out dal.OperationSet) {
	return dal.UpdateOperations()
}

func recDeleteOperations(m *types.Module) (out dal.OperationSet) {
	return dal.DeleteOperations()
}

func recFilterOperations(f types.RecordFilter) (out dal.OperationSet) {
	if f.PageCursor != nil {
		out = append(out, dal.Paging)
	}

	if f.IncPageNavigation {
		out = append(out, dal.Paging)
	}

	if f.Sort != nil {
		out = append(out, dal.Sorting)
	}

	return
}

func recSearchOperations(m *types.Module, f types.RecordFilter) (out dal.OperationSet) {
	return dal.SearchOperations().
		Union(recFilterOperations(f))
}

func recLookupOperations(m *types.Module) (out dal.OperationSet) {
	return dal.LookupOperations()
}