snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/pkg/envoy/store/compose.go

@@ -0,0 +1,579 @@
+package store
+
+import (
+	"context"
+	"strconv"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/compose/dalutils"
+	"github.com/cortezaproject/corteza/server/compose/service"
+	"github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/cortezaproject/corteza/server/pkg/envoy"
+	"github.com/cortezaproject/corteza/server/pkg/envoy/resource"
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/cortezaproject/corteza/server/store"
+	stypes "github.com/cortezaproject/corteza/server/system/types"
+)
+
+type (
+	composeNamespaceFilter types.NamespaceFilter
+	composeModuleFilter    types.ModuleFilter
+	composeRecordFilter    types.RecordFilter
+	composePageFilter      types.PageFilter
+	composeChartFilter     types.ChartFilter
+
+	composeStore interface {
+		store.ComposeAttachments
+		store.ComposeCharts
+		store.ComposeModuleFields
+		store.ComposeModules
+		store.ComposeNamespaces
+		store.ComposePages
+	}
+
+	dalService interface {
+		Create(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, vv ...dal.ValueGetter) error
+		Search(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, f filter.Filter) (dal.Iterator, error)
+		Delete(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, pkv ...dal.ValueGetter) (err error)
+		Update(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, pkv ...dal.ValueGetter) (err error)
+
+		ReplaceModel(ctx context.Context, currentAlts []*dal.Alteration, model *dal.Model) (newAlts []*dal.Alteration, err error)
+		GetConnectionByID(uint64) *dal.ConnectionWrap
+	}
+
+	composeDecoder struct {
+		resourceID  []uint64
+		namespaceID []uint64
+
+		dal dalService
+	}
+)
+
+func newComposeDecoder(dal dalService) *composeDecoder {
+	return &composeDecoder{
+		resourceID:  make([]uint64, 0, 200),
+		namespaceID: make([]uint64, 0, 200),
+		dal:         dal,
+	}
+}
+
+func (d *composeDecoder) decodeComposeNamespace(ctx context.Context, s composeStore, ff []*composeNamespaceFilter) *auxRsp {
+	mm := make([]envoy.Marshaller, 0, 100)
+	if ff == nil {
+		return &auxRsp{
+			mm: mm,
+		}
+	}
+
+	var nn types.NamespaceSet
+	var fn types.NamespaceFilter
+	var err error
+	for _, f := range ff {
+		aux := *f
+
+		if aux.Limit == 0 {
+			aux.Limit = 1000
+		}
+
+		for {
+			nn, fn, err = s.SearchComposeNamespaces(ctx, types.NamespaceFilter(aux))
+			if err != nil {
+				return &auxRsp{
+					err: err,
+				}
+			}
+
+			for _, n := range nn {
+				d.namespaceID = append(d.namespaceID, n.ID)
+				d.resourceID = append(d.resourceID, n.ID)
+
+				mm = append(mm, newComposeNamespace(n))
+			}
+
+			if f.NextPage != nil {
+				aux.PageCursor = fn.NextPage
+			} else {
+				break
+			}
+		}
+	}
+
+	return &auxRsp{
+		mm: mm,
+	}
+}
+
+func (d *composeDecoder) decodeComposeModule(ctx context.Context, s composeStore, ff []*composeModuleFilter) *auxRsp {
+	mm := make([]envoy.Marshaller, 0, 100)
+	if ff == nil {
+		return &auxRsp{
+			mm: mm,
+		}
+	}
+
+	if len(d.namespaceID) > 0 {
+		ffNs := make([]*composeModuleFilter, 0, len(ff)+len(d.namespaceID))
+		for _, nsID := range d.namespaceID {
+			for _, f := range ff {
+				fNs := *f
+				fNs.NamespaceID = nsID
+				ffNs = append(ffNs, &fNs)
+			}
+		}
+		ff = ffNs
+	}
+
+	var nn types.ModuleSet
+	var fn types.ModuleFilter
+	var err error
+	for _, f := range ff {
+		aux := *f
+
+		if aux.Limit == 0 {
+			aux.Limit = 1000
+		}
+
+		for {
+			nn, fn, err = s.SearchComposeModules(ctx, types.ModuleFilter(aux))
+			if err != nil {
+				return &auxRsp{
+					err: err,
+				}
+			}
+
+			for _, n := range nn {
+				d.resourceID = append(d.resourceID, n.ID)
+
+				n.Fields, _, err = s.SearchComposeModuleFields(ctx, types.ModuleFieldFilter{
+					ModuleID: []uint64{n.ID},
+				})
+
+				if err != nil {
+					return &auxRsp{
+						err: err,
+					}
+				}
+
+				mm = append(mm, newComposeModule(n))
+			}
+
+			if f.NextPage != nil {
+				aux.PageCursor = fn.NextPage
+			} else {
+				break
+			}
+		}
+	}
+
+	return &auxRsp{
+		mm: mm,
+	}
+}
+
+func (d *composeDecoder) decodeComposeRecord(ctx context.Context, s store.Storer, ff []*composeRecordFilter) *auxRsp {
+	mm := make([]envoy.Marshaller, 0, 100)
+	if ff == nil {
+		return &auxRsp{
+			mm: mm,
+		}
+	}
+
+	ac := service.AccessControl(s)
+
+	if len(d.namespaceID) > 0 {
+		ffNs := make([]*composeRecordFilter, 0, len(ff)+len(d.namespaceID))
+		for _, nsID := range d.namespaceID {
+			for _, f := range ff {
+				nn, _, err := s.SearchComposeModules(ctx, types.ModuleFilter{
+					NamespaceID: nsID,
+				})
+				if err != nil {
+					continue
+				}
+				for _, n := range nn {
+					fNs := *f
+					fNs.NamespaceID = nsID
+					fNs.ModuleID = n.ID
+					ffNs = append(ffNs, &fNs)
+				}
+			}
+		}
+		ff = ffNs
+	}
+
+	// When decoding large amounts of records (milions) we can probably assume
+	// that each system user exists somewhere in the record set.
+	//
+	// That said, it's probably cheeper to simply say that we probably need all of the
+	// references to hold.
+	// If a user is not there in the preparation step, we shouldn't fail straight away.
+	//
+	// @todo use some heuristic to determine when to just list all of the users
+	//       and when to preprocess the records to find out what users to get.
+
+	relUsers := make(resource.UserstampIndex)
+	uu, _, err := store.SearchUsers(ctx, s, stypes.UserFilter{
+		Paging: filter.Paging{
+			Limit: 0,
+		},
+	})
+	if err != nil {
+		return &auxRsp{
+			err: err,
+		}
+	}
+	relUsers.Add(uu...)
+
+	mapValues := func(r *types.Record) map[string]resource.ComposeRecordRawValue {
+		mod := r.GetModule()
+		out := make(map[string]resource.ComposeRecordRawValue, len(r.Values))
+
+		for _, v := range r.Values {
+			f := mod.Fields.FindByName(v.Name)
+			if f == nil || !f.Multi {
+				out[v.Name] = resource.ComposeRecordRawValue{
+					Name:  v.Name,
+					Value: v.Value,
+				}
+			} else {
+				if _, ok := out[v.Name]; !ok {
+					out[v.Name] = resource.ComposeRecordRawValue{
+						Name:    v.Name,
+						IsMulti: true,
+					}
+				}
+
+				vx := out[v.Name]
+				vx.Values = append(vx.Values, v.Value)
+				out[v.Name] = vx
+			}
+		}
+
+		return out
+	}
+
+	// Prepare a series of resource.ComposeRecord instances; one for each provided filter
+	for _, f := range ff {
+		aux := *f
+
+		if aux.Limit == 0 {
+			aux.Limit = 1000
+		}
+
+		mod, err := store.LookupComposeModuleByID(ctx, s, f.ModuleID)
+		if err != nil {
+			return &auxRsp{
+				err: err,
+			}
+		}
+
+		ff, _, err := store.SearchComposeModuleFields(ctx, s, types.ModuleFieldFilter{ModuleID: []uint64{mod.ID}})
+		if err != nil {
+			return &auxRsp{
+				err: err,
+			}
+		}
+
+		_ = ff.Walk(func(f *types.ModuleField) error {
+			f.NamespaceID = mod.NamespaceID
+			return nil
+		})
+		mod.Fields = ff
+
+		aux.Check = service.ComposeRecordFilterChecker(ctx, ac, mod)
+
+		// Refs
+		auxRecord := &composeRecordAux{
+			refMod:   strconv.FormatUint(f.ModuleID, 10),
+			relMod:   mod,
+			refNs:    strconv.FormatUint(f.NamespaceID, 10),
+			relUsers: relUsers,
+		}
+
+		// Walker
+		auxRecord.walker = func(cb func(r *resource.ComposeRecordRaw) error) error {
+			var rr types.RecordSet
+			var fn types.RecordFilter
+			var err error
+
+			for {
+				rr, fn, err = dalutils.ComposeRecordsList(ctx, d.dal, mod, types.RecordFilter(aux))
+				if err != nil {
+					return err
+				}
+
+				service.ComposeRecordFilterAC(ctx, ac, mod, rr...)
+
+				for _, n := range rr {
+					// Create a raw record
+					r := &resource.ComposeRecordRaw{
+						ID:     strconv.FormatUint(n.ID, 10),
+						Values: mapValues(n),
+						Ts:     resource.MakeTimestampsCUDA(&n.CreatedAt, n.UpdatedAt, n.DeletedAt, nil),
+						Us:     resource.MakeUserstampsCUDO(n.CreatedBy, n.UpdatedBy, n.DeletedBy, n.OwnedBy),
+					}
+
+					err = cb(r)
+					if err != nil {
+						return err
+					}
+				}
+
+				if fn.NextPage != nil {
+					aux.PageCursor = fn.NextPage
+				} else {
+					break
+				}
+			}
+			return nil
+		}
+		mm = append(mm, newComposeRecordFromAux(auxRecord))
+
+	}
+
+	return &auxRsp{
+		mm: mm,
+	}
+}
+
+func (d *composeDecoder) decodeComposePage(ctx context.Context, s composeStore, ff []*composePageFilter) *auxRsp {
+	mm := make([]envoy.Marshaller, 0, 100)
+	if ff == nil {
+		return &auxRsp{
+			mm: mm,
+		}
+	}
+
+	if len(d.namespaceID) > 0 {
+		ffNs := make([]*composePageFilter, 0, len(ff)+len(d.namespaceID))
+		for _, nsID := range d.namespaceID {
+			for _, f := range ff {
+				fNs := *f
+				fNs.NamespaceID = nsID
+				ffNs = append(ffNs, &fNs)
+			}
+		}
+		ff = ffNs
+	}
+
+	var nn types.PageSet
+	var fn types.PageFilter
+	var err error
+	for _, f := range ff {
+		aux := *f
+
+		if aux.Limit == 0 {
+			aux.Limit = 1000
+		}
+
+		for {
+			nn, fn, err = s.SearchComposePages(ctx, types.PageFilter(aux))
+			if err != nil {
+				return &auxRsp{
+					err: err,
+				}
+			}
+
+			for _, n := range nn {
+				d.resourceID = append(d.resourceID, n.ID)
+
+				mm = append(mm, newComposePage(n))
+			}
+
+			if fn.NextPage != nil {
+				aux.PageCursor = fn.NextPage
+			} else {
+				break
+			}
+		}
+	}
+
+	return &auxRsp{
+		mm: mm,
+	}
+}
+
+func (d *composeDecoder) decodeComposeChart(ctx context.Context, s composeStore, ff []*composeChartFilter) *auxRsp {
+	mm := make([]envoy.Marshaller, 0, 100)
+	if ff == nil {
+		return &auxRsp{
+			mm: mm,
+		}
+	}
+
+	if len(d.namespaceID) > 0 {
+		ffNs := make([]*composeChartFilter, 0, len(ff)+len(d.namespaceID))
+		for _, nsID := range d.namespaceID {
+			for _, f := range ff {
+				fNs := *f
+				fNs.NamespaceID = nsID
+				ffNs = append(ffNs, &fNs)
+			}
+		}
+		ff = ffNs
+	}
+
+	var nn types.ChartSet
+	var fn types.ChartFilter
+	var err error
+	for _, f := range ff {
+		aux := *f
+
+		if aux.Limit == 0 {
+			aux.Limit = 1000
+		}
+
+		for {
+			nn, fn, err = s.SearchComposeCharts(ctx, types.ChartFilter(aux))
+			if err != nil {
+				return &auxRsp{
+					err: err,
+				}
+			}
+
+			for _, n := range nn {
+				d.resourceID = append(d.resourceID, n.ID)
+
+				mm = append(mm, newComposeChart(n))
+			}
+
+			if f.Limit > 0 {
+				break
+			} else if fn.NextPage != nil {
+				aux.PageCursor = fn.NextPage
+			} else {
+				break
+			}
+		}
+	}
+
+	return &auxRsp{
+		mm: mm,
+	}
+}
+
+func (df *DecodeFilter) composeFromResource(rr ...string) *DecodeFilter {
+	for _, r := range rr {
+		if !strings.HasPrefix(r, "compose") {
+			continue
+		}
+
+		id := ""
+		if strings.Count(r, ":") == 2 && !strings.HasSuffix(r, "*") {
+			// There is an identifier
+			aux := strings.Split(r, ":")
+
+			id = aux[len(aux)-1]
+			r = strings.Join(aux[:len(aux)-1], ":")
+		}
+
+		switch strings.ToLower(r) {
+
+		case "compose:namespace":
+			df = df.ComposeNamespace(&types.NamespaceFilter{
+				Query: id,
+			})
+		case "compose:module":
+			df = df.ComposeModule(&types.ModuleFilter{
+				Query: id,
+			})
+		case "compose:record":
+			df = df.ComposeRecord(&types.RecordFilter{
+				Query: id,
+			})
+		case "compose:page":
+			df = df.ComposePage(&types.PageFilter{
+				Query: id,
+			})
+		case "compose:chart":
+			df = df.ComposeChart(&types.ChartFilter{
+				Query: id,
+			})
+		}
+	}
+
+	return df
+}
+
+func (df *DecodeFilter) composeFromRef(rr ...*resource.Ref) *DecodeFilter {
+	for _, r := range rr {
+		if strings.Index(r.ResourceType, "compose") < 0 {
+			continue
+		}
+
+		switch r.ResourceType {
+		case types.NamespaceResourceType:
+			for _, i := range r.Identifiers.StringSlice() {
+				df = df.ComposeNamespace(&types.NamespaceFilter{
+					Query: i,
+				})
+			}
+		case types.ModuleResourceType:
+			for _, i := range r.Identifiers.StringSlice() {
+				df = df.ComposeModule(&types.ModuleFilter{
+					Query: i,
+				})
+			}
+		case types.PageResourceType:
+			for _, i := range r.Identifiers.StringSlice() {
+				df = df.ComposePage(&types.PageFilter{
+					Query: i,
+				})
+			}
+		case types.ChartResourceType:
+			for _, i := range r.Identifiers.StringSlice() {
+				df = df.ComposeChart(&types.ChartFilter{
+					Query: i,
+				})
+			}
+		}
+	}
+
+	return df
+}
+
+// ComposeNamespace adds a new compose NamespaceFilter
+func (df *DecodeFilter) ComposeNamespace(f *types.NamespaceFilter) *DecodeFilter {
+	if df.composeNamespace == nil {
+		df.composeNamespace = make([]*composeNamespaceFilter, 0, 1)
+	}
+	df.composeNamespace = append(df.composeNamespace, (*composeNamespaceFilter)(f))
+	return df
+}
+
+// ComposeModule adds a new compose ModuleFilter
+func (df *DecodeFilter) ComposeModule(f *types.ModuleFilter) *DecodeFilter {
+	if df.composeModule == nil {
+		df.composeModule = make([]*composeModuleFilter, 0, 1)
+	}
+	df.composeModule = append(df.composeModule, (*composeModuleFilter)(f))
+	return df
+}
+
+// ComposeRecord adds a new compose RecordFilter
+func (df *DecodeFilter) ComposeRecord(f *types.RecordFilter) *DecodeFilter {
+	if df.composeRecord == nil {
+		df.composeRecord = make([]*composeRecordFilter, 0, 1)
+	}
+	df.composeRecord = append(df.composeRecord, (*composeRecordFilter)(f))
+	return df
+}
+
+// ComposePage adds a new compose PageFilter
+func (df *DecodeFilter) ComposePage(f *types.PageFilter) *DecodeFilter {
+	if df.composePage == nil {
+		df.composePage = make([]*composePageFilter, 0, 1)
+	}
+	df.composePage = append(df.composePage, (*composePageFilter)(f))
+	return df
+}
+
+// ComposeChart adds a new compose ChartFilter
+func (df *DecodeFilter) ComposeChart(f *types.ChartFilter) *DecodeFilter {
+	if df.composeChart == nil {
+		df.composeChart = make([]*composeChartFilter, 0, 1)
+	}
+	df.composeChart = append(df.composeChart, (*composeChartFilter)(f))
+	return df
+}