package store import ( "context" "strconv" "strings" "github.com/cortezaproject/corteza/server/compose/dalutils" "github.com/cortezaproject/corteza/server/compose/service" "github.com/cortezaproject/corteza/server/compose/types" "github.com/cortezaproject/corteza/server/pkg/dal" "github.com/cortezaproject/corteza/server/pkg/envoy" "github.com/cortezaproject/corteza/server/pkg/envoy/resource" "github.com/cortezaproject/corteza/server/pkg/filter" "github.com/cortezaproject/corteza/server/store" stypes "github.com/cortezaproject/corteza/server/system/types" ) type ( composeNamespaceFilter types.NamespaceFilter composeModuleFilter types.ModuleFilter composeRecordFilter types.RecordFilter composePageFilter types.PageFilter composeChartFilter types.ChartFilter composeStore interface { store.ComposeAttachments store.ComposeCharts store.ComposeModuleFields store.ComposeModules store.ComposeNamespaces store.ComposePages } dalService interface { Create(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, vv ...dal.ValueGetter) error Search(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, f filter.Filter) (dal.Iterator, error) Delete(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, pkv ...dal.ValueGetter) (err error) Update(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, pkv ...dal.ValueGetter) (err error) ReplaceModel(ctx context.Context, currentAlts []*dal.Alteration, model *dal.Model) (newAlts []*dal.Alteration, err error) GetConnectionByID(uint64) *dal.ConnectionWrap } composeDecoder struct { resourceID []uint64 namespaceID []uint64 dal dalService } ) func newComposeDecoder(dal dalService) *composeDecoder { return &composeDecoder{ resourceID: make([]uint64, 0, 200), namespaceID: make([]uint64, 0, 200), dal: dal, } } func (d *composeDecoder) decodeComposeNamespace(ctx context.Context, s composeStore, ff []*composeNamespaceFilter) *auxRsp { mm := make([]envoy.Marshaller, 0, 100) if ff == nil { return &auxRsp{ mm: mm, } } var nn types.NamespaceSet var fn types.NamespaceFilter var err error for _, f := range ff { aux := *f if aux.Limit == 0 { aux.Limit = 1000 } for { nn, fn, err = s.SearchComposeNamespaces(ctx, types.NamespaceFilter(aux)) if err != nil { return &auxRsp{ err: err, } } for _, n := range nn { d.namespaceID = append(d.namespaceID, n.ID) d.resourceID = append(d.resourceID, n.ID) mm = append(mm, newComposeNamespace(n)) } if f.NextPage != nil { aux.PageCursor = fn.NextPage } else { break } } } return &auxRsp{ mm: mm, } } func (d *composeDecoder) decodeComposeModule(ctx context.Context, s composeStore, ff []*composeModuleFilter) *auxRsp { mm := make([]envoy.Marshaller, 0, 100) if ff == nil { return &auxRsp{ mm: mm, } } if len(d.namespaceID) > 0 { ffNs := make([]*composeModuleFilter, 0, len(ff)+len(d.namespaceID)) for _, nsID := range d.namespaceID { for _, f := range ff { fNs := *f fNs.NamespaceID = nsID ffNs = append(ffNs, &fNs) } } ff = ffNs } var nn types.ModuleSet var fn types.ModuleFilter var err error for _, f := range ff { aux := *f if aux.Limit == 0 { aux.Limit = 1000 } for { nn, fn, err = s.SearchComposeModules(ctx, types.ModuleFilter(aux)) if err != nil { return &auxRsp{ err: err, } } for _, n := range nn { d.resourceID = append(d.resourceID, n.ID) n.Fields, _, err = s.SearchComposeModuleFields(ctx, types.ModuleFieldFilter{ ModuleID: []uint64{n.ID}, }) if err != nil { return &auxRsp{ err: err, } } mm = append(mm, newComposeModule(n)) } if f.NextPage != nil { aux.PageCursor = fn.NextPage } else { break } } } return &auxRsp{ mm: mm, } } func (d *composeDecoder) decodeComposeRecord(ctx context.Context, s store.Storer, ff []*composeRecordFilter) *auxRsp { mm := make([]envoy.Marshaller, 0, 100) if ff == nil { return &auxRsp{ mm: mm, } } ac := service.AccessControl(s) if len(d.namespaceID) > 0 { ffNs := make([]*composeRecordFilter, 0, len(ff)+len(d.namespaceID)) for _, nsID := range d.namespaceID { for _, f := range ff { nn, _, err := s.SearchComposeModules(ctx, types.ModuleFilter{ NamespaceID: nsID, }) if err != nil { continue } for _, n := range nn { fNs := *f fNs.NamespaceID = nsID fNs.ModuleID = n.ID ffNs = append(ffNs, &fNs) } } } ff = ffNs } // When decoding large amounts of records (milions) we can probably assume // that each system user exists somewhere in the record set. // // That said, it's probably cheeper to simply say that we probably need all of the // references to hold. // If a user is not there in the preparation step, we shouldn't fail straight away. // // @todo use some heuristic to determine when to just list all of the users // and when to preprocess the records to find out what users to get. relUsers := make(resource.UserstampIndex) uu, _, err := store.SearchUsers(ctx, s, stypes.UserFilter{ Paging: filter.Paging{ Limit: 0, }, }) if err != nil { return &auxRsp{ err: err, } } relUsers.Add(uu...) mapValues := func(r *types.Record) map[string]resource.ComposeRecordRawValue { mod := r.GetModule() out := make(map[string]resource.ComposeRecordRawValue, len(r.Values)) for _, v := range r.Values { f := mod.Fields.FindByName(v.Name) if f == nil || !f.Multi { out[v.Name] = resource.ComposeRecordRawValue{ Name: v.Name, Value: v.Value, } } else { if _, ok := out[v.Name]; !ok { out[v.Name] = resource.ComposeRecordRawValue{ Name: v.Name, IsMulti: true, } } vx := out[v.Name] vx.Values = append(vx.Values, v.Value) out[v.Name] = vx } } return out } // Prepare a series of resource.ComposeRecord instances; one for each provided filter for _, f := range ff { aux := *f if aux.Limit == 0 { aux.Limit = 1000 } mod, err := store.LookupComposeModuleByID(ctx, s, f.ModuleID) if err != nil { return &auxRsp{ err: err, } } ff, _, err := store.SearchComposeModuleFields(ctx, s, types.ModuleFieldFilter{ModuleID: []uint64{mod.ID}}) if err != nil { return &auxRsp{ err: err, } } _ = ff.Walk(func(f *types.ModuleField) error { f.NamespaceID = mod.NamespaceID return nil }) mod.Fields = ff aux.Check = service.ComposeRecordFilterChecker(ctx, ac, mod) // Refs auxRecord := &composeRecordAux{ refMod: strconv.FormatUint(f.ModuleID, 10), relMod: mod, refNs: strconv.FormatUint(f.NamespaceID, 10), relUsers: relUsers, } // Walker auxRecord.walker = func(cb func(r *resource.ComposeRecordRaw) error) error { var rr types.RecordSet var fn types.RecordFilter var err error for { rr, fn, err = dalutils.ComposeRecordsList(ctx, d.dal, mod, types.RecordFilter(aux)) if err != nil { return err } service.ComposeRecordFilterAC(ctx, ac, mod, rr...) for _, n := range rr { // Create a raw record r := &resource.ComposeRecordRaw{ ID: strconv.FormatUint(n.ID, 10), Values: mapValues(n), Ts: resource.MakeTimestampsCUDA(&n.CreatedAt, n.UpdatedAt, n.DeletedAt, nil), Us: resource.MakeUserstampsCUDO(n.CreatedBy, n.UpdatedBy, n.DeletedBy, n.OwnedBy), } err = cb(r) if err != nil { return err } } if fn.NextPage != nil { aux.PageCursor = fn.NextPage } else { break } } return nil } mm = append(mm, newComposeRecordFromAux(auxRecord)) } return &auxRsp{ mm: mm, } } func (d *composeDecoder) decodeComposePage(ctx context.Context, s composeStore, ff []*composePageFilter) *auxRsp { mm := make([]envoy.Marshaller, 0, 100) if ff == nil { return &auxRsp{ mm: mm, } } if len(d.namespaceID) > 0 { ffNs := make([]*composePageFilter, 0, len(ff)+len(d.namespaceID)) for _, nsID := range d.namespaceID { for _, f := range ff { fNs := *f fNs.NamespaceID = nsID ffNs = append(ffNs, &fNs) } } ff = ffNs } var nn types.PageSet var fn types.PageFilter var err error for _, f := range ff { aux := *f if aux.Limit == 0 { aux.Limit = 1000 } for { nn, fn, err = s.SearchComposePages(ctx, types.PageFilter(aux)) if err != nil { return &auxRsp{ err: err, } } for _, n := range nn { d.resourceID = append(d.resourceID, n.ID) mm = append(mm, newComposePage(n)) } if fn.NextPage != nil { aux.PageCursor = fn.NextPage } else { break } } } return &auxRsp{ mm: mm, } } func (d *composeDecoder) decodeComposeChart(ctx context.Context, s composeStore, ff []*composeChartFilter) *auxRsp { mm := make([]envoy.Marshaller, 0, 100) if ff == nil { return &auxRsp{ mm: mm, } } if len(d.namespaceID) > 0 { ffNs := make([]*composeChartFilter, 0, len(ff)+len(d.namespaceID)) for _, nsID := range d.namespaceID { for _, f := range ff { fNs := *f fNs.NamespaceID = nsID ffNs = append(ffNs, &fNs) } } ff = ffNs } var nn types.ChartSet var fn types.ChartFilter var err error for _, f := range ff { aux := *f if aux.Limit == 0 { aux.Limit = 1000 } for { nn, fn, err = s.SearchComposeCharts(ctx, types.ChartFilter(aux)) if err != nil { return &auxRsp{ err: err, } } for _, n := range nn { d.resourceID = append(d.resourceID, n.ID) mm = append(mm, newComposeChart(n)) } if f.Limit > 0 { break } else if fn.NextPage != nil { aux.PageCursor = fn.NextPage } else { break } } } return &auxRsp{ mm: mm, } } func (df *DecodeFilter) composeFromResource(rr ...string) *DecodeFilter { for _, r := range rr { if !strings.HasPrefix(r, "compose") { continue } id := "" if strings.Count(r, ":") == 2 && !strings.HasSuffix(r, "*") { // There is an identifier aux := strings.Split(r, ":") id = aux[len(aux)-1] r = strings.Join(aux[:len(aux)-1], ":") } switch strings.ToLower(r) { case "compose:namespace": df = df.ComposeNamespace(&types.NamespaceFilter{ Query: id, }) case "compose:module": df = df.ComposeModule(&types.ModuleFilter{ Query: id, }) case "compose:record": df = df.ComposeRecord(&types.RecordFilter{ Query: id, }) case "compose:page": df = df.ComposePage(&types.PageFilter{ Query: id, }) case "compose:chart": df = df.ComposeChart(&types.ChartFilter{ Query: id, }) } } return df } func (df *DecodeFilter) composeFromRef(rr ...*resource.Ref) *DecodeFilter { for _, r := range rr { if strings.Index(r.ResourceType, "compose") < 0 { continue } switch r.ResourceType { case types.NamespaceResourceType: for _, i := range r.Identifiers.StringSlice() { df = df.ComposeNamespace(&types.NamespaceFilter{ Query: i, }) } case types.ModuleResourceType: for _, i := range r.Identifiers.StringSlice() { df = df.ComposeModule(&types.ModuleFilter{ Query: i, }) } case types.PageResourceType: for _, i := range r.Identifiers.StringSlice() { df = df.ComposePage(&types.PageFilter{ Query: i, }) } case types.ChartResourceType: for _, i := range r.Identifiers.StringSlice() { df = df.ComposeChart(&types.ChartFilter{ Query: i, }) } } } return df } // ComposeNamespace adds a new compose NamespaceFilter func (df *DecodeFilter) ComposeNamespace(f *types.NamespaceFilter) *DecodeFilter { if df.composeNamespace == nil { df.composeNamespace = make([]*composeNamespaceFilter, 0, 1) } df.composeNamespace = append(df.composeNamespace, (*composeNamespaceFilter)(f)) return df } // ComposeModule adds a new compose ModuleFilter func (df *DecodeFilter) ComposeModule(f *types.ModuleFilter) *DecodeFilter { if df.composeModule == nil { df.composeModule = make([]*composeModuleFilter, 0, 1) } df.composeModule = append(df.composeModule, (*composeModuleFilter)(f)) return df } // ComposeRecord adds a new compose RecordFilter func (df *DecodeFilter) ComposeRecord(f *types.RecordFilter) *DecodeFilter { if df.composeRecord == nil { df.composeRecord = make([]*composeRecordFilter, 0, 1) } df.composeRecord = append(df.composeRecord, (*composeRecordFilter)(f)) return df } // ComposePage adds a new compose PageFilter func (df *DecodeFilter) ComposePage(f *types.PageFilter) *DecodeFilter { if df.composePage == nil { df.composePage = make([]*composePageFilter, 0, 1) } df.composePage = append(df.composePage, (*composePageFilter)(f)) return df } // ComposeChart adds a new compose ChartFilter func (df *DecodeFilter) ComposeChart(f *types.ChartFilter) *DecodeFilter { if df.composeChart == nil { df.composeChart = make([]*composeChartFilter, 0, 1) } df.composeChart = append(df.composeChart, (*composeChartFilter)(f)) return df }