snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/discovery/rest/internal/documents/compose.go

@@ -0,0 +1,551 @@
+package documents
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+
+	cmpService "github.com/cortezaproject/corteza/server/compose/service"
+	cmpTypes "github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/discovery/service"
+	"github.com/cortezaproject/corteza/server/pkg/errors"
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/cortezaproject/corteza/server/pkg/id"
+	"github.com/cortezaproject/corteza/server/pkg/options"
+	"github.com/cortezaproject/corteza/server/pkg/rbac"
+	sysService "github.com/cortezaproject/corteza/server/system/service"
+	sysTypes "github.com/cortezaproject/corteza/server/system/types"
+)
+
+type (
+	composeResources struct {
+		opt      options.DiscoveryOpt
+		settings *sysTypes.AppSettings
+
+		rbac interface {
+			SignificantRoles(res rbac.Resource, op string) (aRR, dRR []uint64)
+		}
+
+		ac interface {
+			CanReadModule(ctx context.Context, r *cmpTypes.Module) bool
+			CanReadNamespace(ctx context.Context, r *cmpTypes.Namespace) bool
+			CanReadRecord(ctx context.Context, r *cmpTypes.Record) bool
+			CanReadRecordValueOnModuleField(ctx context.Context, r *cmpTypes.ModuleField) bool
+			CanReadChart(ctx context.Context, r *cmpTypes.Chart) bool
+			CanReadPage(ctx context.Context, r *cmpTypes.Page) bool
+		}
+
+		ns interface {
+			FindByID(context.Context, uint64) (*cmpTypes.Namespace, error)
+			Find(context.Context, cmpTypes.NamespaceFilter) (cmpTypes.NamespaceSet, cmpTypes.NamespaceFilter, error)
+		}
+
+		mod interface {
+			FindByID(context.Context, uint64, uint64) (*cmpTypes.Module, error)
+			Find(ctx context.Context, filter cmpTypes.ModuleFilter) (set cmpTypes.ModuleSet, f cmpTypes.ModuleFilter, err error)
+		}
+
+		rec interface {
+			Find(ctx context.Context, filter cmpTypes.RecordFilter) (set cmpTypes.RecordSet, f cmpTypes.RecordFilter, err error)
+		}
+
+		page interface {
+			Find(ctx context.Context, filter cmpTypes.PageFilter) (set cmpTypes.PageSet, f cmpTypes.PageFilter, err error)
+		}
+
+		usr interface {
+			FindByID(ctx context.Context, id uint64) (*sysTypes.User, error)
+		}
+	}
+
+	pageDetail struct {
+		namespaceSlug string
+		moduleID      uint64
+		pageID        uint64
+		recordID      uint64
+	}
+)
+
+var (
+	modulePages map[uint64]pageDetail
+)
+
+const (
+	maxValueLength = 512
+)
+
+func ComposeResources() *composeResources {
+	return &composeResources{
+		opt:      service.DefaultOption,
+		settings: sysService.CurrentSettings,
+		rbac:     rbac.Global(),
+		ac:       cmpService.DefaultAccessControl,
+		ns:       cmpService.DefaultNamespace,
+		mod:      cmpService.DefaultModule,
+		rec:      cmpService.DefaultRecord,
+		page:     cmpService.DefaultPage,
+		usr:      sysService.DefaultUser,
+	}
+}
+
+func (d composeResources) Namespaces(ctx context.Context, limit uint, cur string, namespaceID uint64, deleted uint) (rsp *Response, err error) {
+	return rsp, func() (err error) {
+		if !d.settings.Discovery.ComposeNamespaces.Enabled {
+			return errors.Internal("compose namespace indexing disabled")
+		}
+
+		var (
+			resourceType = "compose:namespace" // @todo use RBAC resourceType
+			ps           cmpTypes.PageSet
+			pf           cmpTypes.PageFilter
+			nss          cmpTypes.NamespaceSet
+			f            = cmpTypes.NamespaceFilter{
+				Deleted: filter.State(deleted),
+			}
+		)
+
+		if namespaceID > 0 {
+			f.NamespaceID = append(f.NamespaceID, id.String(namespaceID))
+		}
+
+		if f.Paging, err = filter.NewPaging(limit, cur); err != nil {
+			return err
+		}
+
+		if nss, f, err = d.ns.Find(ctx, f); err != nil {
+			return err
+		}
+
+		rsp = &Response{
+			Documents: make([]Document, len(nss)),
+			Filter: Filter{
+				Limit:    limit,
+				NextPage: f.NextPage,
+			},
+		}
+
+		for i, ns := range nss {
+			nsID := ns.ID
+			nsSlug := ns.Slug
+			rsp.Documents[i].ID = nsID
+			pf.NamespaceID = nsID
+
+			// namespace pages
+			if ps, pf, err = d.page.Find(ctx, pf); err != nil {
+				return err
+			}
+
+			if modulePages == nil {
+				modulePages = make(map[uint64]pageDetail)
+			}
+			for _, p := range ps {
+				moduleID := p.ModuleID
+				if len(nsSlug) > 0 && moduleID > 0 {
+					modulePages[moduleID] = pageDetail{
+						namespaceSlug: nsSlug,
+						moduleID:      moduleID,
+						pageID:        p.ID,
+					}
+				}
+			}
+
+			// where should this link to?
+			// namespace root page on the compose?
+			// rsp.Documents[i].URL = ""  // added to source
+			doc := &docComposeNamespace{
+				ResourceType: resourceType,
+				NamespaceID:  nsID,
+				Name:         ns.Name,
+				Handle:       nsSlug,
+				Url:          d.getUrlToResource(pageDetail{namespaceSlug: nsSlug}),
+				Meta: docPartialComposeNamespaceMeta{
+					Subtitle:    ns.Meta.Subtitle,
+					Description: ns.Meta.Description,
+				},
+				Created: makePartialChange(&ns.CreatedAt),
+				Updated: makePartialChange(ns.UpdatedAt),
+				Deleted: makePartialChange(ns.DeletedAt),
+
+				Namespace: docPartialComposeNamespace{
+					NamespaceID: nsID,
+					Name:        ns.Name,
+					Handle:      nsSlug,
+				},
+
+				CatchAll: []any{nsID, ns.Name, nsSlug},
+			}
+
+			allowedRoles, deniedRoles := d.rbac.SignificantRoles(ns, "read")
+
+			doc.Security = append(doc.Security, docSecurity{
+				AllowedRoles: stringifyUints64(allowedRoles),
+				DeniedRoles:  stringifyUints64(deniedRoles),
+			})
+
+			rsp.Documents[i].Source = doc
+		}
+
+		return nil
+	}()
+}
+
+func (d composeResources) Modules(ctx context.Context, namespaceID uint64, limit uint, cur string, moduleID uint64, deleted uint) (rsp *Response, err error) {
+	return rsp, func() (err error) {
+		if !d.settings.Discovery.ComposeModules.Enabled {
+			return errors.Internal("compose module indexing disabled")
+		}
+
+		var (
+			ns *cmpTypes.Namespace
+			mm cmpTypes.ModuleSet
+			f  = cmpTypes.ModuleFilter{
+				NamespaceID: namespaceID,
+				Deleted:     filter.State(deleted),
+			}
+		)
+
+		if moduleID > 0 {
+			f.ModuleID = append(f.ModuleID, id.String(moduleID))
+		}
+
+		if f.Paging, err = filter.NewPaging(limit, cur); err != nil {
+			return
+		}
+
+		if mm, f, err = d.mod.Find(ctx, f); err != nil {
+			return
+		}
+
+		rsp = &Response{
+			Documents: make([]Document, len(mm)),
+			Filter: Filter{
+				Limit:    limit,
+				NextPage: f.NextPage,
+			},
+		}
+
+		if ns, err = d.ns.FindByID(ctx, namespaceID); err != nil {
+			return
+		}
+
+		nsPartial := docPartialComposeNamespace{
+			NamespaceID: namespaceID,
+			Name:        ns.Name,
+			Handle:      ns.Slug,
+		}
+
+		for i, mod := range mm {
+			rsp.Documents[i].ID = mod.ID
+			// Where should this link to?
+			// module edit screen in the administration? does this make sense?
+			//rsp.Documents[i].URL = "@todo"
+			doc := &docComposeModule{
+				ResourceType: "compose:module", // @todo use RBAC resourceType
+				ModuleID:     mod.ID,
+				Name:         mod.Name,
+				Handle:       mod.Handle,
+				Fields: func() []*docPartialComposeModuleField {
+					out := make([]*docPartialComposeModuleField, len(mod.Fields))
+					for i, f := range mod.Fields {
+						out[i] = &docPartialComposeModuleField{
+							Name:  f.Name,
+							Label: f.Label,
+						}
+					}
+					return out
+				}(),
+				 Labels: func() map[string]string {
+					labels := make(map[string]string, len(mod.Labels))
+					for k, v := range mod.Labels {
+						if v.Val != "" {
+							labels[k] = v.Val
+						} else if len(v.Values) > 0 {
+							labels[k] = strings.Join(v.Values, ",")
+						}
+					}
+					return labels
+				}(),
+
+				Created: makePartialChange(&mod.CreatedAt),
+				Updated: makePartialChange(mod.UpdatedAt),
+				Deleted: makePartialChange(mod.DeletedAt),
+
+				CatchAll: []any{mod.ID, mod.Name, mod.Handle},
+
+				Namespace: nsPartial,
+				Module: docPartialComposeModule{
+					ModuleID: mod.ID,
+					Name:     mod.Name,
+					Handle:   mod.Handle,
+				},
+			}
+
+			for _, f := range mod.Fields {
+				if f.Name != "" {
+					doc.CatchAll = append(doc.CatchAll, f.Name)
+				}
+
+				if f.Label != "" {
+					doc.CatchAll = append(doc.CatchAll, f.Label)
+				}
+			}
+
+			allowedRoles, deniedRoles := d.rbac.SignificantRoles(mod, "read")
+
+			doc.Security = append(doc.Security, docSecurity{
+				AllowedRoles: stringifyUints64(allowedRoles),
+				DeniedRoles:  stringifyUints64(deniedRoles),
+			})
+
+			rsp.Documents[i].Source = doc
+		}
+
+		return nil
+	}()
+}
+
+func (d composeResources) Records(ctx context.Context, namespaceID, moduleID uint64, limit uint, cur string, recordID uint64, deleted uint) (rsp *Response, err error) {
+	return rsp, func() (err error) {
+		if !d.settings.Discovery.ComposeRecords.Enabled {
+			return errors.Internal("compose record indexing disabled")
+		}
+
+		var (
+			ns  *cmpTypes.Namespace
+			mod *cmpTypes.Module
+			rr  cmpTypes.RecordSet
+			f   = cmpTypes.RecordFilter{
+				NamespaceID: namespaceID,
+				ModuleID:    moduleID,
+				Deleted:     filter.State(deleted),
+			}
+		)
+
+		if recordID > 0 {
+			f.Query = fmt.Sprintf("recordID=%d", recordID)
+		}
+
+		if f.Paging, err = filter.NewPaging(limit, cur); err != nil {
+			return err
+		}
+
+		if rr, f, err = d.rec.Find(ctx, f); err != nil {
+			return err
+		}
+
+		rsp = &Response{
+			Documents: make([]Document, len(rr)),
+			Filter: Filter{
+				Limit:    limit,
+				NextPage: f.NextPage,
+			},
+		}
+
+		// @todo handle unreadable (access-control) namespaces
+		if ns, err = d.ns.FindByID(ctx, namespaceID); err != nil {
+			return
+		}
+
+		nsPartial := docPartialComposeNamespace{
+			NamespaceID: namespaceID,
+			Name:        ns.Name,
+			Handle:      ns.Slug,
+		}
+
+		// @todo handle unreadable (access-control) modules
+		if mod, err = d.mod.FindByID(ctx, namespaceID, moduleID); err != nil {
+			return
+		}
+
+		modPartial := docPartialComposeModule{
+			ModuleID: f.ModuleID,
+			Name:     mod.Name,
+			Handle:   mod.Handle,
+		}
+
+		users := make(map[uint64]sysTypes.User)
+		for i, rec := range rr {
+			recordID := rec.ID
+			rsp.Documents[i].ID = recordID
+			// where should this link to? record page in the compose?
+			// rsp.Documents[i].URL = "" // added to source
+
+			// fixme refactor me
+			var (
+				createdBy *sysTypes.User
+				updatedBy *sysTypes.User
+				deletedBy *sysTypes.User
+			)
+			if rec.CreatedBy > 0 {
+				createdBy, users, err = d.getUserInfo(ctx, rec.CreatedBy, users)
+				if err != nil {
+					return
+				}
+			}
+
+			if rec.UpdatedBy > 0 {
+				updatedBy, users, err = d.getUserInfo(ctx, rec.UpdatedBy, users)
+				if err != nil {
+					return
+				}
+			}
+
+			if rec.DeletedBy > 0 {
+				deletedBy, users, err = d.getUserInfo(ctx, rec.DeletedBy, users)
+				if err != nil {
+					return
+				}
+			}
+			// fixme refactor me END
+
+			doc := &docComposeRecord{
+				ResourceType: "compose:record", // @todo use RBAC resourceType
+				RecordID:     recordID,
+				Namespace:    nsPartial,
+				Module:       modPartial,
+				Url:          d.getUrlToResource(pageDetail{moduleID: moduleID, recordID: recordID}),
+				Meta:         rec.Meta,
+				//Values:       d.recordValues(ctx, rec, nil),
+				//Created:      makePartialChange(&rec.CreatedAt),
+				//Updated:      makePartialChange(rec.UpdatedAt),
+				//Deleted:      makePartialChange(rec.DeletedAt),
+
+				Created: makeChange(&rec.CreatedAt, createdBy),
+				Updated: makeChange(rec.UpdatedAt, updatedBy),
+				Deleted: makeChange(rec.DeletedAt, deletedBy),
+			}
+
+			// Values and value labels
+			doc.Values, doc.ValueLabels, doc.CatchAll = d.recordValues(ctx, rec, mod.Fields)
+
+			allowedRoles, deniedRoles := d.rbac.SignificantRoles(rec, "read")
+
+			doc.Security = append(doc.Security, docSecurity{
+				AllowedRoles: stringifyUints64(allowedRoles),
+				DeniedRoles:  stringifyUints64(deniedRoles),
+			})
+
+			rsp.Documents[i].Source = doc
+		}
+
+		return nil
+	}()
+}
+
+func (d composeResources) recordValues(ctx context.Context, rec *cmpTypes.Record, ff cmpTypes.ModuleFieldSet) (map[string][]interface{}, map[string]string, []any) {
+	var (
+		rval     = make(map[string][]interface{})
+		fields   = make(map[string]string)
+		catchAll = make([]any, 0, len(rec.Values))
+	)
+
+	if rec.GetModule() == nil {
+		return nil, nil, nil
+	}
+
+	_ = rec.GetModule().Fields.Walk(func(f *cmpTypes.ModuleField) error {
+		if !d.ac.CanReadRecordValueOnModuleField(ctx, f) {
+			return nil
+		}
+
+		var (
+			rv    = rec.Values.FilterByName(f.Name)
+			vv    = make([]interface{}, 0, len(rv))
+			field = ff.FindByName(f.Name)
+		)
+
+		if len(rv) == 0 {
+			return nil
+		}
+
+		for _, val := range rv {
+			// refs needs to be casted to string (json & unsigned 64-bit integers)!
+			if f.IsRef() {
+				aux := fmt.Sprintf("%d", val.Ref)
+				vv = append(vv, aux)
+				catchAll = append(catchAll, aux)
+				continue
+			}
+
+			if tmp, err := val.Cast(f); err == nil {
+				vv = append(vv, tmp)
+			}
+
+			catchAll = append(catchAll, clampString(val.Value, maxValueLength))
+		}
+
+		if len(vv) == 0 {
+			return nil
+		}
+
+		rval[f.Name] = vv
+		if field != nil {
+			fields[f.Name] = field.Label
+		}
+
+		return nil
+	})
+
+	return rval, fields, catchAll
+}
+
+// getUrlToResource construct page url for compose resources
+func (d composeResources) getUrlToResource(page pageDetail) (url string) {
+	var (
+		host          = d.opt.CortezaDomain
+		validNsSlung  = len(page.namespaceSlug) > 0
+		validModuleID = page.moduleID > 0
+		validRecord   = page.recordID > 0
+		validPageID   = page.pageID > 0
+	)
+
+	if len(host) == 0 {
+		return
+	}
+	if validModuleID && validRecord {
+		// For record, take moduleID & recordID from params(page) then construct url from modulePages map
+		if p, is := modulePages[page.moduleID]; is {
+			url = fmt.Sprintf("%s/compose/ns/%s/pages/%d/record/%d", host, p.namespaceSlug, p.pageID, page.recordID)
+		}
+	} else if validNsSlung && validPageID {
+		// for module? @todo, we need moduleID ref for module home page
+		// we can check page.self_id is 0 that is modules home pages, but ref_module is also 0
+		url = fmt.Sprintf("%s/compose/ns/%s/pages/%d", host, page.namespaceSlug, page.pageID)
+	} else if validNsSlung {
+		// For namespace, just send to /pages FE handles fallback to home(default) page
+		url = fmt.Sprintf("%s/compose/ns/%s/pages", host, page.namespaceSlug)
+	}
+	return
+}
+
+func (d composeResources) getUserInfo(ctx context.Context, ID uint64, users map[uint64]sysTypes.User) (*sysTypes.User, map[uint64]sysTypes.User, error) {
+	if users == nil {
+		users = make(map[uint64]sysTypes.User)
+	}
+	user, ok := users[ID]
+	if ok {
+		return &user, users, nil
+	} else {
+		u, err := d.usr.FindByID(ctx, ID)
+		if err != nil && u != nil {
+			users[ID] = *u
+		}
+		return u, users, err
+	}
+}
+
+func stringifyUints64(uints []uint64) []string {
+	strings := make([]string, len(uints))
+	for i, num := range uints {
+		strings[i] = strconv.FormatUint(num, 10)
+	}
+	return strings
+}
+
+func clampString(input string, maxLength int) string {
+	if len(input) > maxLength {
+		return input[:maxLength]
+	}
+	return input
+}

server/discovery/rest/internal/documents/documents.go

@@ -0,0 +1,172 @@
+package documents
+
+import (
+	"time"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	sysTypes "github.com/cortezaproject/corteza/server/system/types"
+)
+
+type (
+	Response struct {
+		Filter    Filter     `json:"filter"`
+		Documents []Document `json:"documents"`
+	}
+
+	Document struct {
+		ID     uint64      `json:"id,string"`
+		URL    string      `json:"url,omitempty"`
+		Source interface{} `json:"source"`
+	}
+
+	Filter struct {
+		Limit    uint                 `json:"limit"`
+		NextPage *filter.PagingCursor `json:"nextPage"`
+	}
+
+	docPartialUser struct {
+		UserID   uint64 `json:"userID,string"`
+		Email    string `json:"email,omitempty"`
+		Name     string `json:"name,omitempty"`
+		Username string `json:"username,omitempty"`
+		Handle   string `json:"handle,omitempty"`
+	}
+
+	docPartialChange struct {
+		At *time.Time      `json:"at,omitempty"`
+		By *docPartialUser `json:"by,omitempty"`
+	}
+
+	docUser struct {
+		ResourceType string            `json:"resourceType"`
+		UserID       uint64            `json:"userID,string"`
+		Email        string            `json:"email"`
+		Name         string            `json:"name,omitempty"`
+		Handle       string            `json:"handle,omitempty"`
+		Suspended    *time.Time        `json:"suspendedAt,omitempty"`
+		Url          string            `json:"url,omitempty"`
+		CatchAll     []any             `json:"catch_all"`
+		Updated      *docPartialChange `json:"updated,omitempty"`
+		Created      *docPartialChange `json:"created,omitempty"`
+		Deleted      *docPartialChange `json:"deleted,omitempty"`
+		Security     []docSecurity     `json:"security"`
+	}
+
+	docComposeNamespace struct {
+		ResourceType string                         `json:"resourceType"`
+		NamespaceID  uint64                         `json:"namespaceID,string"`
+		Name         string                         `json:"name,omitempty"`
+		Handle       string                         `json:"handle,omitempty"`
+		Url          string                         `json:"url,omitempty"`
+		Enabled      bool                           `json:"enabled"`
+		Meta         docPartialComposeNamespaceMeta `json:"meta"`
+		CatchAll     []interface{}                  `json:"catch_all"`
+
+		Updated  *docPartialChange `json:"updated,omitempty"`
+		Created  *docPartialChange `json:"created,omitempty"`
+		Deleted  *docPartialChange `json:"deleted,omitempty"`
+		Security []docSecurity     `json:"security"`
+
+		// Aggregation update
+		Namespace docPartialComposeNamespace `json:"namespace"`
+	}
+
+	docPartialComposeNamespaceMeta struct {
+		Subtitle    string `json:"subtitle,omitempty"`
+		Description string `json:"description,omitempty"`
+	}
+
+	docPartialComposeNamespace struct {
+		NamespaceID uint64 `json:"namespaceID,string"`
+		Name        string `json:"name,omitempty"`
+		Handle      string `json:"handle,omitempty"`
+	}
+
+	docComposeModule struct {
+		ResourceType string                          `json:"resourceType"`
+		ModuleID     uint64                          `json:"moduleID,string"`
+		Name         string                          `json:"name,omitempty"`
+		Handle       string                          `json:"handle,omitempty"`
+		Url          string                          `json:"url,omitempty"`
+		Labels       map[string]string               `json:"labels,omitempty"`
+		Fields       []*docPartialComposeModuleField `json:"fields"`
+		Updated      *docPartialChange               `json:"updated,omitempty"`
+		Created      *docPartialChange               `json:"created,omitempty"`
+		Deleted      *docPartialChange               `json:"deleted,omitempty"`
+		Security     []docSecurity                   `json:"security"`
+
+		// Aggregation update
+		Namespace docPartialComposeNamespace `json:"namespace"`
+		Module    docPartialComposeModule    `json:"module"`
+
+		CatchAll []any `json:"catch_all"`
+	}
+
+	docPartialComposeModuleField struct {
+		Name  string `json:"name,omitempty"`
+		Label string `json:"label,omitempty"`
+	}
+
+	docPartialComposeModule struct {
+		ModuleID uint64 `json:"moduleID,string"`
+		Name     string `json:"name,omitempty"`
+		Handle   string `json:"handle,omitempty"`
+	}
+
+	docComposeRecord struct {
+		ResourceType string                   `json:"resourceType"`
+		RecordID     uint64                   `json:"recordID,string"`
+		Url          string                   `json:"url,omitempty"`
+		Meta         map[string]any           `json:"labels,omitempty"`
+		ValueLabels  map[string]string        `json:"valueLabels,omitempty"`
+		Values       map[string][]interface{} `json:"values"`
+		CatchAll     []any                    `json:"catch_all"`
+		Updated      *docPartialChange        `json:"updated,omitempty"`
+		Created      *docPartialChange        `json:"created,omitempty"`
+		Deleted      *docPartialChange        `json:"deleted,omitempty"`
+		Security     []docSecurity            `json:"security"`
+
+		// Aggregation update
+		Namespace docPartialComposeNamespace `json:"namespace"`
+		Module    docPartialComposeModule    `json:"module"`
+
+		// Vectors content field
+		VectorsValue []float32 `json:"vectorsValue,omitempty"`
+	}
+
+	docSecurity struct {
+		// list of roles that are allowed to read the resource
+		AllowedRoles []string `json:"allowedRoles"`
+
+		// list of roles that are disallowed to read the resource
+		DeniedRoles []string `json:"deniedRoles"`
+	}
+)
+
+func makePartialChange(at *time.Time) *docPartialChange {
+	if at == nil {
+		return nil
+	}
+
+	// @todo handle unreadable (access-control) modules
+	// @todo attach security info (allow/deny roles)
+	return &docPartialChange{At: at}
+}
+
+func makeChange(at *time.Time, user *sysTypes.User) (out *docPartialChange) {
+	out = &docPartialChange{}
+	if at != nil {
+		out.At = at
+	}
+	if user != nil {
+		out.By = &docPartialUser{
+			UserID:   user.ID,
+			Email:    user.Email,
+			Name:     user.Name,
+			Username: user.Username,
+			Handle:   user.Handle,
+		}
+	}
+
+	return
+}

server/discovery/rest/internal/documents/system.go

@@ -0,0 +1,111 @@
+package documents
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cortezaproject/corteza/server/discovery/service"
+	"github.com/cortezaproject/corteza/server/pkg/errors"
+	"github.com/cortezaproject/corteza/server/pkg/id"
+	"github.com/cortezaproject/corteza/server/pkg/options"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/cortezaproject/corteza/server/pkg/rbac"
+	sysService "github.com/cortezaproject/corteza/server/system/service"
+	"github.com/cortezaproject/corteza/server/system/types"
+)
+
+type (
+	systemResources struct {
+		opt      options.DiscoveryOpt
+		settings *types.AppSettings
+
+		rbac interface {
+			SignificantRoles(res rbac.Resource, op string) (aRR, dRR []uint64)
+		}
+
+		ac interface {
+			CanReadUser(ctx context.Context, r *types.User) bool
+		}
+		usr interface {
+			Find(context.Context, types.UserFilter) (types.UserSet, types.UserFilter, error)
+		}
+	}
+)
+
+func SystemResources() *systemResources {
+	return &systemResources{
+		opt:      service.DefaultOption,
+		settings: sysService.CurrentSettings,
+		rbac:     rbac.Global(),
+		ac:       sysService.DefaultAccessControl,
+		usr:      sysService.DefaultUser,
+	}
+}
+
+func (d systemResources) Users(ctx context.Context, limit uint, cur string, userID uint64, deleted uint) (rsp *Response, err error) {
+	return rsp, func() (err error) {
+		if !d.settings.Discovery.SystemUsers.Enabled {
+			return errors.Internal("system user indexing disabled")
+		}
+
+		var (
+			uu types.UserSet
+			f  = types.UserFilter{
+				Deleted: filter.State(deleted),
+			}
+		)
+
+		if userID > 0 {
+			f.UserID = append(f.UserID, id.String(userID))
+		}
+
+		if f.Paging, err = filter.NewPaging(limit, cur); err != nil {
+			return err
+		}
+
+		if uu, f, err = d.usr.Find(ctx, f); err != nil {
+			return err
+		}
+
+		rsp = &Response{
+			Documents: make([]Document, len(uu)),
+			Filter: Filter{
+				Limit:    limit,
+				NextPage: f.NextPage,
+			},
+		}
+
+		for i, u := range uu {
+			doc := &docUser{
+				ResourceType: "system:user",
+				UserID:       u.ID,
+				Email:        u.Email,
+				Name:         u.Name,
+				Handle:       u.Handle,
+				Suspended:    u.SuspendedAt,
+
+				CatchAll: []any{u.ID, u.Email, u.Name, u.Handle},
+
+				Created: makePartialChange(&u.CreatedAt),
+				Updated: makePartialChange(u.UpdatedAt),
+				Deleted: makePartialChange(u.DeletedAt),
+			}
+			if len(d.opt.CortezaDomain) > 0 && u.ID > 0 {
+				doc.Url = fmt.Sprintf("%s/admin/system/user/edit/%d", d.opt.CortezaDomain, u.ID)
+			}
+
+			allowedRoles, deniedRoles := d.rbac.SignificantRoles(u, "read")
+
+			doc.Security = append(doc.Security, docSecurity{
+				AllowedRoles: stringifyUints64(allowedRoles),
+				DeniedRoles:  stringifyUints64(deniedRoles),
+			})
+
+			rsp.Documents[i].ID = u.ID
+			rsp.Documents[i].Source = doc
+		}
+
+		return nil
+	}()
+}

server/discovery/rest/internal/feed/feed.go

@@ -0,0 +1,28 @@
+package feed
+
+import (
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/jmoiron/sqlx/types"
+	"time"
+)
+
+type (
+	Response struct {
+		Filter       Filter        `json:"filter"`
+		ActivityLogs []ActivityLog `json:"activityLogs"`
+	}
+
+	ActivityLog struct {
+		ID             uint64         `json:"activityID,string"`
+		ResourceID     uint64         `json:"resourceID,string"`
+		ResourceType   string         `json:"resourceType"`
+		ResourceAction string         `json:"resourceAction"`
+		Timestamp      time.Time      `json:"timestamp"`
+		Meta           types.JSONText `json:"meta"`
+	}
+
+	Filter struct {
+		Limit    uint                 `json:"limit"`
+		NextPage *filter.PagingCursor `json:"nextPage"`
+	}
+)

server/discovery/rest/internal/feed/resource.go

@@ -0,0 +1,105 @@
+package feed
+
+import (
+	"context"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/discovery/service"
+	"github.com/cortezaproject/corteza/server/discovery/types"
+	"github.com/cortezaproject/corteza/server/pkg/errors"
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/cortezaproject/corteza/server/pkg/options"
+
+	"github.com/cortezaproject/corteza/server/pkg/rbac"
+)
+
+type (
+	resourceActivity struct {
+		opt options.DiscoveryOpt
+
+		rbac interface {
+			SignificantRoles(res rbac.Resource, op string) (aRR, dRR []uint64)
+		}
+
+		ac interface {
+		}
+
+		resActivity service.ResourceActivityService
+	}
+)
+
+func ResourceActivity() *resourceActivity {
+	return &resourceActivity{
+		opt:         service.DefaultOption,
+		rbac:        rbac.Global(),
+		resActivity: service.DefaultResourceActivity,
+	}
+}
+
+func (a resourceActivity) ResourceActivities(ctx context.Context, limit uint, cur string, from *time.Time, to *time.Time) (rsp *Response, err error) {
+	return rsp, func() (err error) {
+		var (
+			aa types.ResourceActivitySet
+
+			f = types.ResourceActivityFilter{}
+		)
+
+		if from == nil || from.IsZero() {
+			return errors.Internal("invalid or missing from timestamp")
+		}
+
+		if to == nil || to.IsZero() {
+			now := time.Now()
+			to = &now
+		}
+
+		if from.After(*to) {
+			return errors.Internal("invalid from timestamp, it must be before to timestamp")
+		}
+
+		// Ensure both 'from' and 'to' are in UTC
+		utcFrom := from.UTC()
+		utcTo := to.UTC()
+
+		from = &utcFrom
+		to = &utcTo
+
+		f.FromTimestamp = from
+		f.ToTimestamp = to
+
+		if f.Paging, err = filter.NewPaging(limit, cur); err != nil {
+			return err
+		}
+
+		if aa, f, err = a.resActivity.Find(ctx, f); err != nil {
+			return err
+		}
+
+		rsp = &Response{
+			ActivityLogs: make([]ActivityLog, 0),
+			Filter: Filter{
+				Limit:    limit,
+				NextPage: f.NextPage,
+			},
+		}
+
+		err = aa.Walk(func(a *types.ResourceActivity) error {
+			rsp.ActivityLogs = append(rsp.ActivityLogs, ActivityLog{
+				ID:             a.ID,
+				ResourceID:     a.ResourceID,
+				ResourceType:   a.ResourceType,
+				ResourceAction: a.ResourceAction,
+				Timestamp:      a.Timestamp,
+				Meta:           a.Meta,
+			})
+
+			return nil
+		})
+
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}()
+}

server/discovery/rest/internal/mapping/compose.go

@@ -0,0 +1,324 @@
+package mapping
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/compose/service"
+	"github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/pkg/options"
+)
+
+type (
+	namespaceSearcher interface {
+		Find(ctx context.Context, filter types.NamespaceFilter) (types.NamespaceSet, types.NamespaceFilter, error)
+	}
+
+	moduleSearcher interface {
+		Find(ctx context.Context, filter types.ModuleFilter) (types.ModuleSet, types.ModuleFilter, error)
+	}
+
+	composeAccessControl interface {
+		CanReadRecordValueOnModuleField(ctx context.Context, r *types.ModuleField) bool
+		CanSearchRecordsOnModule(ctx context.Context, r *types.Module) bool
+	}
+
+	composeMapping struct {
+		ac   composeAccessControl
+		ns   namespaceSearcher
+		mod  moduleSearcher
+		opts options.DiscoveryOpt
+	}
+)
+
+func ComposeMapping(discoveryOpts options.DiscoveryOpt) *composeMapping {
+	return &composeMapping{
+		ac:   service.DefaultAccessControl,
+		ns:   service.DefaultNamespace,
+		mod:  service.DefaultModule,
+		opts: discoveryOpts,
+	}
+}
+
+func (m composeMapping) Namespaces(ctx context.Context) ([]*Mapping, error) {
+	return []*Mapping{&Mapping{
+		Index: "compose-namespaces",
+		Mapping: map[string]*property{
+			"resourceType": {Type: "keyword"},
+
+			"namespaceID": {Type: "long"},
+
+			"name":    {Type: "keyword", Boost: 2},
+			"handle":  {Type: "keyword", Boost: 2},
+			"labels":  {Type: "keyword"},
+			"enabled": {Type: "boolean"},
+			"url":     {Type: "text"},
+
+			"meta": {Type: "object", Properties: map[string]*property{
+				"subtitle":    {Type: "keyword"},
+				"description": {Type: "text"},
+			}},
+
+			"created": change(),
+			"updated": change(),
+			"deleted": change(),
+
+			"security": security(),
+
+			"namespace": {
+				Type: "nested",
+				Properties: map[string]*property{
+					"namespaceID": {Type: "long"},
+					"name":        {Type: "text"},
+					"handle":      {Type: "keyword"},
+				},
+			},
+
+			"catch_all": {
+				Type:     "text",
+				Analyzer: "standard",
+			},
+		},
+	}}, nil
+}
+
+func (m composeMapping) Modules(ctx context.Context) ([]*Mapping, error) {
+	return []*Mapping{&Mapping{
+		Index: fmt.Sprintf("compose-modules"),
+		Mapping: map[string]*property{
+			"resourceType": {Type: "keyword"},
+
+			"moduleID": {Type: "long"},
+			"created":  change(),
+			"updated":  change(),
+			"deleted":  change(),
+
+			"security": security(),
+
+			"name":   {Type: "text", Boost: 2},
+			"handle": {Type: "keyword", Boost: 2},
+			"url":    {Type: "text"},
+			"labels": {
+				Type: "object",
+			},
+			"fields": {
+				Type: "object",
+				Properties: map[string]*property{
+					"name":  {Type: "keyword"},
+					"label": {Type: "text"},
+				},
+			},
+
+			"namespace": {
+				Type: "nested",
+				Properties: map[string]*property{
+					"namespaceID": {Type: "long"},
+					"name":        {Type: "text"},
+					"handle":      {Type: "keyword"},
+				},
+			},
+			"module": {
+				Type: "nested",
+				Properties: map[string]*property{
+					"moduleID": {Type: "long"},
+					"name":     {Type: "text"},
+					"handle":   {Type: "keyword"},
+				},
+			},
+
+			"catch_all": {
+				Type:     "text",
+				Analyzer: "standard",
+			},
+		},
+	}}, nil
+}
+
+func (m composeMapping) Records(ctx context.Context) ([]*Mapping, error) {
+	var (
+		nn, _, err = m.ns.Find(ctx, types.NamespaceFilter{})
+		out        = make([]*Mapping, 0, len(nn)*50)
+	)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return out, nn.Walk(func(ns *types.Namespace) (err error) {
+		// @todo do we index records in this namespace?
+		var (
+			mm types.ModuleSet
+		)
+
+		mm, _, err = m.mod.Find(ctx, types.ModuleFilter{NamespaceID: ns.ID})
+		if err != nil {
+			return err
+		}
+
+		// collect record mappings for all modules
+		return mm.Walk(func(mod *types.Module) error {
+			// @todo do we index records of module?
+
+			if m := m.records(ctx, mod, mm); m != nil {
+				out = append(out, m)
+			}
+
+			return nil
+		})
+	})
+}
+
+// Module mapping is always returned, even if it contains no properties
+//
+// If there are no properties indexer can assume
+func (m composeMapping) records(ctx context.Context, mod *types.Module, mm types.ModuleSet) (mapping *Mapping) {
+	//	@todo check if module can be Mappinged
+
+	mapping = &Mapping{
+		Index:   fmt.Sprintf("compose-records-%d-%d", mod.NamespaceID, mod.ID),
+		Mapping: make(map[string]*property),
+	}
+
+	props := m.moduleFieldsMappingProperties(ctx, mod, mm)
+	if len(props) == 0 {
+		// no mapping
+		return
+	}
+	mapping.Mapping["resourceType"] = &property{Type: "keyword"}
+
+	mapping.Mapping["recordID"] = &property{Type: "long"}
+	mapping.Mapping["url"] = &property{Type: "text"}
+	mapping.Mapping["created"] = change()
+	mapping.Mapping["updated"] = change()
+	mapping.Mapping["deleted"] = change()
+	mapping.Mapping["security"] = security()
+
+	mapping.Mapping["meta"] = &property{
+		Type: "object",
+	}
+	mapping.Mapping["values"] = &property{
+		Type:       "nested",
+		Properties: props,
+	}
+
+	mapping.Mapping["catch_all"] = &property{
+		Type:     "text",
+		Analyzer: "standard",
+	}
+
+	// mapping for record is there, add fields for namespace & module
+	mapping.Mapping["module"] = &property{
+		Type: "nested",
+		Properties: map[string]*property{
+			"moduleID": {Type: "long"},
+			"name":     {Type: "text"},
+			"handle":   {Type: "keyword"},
+		},
+	}
+	mapping.Mapping["namespace"] = &property{
+		Type: "nested",
+		Properties: map[string]*property{
+			"namespaceID": {Type: "long"},
+			"name":        {Type: "text"},
+			"handle":      {Type: "keyword"},
+		},
+	}
+
+	if m.opts.EmbeddingsEnabled {
+		mapping.Mapping["vectorsValue"] = vector(m.opts)
+	}
+
+	return
+}
+
+func (m composeMapping) moduleFieldsMappingProperties(ctx context.Context, mod *types.Module, mm types.ModuleSet) map[string]*property {
+	mf := make(map[string]*property)
+
+	for _, f := range mod.Fields {
+		for k, m := range m.moduleFieldMappingProperties(ctx, f, mm) {
+			mf[k] = m
+		}
+	}
+
+	return mf
+}
+
+func (m composeMapping) moduleFieldMappingProperties(ctx context.Context, f *types.ModuleField, mm types.ModuleSet) map[string]*property {
+	if !m.ac.CanReadRecordValueOnModuleField(ctx, f) {
+		return nil
+	}
+
+	var (
+		p    = &property{Type: "text"}
+		name = f.Name
+	)
+	// @todo get indexing props from f.Options!
+
+	if p.Type == "" {
+		// Type not explicitly set, get it by converting
+		switch strings.ToLower(f.Kind) {
+		case "bool":
+			p.Type = "boolean"
+
+		case "datetime":
+			p.Type = "date"
+		// https://www.elastic.co/guide/en/elasticsearch/reference/current/date.html
+		// @todo format param!
+
+		case "email":
+			p.Type = "keyword"
+
+		case "file":
+		// @todo ref
+
+		case "number":
+			// @todo what number type to use?
+			return nil
+
+		case "record":
+			//iopt := f.Options.Index()
+
+			// @todo make sure we do not fall into recursive loop
+
+			// if ifc.parents[]
+
+			// @todo if single-value then object
+			// https://www.elastic.co/guide/en/elasticsearch/reference/current/object.html
+			// @todo if multi-value then nested
+			// https://www.elastic.co/guide/en/elasticsearch/reference/current/nested.html
+
+			//var deepDive = 1
+			//
+			//if deepDive == 0 {
+			//	// no
+			//	return nil
+			//}
+			//
+			//m := mm.FindByID(f.Options.Uint64("moduleID"))
+			//if m == nil {
+			//	return nil
+			//}
+			//
+			//if opt.nestingLevel < 2 {
+			//	return append(composeModuleFieldsMappingProperties(m, mm, Context{prefix: f.Name, nestingLevel: opt.nestingLevel + 1}), m)
+			//}
+			return nil
+
+		case "select":
+			p.Type = "keyword"
+
+		case "string":
+			p.Type = "text"
+
+		case "url":
+			p.Type = "keyword"
+
+		case "user":
+			// @todo ref
+			return nil
+		}
+	}
+
+	return map[string]*property{name: p}
+}

server/discovery/rest/internal/mapping/mapping.go

@@ -0,0 +1,70 @@
+package mapping
+
+import "github.com/cortezaproject/corteza/server/pkg/options"
+
+type (
+	Mapping struct {
+		Index   string               `json:"index"`
+		Mapping map[string]*property `json:"mapping"`
+	}
+
+	property struct {
+		// https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html
+		Type string `json:"type,omitempty"`
+
+		// Boost factor
+		// https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-boost.html
+		Boost float32 `json:"boost,omitempty"`
+
+		Analyzer string `json:"analyzer,omitempty"`
+
+		Properties map[string]*property `json:"properties,omitempty"`
+
+		// for vector indexing
+		Dimension int            `json:"dimension,omitempty"`
+		Method    map[string]any `json:"method,omitempty"`
+	}
+
+	Context struct {
+		AccessRestriction string
+	}
+)
+
+func change() *property {
+	return &property{
+		Type: "nested",
+		Properties: map[string]*property{
+			"at": {Type: "date"},
+			"by": {Type: "nested", Properties: map[string]*property{
+				"id":     {Type: "long"},
+				"email":  {Type: "keyword"},
+				"name":   {Type: "keyword"},
+				"handle": {Type: "keyword"},
+			}},
+		},
+	}
+}
+
+func security() *property {
+	return &property{
+		Properties: map[string]*property{
+			"allowedRoles": {Type: "long"},
+			"deniedRoles":  {Type: "long"},
+		},
+	}
+}
+
+func vector(opts options.DiscoveryOpt) *property {
+	return &property{
+		Type:      "knn_vector",
+		Dimension: opts.EmbeddingsDimension,
+		Method: map[string]any{
+			"name":   "hnsw",
+			"engine": "lucene",
+			"parameters": map[string]any{
+				"ef_construction": opts.HnswEfConstruction,
+				"m":               opts.HnswM,
+			},
+		},
+	}
+}

server/discovery/rest/internal/mapping/system.go

@@ -0,0 +1,54 @@
+package mapping
+
+import (
+	"context"
+
+	"github.com/cortezaproject/corteza/server/system/service"
+)
+
+type (
+	systemAccessControl interface {
+		CanSearchUsers(ctx context.Context) bool
+		CanSearchRoles(ctx context.Context) bool
+		CanSearchApplications(ctx context.Context) bool
+		CanSearchAuthClients(ctx context.Context) bool
+	}
+
+	systemMapping struct {
+		ac systemAccessControl
+	}
+)
+
+func SystemMapping() *systemMapping {
+	return &systemMapping{
+		ac: service.DefaultAccessControl,
+	}
+}
+
+func (svc systemMapping) Users(_ context.Context) ([]*Mapping, error) {
+	return []*Mapping{{
+		Index: "system-users",
+		Mapping: map[string]*property{
+			"resourceType": {Type: "keyword"},
+
+			"userID": {Type: "long"},
+
+			"email":  {Type: "keyword", Boost: 2},
+			"name":   {Type: "keyword", Boost: 10},
+			"handle": {Type: "keyword", Boost: 2},
+
+			"created": change(),
+			"updated": change(),
+			"deleted": change(),
+
+			"suspendedAt": {Type: "date"},
+
+			"security": security(),
+
+			"catch_all": {
+				Type:     "text",
+				Analyzer: "standard",
+			},
+		},
+	}}, nil
+}