snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

extra/server-discovery/searcher/rest/conv.go

@@ -0,0 +1,446 @@
+package rest
+
+import (
+	"encoding/json"
+	"fmt"
+	"html"
+	"reflect"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/microcosm-cc/bluemonday"
+	"github.com/spf13/cast"
+)
+
+type (
+	// corteza discovery results
+	cdResults struct {
+		Total struct {
+			Value   int    `json:"value"`
+			TotalOp string `json:"op"`
+		} `json:"-"`
+
+		Hits         []cdHit         `json:"hits"`
+		TotalHits    int             `json:"total_hits"`
+		Aggregations []cdAggregation `json:"aggregations"`
+
+		pagination
+
+		// Context ldCtx `json:"@context"`
+	}
+
+	pagination struct {
+		Size         int `json:"size"`
+		From         int `json:"from"`
+		TotalResults int `json:"total_results"`
+	}
+
+	cdHit struct {
+		Type  string                 `json:"type"`
+		Value map[string]interface{} `json:"value"`
+	}
+
+	cdAggregation struct {
+		Resource     string              `json:"resource"`
+		Name         string              `json:"name"`
+		Hits         int                 `json:"hits"`
+		ResourceName []cdAggregationHits `json:"resource_name"`
+	}
+
+	cdAggregationHits struct {
+		Name  string `json:"name"`
+		Label string `json:"label"`
+		Hits  int    `json:"hits"`
+	}
+	// ldCtx map[string]interface{}
+
+	createdBy struct {
+		UserID   uint64 `json:"userID,string"`
+		Email    string `json:"email,omitempty"`
+		Name     string `json:"name,omitempty"`
+		Username string `json:"username,omitempty"`
+		Handle   string `json:"handle,omitempty"`
+	}
+)
+
+// conv converts results from the backend into corteza-discovery (jsonld-ish) format
+func conv(sr *esSearchResponse, aggregation *esSearchResponse, noHits bool, moduleMeta map[string][]string, nsHandleMap map[string]nsMeta, mHandleMap map[string]mMeta, page pagination) (out *cdResults, err error) {
+	if sr == nil {
+		return
+	}
+
+	out = &cdResults{}
+	out.Total.Value = sr.Hits.Total.Value
+	out.Total.TotalOp = sr.Hits.Total.Relation
+
+	// Pagination
+	out.TotalResults = sr.Hits.Total.Value
+	out.From = page.From
+	out.Size = page.Size
+
+	// Aggregation
+	out.Aggregations = []cdAggregation{}
+
+	nsTotalHits := make(map[string]cdAggregationHits)
+	mTotalHits := make(map[string]cdAggregationHits)
+
+	aggsRes := sr.Aggregations
+	if aggregation != nil {
+		aggsRes = aggregation.Aggregations
+	}
+	// for _, bucket := range aggsRes.Resource.Buckets {
+	//	bucketName := getResourceName(bucket.Key)
+	//	if bucketName == "User" {
+	//		continue
+	//	}
+	//
+	//	for _, subBucket := range bucket.ResourceName.Buckets {
+	//		resourceName := subBucket.Key
+	//		if bucketName == "Namespace" {
+	//			if val, is := nsTotalHits[resourceName]; is {
+	//				val.Hits += subBucket.DocCount
+	//				nsTotalHits[resourceName] = val
+	//			} else {
+	//				nsTotalHits[resourceName] = cdAggregationHits{
+	//					Name:  resourceName,
+	//					Label: nsHandleMap[resourceName],
+	//					Hits:  subBucket.DocCount,
+	//				}
+	//			}
+	//		}
+	//
+	//		if bucketName == "Module" {
+	//			if val, is := mTotalHits[resourceName]; is {
+	//				val.Hits += subBucket.DocCount
+	//				mTotalHits[resourceName] = val
+	//			} else {
+	//				mTotalHits[resourceName] = cdAggregationHits{
+	//					Name:  resourceName,
+	//					Label: mHandleMap[resourceName],
+	//					Hits:  subBucket.DocCount,
+	//				}
+	//			}
+	//		}
+	//	}
+	//
+	//	// Namespace total aggs hit counts
+	//	for _, nsBucket := range bucket.Namespaces.Buckets {
+	//		resourceName := nsBucket.Key
+	//		if val, is := nsTotalHits[resourceName]; is {
+	//			val.Hits += nsBucket.DocCount
+	//			nsTotalHits[resourceName] = val
+	//		} else {
+	//			nsTotalHits[resourceName] = cdAggregationHits{
+	//				Name:  resourceName,
+	//				Label: nsHandleMap[resourceName],
+	//				Hits:  nsBucket.DocCount,
+	//			}
+	//		}
+	//	}
+	//
+	//	// Module total aggs hit counts
+	//	for _, mBucket := range bucket.Modules.Buckets {
+	//		resourceName := mBucket.Key
+	//		if val, is := mTotalHits[resourceName]; is {
+	//			val.Hits += mBucket.DocCount
+	//			mTotalHits[resourceName] = val
+	//
+	//		} else {
+	//			mTotalHits[resourceName] = cdAggregationHits{
+	//				Name:  resourceName,
+	//				Label: mHandleMap[resourceName],
+	//				Hits:  mBucket.DocCount,
+	//			}
+	//		}
+	//	}
+	// }
+
+	nsAggregation := cdAggregation{
+		Name:         "Namespace",
+		Resource:     "compose:namespace",
+		Hits:         0,
+		ResourceName: []cdAggregationHits{},
+	}
+	for _, bucket := range aggsRes.Namespace.Namespace.Buckets {
+		resourceName := bucket.Key
+
+		if val, is := nsTotalHits[resourceName]; is {
+			val.Hits += bucket.DocCount
+			nsTotalHits[resourceName] = val
+		} else {
+			var name string
+			if nsHandleMap != nil {
+				name = nsHandleMap[resourceName].Name
+			}
+			if len(name) > 0 {
+				nsTotalHits[resourceName] = cdAggregationHits{
+					Name:  name,
+					Label: resourceName,
+					Hits:  bucket.DocCount,
+				}
+			}
+		}
+	}
+	for _, nsHits := range nsTotalHits {
+		nsAggregation.Hits += nsHits.Hits
+		nsAggregation.ResourceName = append(nsAggregation.ResourceName, nsHits)
+	}
+	if len(nsAggregation.ResourceName) > 0 {
+		sort.Slice(nsAggregation.ResourceName, func(i, j int) bool {
+			return nsAggregation.ResourceName[i].Name < nsAggregation.ResourceName[j].Name
+		})
+		out.Aggregations = append(out.Aggregations, nsAggregation)
+	}
+
+	mAggregation := cdAggregation{
+		Name:         "Module",
+		Resource:     "compose:module",
+		Hits:         0,
+		ResourceName: []cdAggregationHits{},
+	}
+
+	for _, bucket := range aggsRes.Module.Module.Buckets {
+		resourceName := bucket.Key
+
+		if val, is := mTotalHits[resourceName]; is {
+			val.Hits += bucket.DocCount
+			mTotalHits[resourceName] = val
+		} else {
+			var name string
+			if mHandleMap != nil {
+				name = mHandleMap[resourceName].Name
+			}
+			mTotalHits[resourceName] = cdAggregationHits{
+				Name:  name,
+				Label: resourceName,
+				Hits:  bucket.DocCount,
+			}
+		}
+	}
+
+	for _, mHits := range mTotalHits {
+		mAggregation.Hits += mHits.Hits
+		mAggregation.ResourceName = append(mAggregation.ResourceName, mHits)
+	}
+	if len(mAggregation.ResourceName) > 0 {
+		sort.Slice(mAggregation.ResourceName, func(i, j int) bool {
+			return mAggregation.ResourceName[i].Name < mAggregation.ResourceName[j].Name
+		})
+		out.Aggregations = append(out.Aggregations, mAggregation)
+	}
+
+	// if !noHits {
+hits:
+	for _, h := range sr.Hits.Hits {
+		aux := map[string]interface{}{}
+		if err = json.Unmarshal(h.Source, &aux); err != nil {
+			return
+		}
+
+		resType := cast.ToString(aux["resourceType"])
+		delete(aux, "resourceType")
+		switch resType {
+		case "system:user":
+			aux["id"] = aux["userID"]
+			delete(aux, "userID")
+
+		case "compose:record":
+			// @todo: Remove below line and find proper solution for searsia as value needs to be in json
+			ssVal := make(map[string]interface{})
+			// @todo refactor record related values
+			type (
+				created struct {
+					At *time.Time `json:"at,omitempty"`
+					By string     `json:"by,omitempty"`
+				}
+
+				record struct {
+					Created struct {
+						At *time.Time `json:"at,omitempty"`
+						By *createdBy `json:"by,omitempty"`
+					} `json:"created"`
+					Module struct {
+						Name     string `json:"name"`
+						Handle   string `json:"handle"`
+						ModuleId uint64 `json:"moduleId,string"`
+					} `json:"module"`
+					Namespace struct {
+						Name        string `json:"name"`
+						Handle      string `json:"handle"`
+						NamespaceId uint64 `json:"namespaceId,string"`
+					} `json:"namespace"`
+					Values      map[string]interface{} `json:"values"`
+					ValueLabels map[string]string      `json:"valueLabels"`
+				}
+			)
+
+			var r record
+			if err = json.Unmarshal(h.Source, &r); err != nil {
+				return
+			}
+
+			type valueJson struct {
+				Name  string      `json:"name"`
+				Label string      `json:"label"`
+				Value interface{} `json:"value"`
+			}
+			key := fmt.Sprintf("%d-%d", r.Namespace.NamespaceId, r.Module.ModuleId)
+			var (
+				slice []valueJson
+				uc    = created{
+					At: r.Created.At,
+					By: getCreatedBy(r.Created.By),
+				}
+			)
+
+			if val, is := moduleMeta[key]; is {
+				for _, f := range val {
+					slice = append(slice, valueJson{
+						Name:  f,
+						Label: r.ValueLabels[f],
+						Value: sanitize(r.Values[f]),
+					})
+
+					if vv, ok := r.Values[f].([]interface{}); ok {
+						if len(vv) > 0 {
+							ssVal[f] = sanitize(vv[0])
+						}
+					}
+				}
+			} else {
+				for k, v := range r.Values {
+					// @todo hardcoded value
+					sanitizedVal := sanitize(v)
+					if len(slice) < 5 && sanitizedVal != nil {
+						slice = append(slice, valueJson{
+							Name:  k,
+							Label: r.ValueLabels[k],
+							Value: sanitizedVal,
+						})
+
+						if vv, ok := v.([]interface{}); ok {
+							if len(vv) > 0 {
+								ssVal[k] = sanitize(vv[0])
+							}
+						}
+					}
+				}
+			}
+
+			matching_fields := make(map[string]any)
+
+			for key, value := range h.Highlight {
+				newKey := strings.TrimPrefix(key, "values.")
+				matching_fields[newKey] = value
+			}
+
+			aux["created"] = uc
+			aux["customValues"] = ssVal
+			aux["values"] = slice
+			aux["matching_fields"] = matching_fields
+			aux["@id"] = aux["_id"]
+			delete(aux, "_id")
+			delete(aux, "valueLabels")
+
+		case "compose:namespace":
+			aux["@id"] = aux["_id"]
+			delete(aux, "_id")
+			delete(aux, "Namespace")
+
+		case "compose:module":
+			aux["@id"] = aux["_id"]
+			delete(aux, "_id")
+			delete(aux, "Namespace")
+			delete(aux, "Module")
+
+		default:
+			continue hits
+		}
+
+		out.Hits = append(out.Hits, cdHit{
+			Type:  resType,
+			Value: aux,
+		})
+	}
+	out.TotalHits = len(out.Hits)
+	// }
+	return
+}
+
+// @todo use RBAC resource stringify
+// getResourceName return name of resource based on resource type
+func getResourceName(resType string) string {
+	switch resType {
+	case "system:user":
+		return "User"
+	case "compose:record":
+		return "Record"
+	case "compose:namespace":
+		return "Namespace"
+	case "compose:module":
+		return "Module"
+	default:
+		return "Resource"
+	}
+}
+
+func getCreatedBy(user *createdBy) string {
+	if user == nil {
+		return ""
+	}
+	if len(user.Name) > 0 {
+		return user.Name
+	} else if len(user.Username) > 0 {
+		return user.Username
+	} else if len(user.Email) > 0 {
+		return user.Email
+	} else if user.UserID > 0 {
+		return fmt.Sprintf("%d", user.UserID)
+	}
+	return ""
+}
+
+func sanitize(v interface{}) interface{} {
+	if v == nil {
+		return v
+	}
+
+	switch v.(type) {
+	case string:
+		v = richText(v.(string))
+	case []interface{}:
+		vv := reflect.ValueOf(v)
+		for i, val := range v.([]interface{}) {
+			// Wee need to do a failsafe here because we're working with reflect values
+			// In case sanitize returns nil, we'll get a runtime panic
+			sanitized := sanitize(val)
+			var v reflect.Value
+
+			// Since we're working with types, we need the zero value of interface{} over nil
+			if sanitized == nil {
+				v = reflect.Zero(reflect.TypeOf((*any)(nil)).Elem())
+			} else {
+				v = reflect.ValueOf(sanitized)
+			}
+			vv.Index(i).Set(v)
+		}
+	default:
+		return v
+	}
+
+	return v
+}
+
+// RichText assures safe HTML content
+func richText(in string) string {
+	// use standard html escaping policy
+	p := bluemonday.StrictPolicy()
+
+	sanitized := p.Sanitize(in)
+
+	// handle escaped strings and unescape them
+	// all the dangerous chars should have been stripped
+	// by now
+	return html.UnescapeString(sanitized)
+}

extra/server-discovery/searcher/rest/es.go

@@ -0,0 +1,645 @@
+package rest
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/elastic/go-elasticsearch/v7/esapi"
+	"github.com/go-chi/jwtauth"
+	"go.uber.org/zap"
+)
+
+type (
+	// EsSearchAggrTerms is aggregations parameter for es search api.
+	EsSearchAggrTerms       map[string]esSearchAggr
+	EsSearchNestedAggrTerms map[string]esSearchNestedAggr
+
+	EsSearchHighlight struct {
+		RequireFieldMatch bool                        `json:"require_field_match"`
+		Fields            map[string]highlightSetting `json:"fields"`
+	}
+
+	esSearchParamsIndex struct {
+		Prefix struct {
+			Index struct {
+				Value string `json:"value"`
+			} `json:"_index"`
+		} `json:"prefix"`
+	}
+
+	esSimpleQueryString struct {
+		Wrap struct {
+			Query string `json:"query"`
+		} `json:"simple_query_string"`
+	}
+
+	esDisMax struct {
+		TieBreaker float64       `json:"tie_breaker,omitempty"`
+		Boost      float64       `json:"boost,omitempty"`
+		Queries    []interface{} `json:"queries,omitempty"`
+	}
+
+	esDisMaxWrap struct {
+		Wrap esDisMax `json:"dis_max,omitempty"`
+	}
+
+	esNestedDisMax struct {
+		Wrap struct {
+			IgnoreUnmapped bool         `json:"ignore_unmapped"`
+			Path           string       `json:"path,omitempty"`
+			Query          esDisMaxWrap `json:"query,omitempty"`
+		} `json:"nested,omitempty"`
+	}
+
+	esMultiMatch struct {
+		Wrap struct {
+			Query string `json:"query"`
+			Type  string `json:"type"`
+			// Operator string   `json:"operator"`
+			Fields             []string `json:"fields"`
+			MinimumShouldMatch string   `json:"minimum_should_match"`
+		} `json:"multi_match"`
+	}
+
+	VectorValue struct {
+		Vector []float32     `json:"vector"`
+		K      int           `json:"k"`
+		Filter []interface{} `json:"filter,omitempty"`
+	}
+
+	esSearchParams struct {
+		Query struct {
+			Bool *struct {
+				// query context
+				Must   []interface{} `json:"must,omitempty"`
+				Should []interface{} `json:"should,omitempty"`
+
+				// filter context
+				Filter  []interface{} `json:"filter,omitempty"`
+				MustNot []interface{} `json:"must_not,omitempty"`
+			} `json:"bool,omitempty"`
+		} `json:"query"`
+
+		Aggregations EsSearchNestedAggrTerms `json:"aggs,omitempty"`
+		Source       SourceFilter            `json:"_source,omitempty"`
+		MinScore     float32                 `json:"min_score,omitempty"`
+		Highlight    EsSearchHighlight       `json:"highlight,omitempty"`
+	}
+
+	esSearchAggrTerm struct {
+		Field string `json:"field,omitempty"`
+		Size  int    `json:"size,omitempty"`
+	}
+
+	esSearchAggrComposite struct {
+		Sources interface{} `json:"sources"` // it can be esSearchAggrTerm,.. (Histogram, Date histogram, GeoTile grid)
+		Size    int         `json:"size,omitempty"`
+	}
+
+	SourceFilter struct {
+		Excludes []string `json:"excludes"` //columns to exclude on the query response
+	}
+
+	esSearchNestedAggs struct {
+		Path string `json:"path,omitempty"`
+	}
+
+	esSearchAggr struct {
+		Nested       esSearchNestedAggs `json:"nested,omitempty"`
+		Terms        esSearchAggrTerm   `json:"terms,omitempty"`
+		Aggregations EsSearchAggrTerms  `json:"aggs,omitempty"`
+		// Composite *esSearchAggrComposite `json:"composite"`
+	}
+
+	esSearchNestedAggrTerm struct {
+		Terms esSearchAggrTerm `json:"terms,omitempty"`
+	}
+	esSearchNestedAggr struct {
+		Nested       esSearchNestedAggs                `json:"nested,omitempty"`
+		Aggregations map[string]esSearchNestedAggrTerm `json:"aggs,omitempty"`
+		// Composite *esSearchAggrComposite `json:"composite"`
+	}
+
+	esSearchResponse struct {
+		Took         int                  `json:"took"`
+		TimedOut     bool                 `json:"timed_out"`
+		Hits         esSearchHits         `json:"hits"`
+		Aggregations esSearchAggregations `json:"aggregations"`
+	}
+
+	esSearchTotal struct {
+		Value    int    `json:"value"`
+		Relation string `json:"relation"`
+	}
+
+	esSearchHits struct {
+		Total esSearchTotal  `json:"total"`
+		Hits  []*esSearchHit `json:"hits"`
+	}
+
+	esSearchHit struct {
+		Index     string          `json:"_index"`
+		ID        string          `json:"_id"`
+		Source    json.RawMessage `json:"_source"`
+		Highlight map[string]any  `json:"highlight,omitempty"`
+	}
+
+	esSearchAggregations struct {
+		Resource struct {
+			DocCountErrorUpperBound int `json:"-"`
+			SumOtherDocCount        int `json:"-"`
+			Buckets                 []struct {
+				Key          string `json:"key"`
+				DocCount     int    `json:"doc_count"`
+				ResourceName struct {
+					DocCountErrorUpperBound int `json:"-"`
+					SumOtherDocCount        int `json:"-"`
+					Buckets                 []struct {
+						Key      string `json:"key"`
+						DocCount int    `json:"doc_count"`
+					} `json:"buckets"`
+				} `json:"resourceName"`
+				Namespaces struct {
+					DocCountErrorUpperBound int `json:"-"`
+					SumOtherDocCount        int `json:"-"`
+					Buckets                 []struct {
+						Key      string `json:"key"`
+						DocCount int    `json:"doc_count"`
+					} `json:"buckets"`
+				} `json:"namespaces"`
+				Modules struct {
+					DocCountErrorUpperBound int `json:"-"`
+					SumOtherDocCount        int `json:"-"`
+					Buckets                 []struct {
+						Key      string `json:"key"`
+						DocCount int    `json:"doc_count"`
+					} `json:"buckets"`
+				} `json:"modules"`
+			} `json:"buckets"`
+		} `json:"resource"`
+		Module struct {
+			DocCount int `json:"doc_count"`
+			Module   struct {
+				DocCountErrorUpperBound int `json:"doc_count_error_upper_bound"`
+				SumOtherDocCount        int `json:"sum_other_doc_count"`
+				Buckets                 []struct {
+					Key      string `json:"key"`
+					DocCount int    `json:"doc_count"`
+				} `json:"buckets"`
+			} `json:"module"`
+		} `json:"module"`
+
+		Namespace struct {
+			DocCount  int `json:"doc_count"`
+			Namespace struct {
+				DocCountErrorUpperBound int `json:"doc_count_error_upper_bound"`
+				SumOtherDocCount        int `json:"sum_other_doc_count"`
+				Buckets                 []struct {
+					Key      string `json:"key"`
+					DocCount int    `json:"doc_count"`
+				} `json:"buckets"`
+			} `json:"namespace"`
+		} `json:"namespace"`
+	}
+
+	searchParams struct {
+		title         string
+		query         string
+		moduleAggs    []string
+		namespaceAggs []string
+		dumpRaw       bool
+		from          int
+		size          int
+
+		resourceType []string
+
+		aggOnly  bool
+		mAggOnly bool
+
+		allowedRoles map[interface{}]bool
+		searchMode   string
+	}
+
+	highlightSetting struct {
+		PreTags           []string `json:"pre_tags"`
+		PostTags          []string `json:"post_tags"`
+		NumberOfFragments int      `json:"number_of_fragments"`
+	}
+)
+
+var (
+	rtIndexMapping = map[string]string{
+		"compose:module":    "compose-modules",
+		"compose:record":    "compose-records-*",
+		"compose:namespace": "compose-namespaces",
+		"system:user":       "system-users",
+	}
+)
+
+func esSearch(ctx context.Context, log *zap.Logger, esc *elasticsearch.Client, p searchParams) (sr *esSearchResponse, page pagination, err error) {
+	var (
+		buf          bytes.Buffer
+		roles        []string
+		userID       uint64
+		_, claims, _ = jwtauth.FromContext(ctx)
+
+		allowedRoleExist = false
+	)
+
+	if _, has := claims["roles"]; has {
+		if rr, is := claims["roles"].([]interface{}); is {
+
+			for _, role := range rr {
+				if _, ok := p.allowedRoles[role]; ok {
+					allowedRoleExist = true
+				}
+
+				roles = append(roles, fmt.Sprintf("%s", role))
+			}
+		}
+	}
+	if _, has := claims["sub"]; has {
+		if sub, is := claims["sub"].(string); is {
+			userID, _ = extractFromSubClaim(sub)
+		}
+	}
+
+	noQ := len(p.query) == 0
+	noNSFilter := len(p.namespaceAggs) == 0
+	// noMFilter := len(p.moduleAggs) == 0
+	sqs := esSimpleQueryString{}
+	sqs.Wrap.Query = strings.TrimSpace(strings.ToLower(p.query))
+
+	query := &esSearchParams{}
+	index := esSearchParamsIndex{}
+
+	if query.Query.Bool == nil {
+		query.Query.Bool = &struct {
+			Must    []interface{} `json:"must,omitempty"`
+			Should  []interface{} `json:"should,omitempty"`
+			Filter  []interface{} `json:"filter,omitempty"`
+			MustNot []interface{} `json:"must_not,omitempty"`
+		}{}
+	}
+
+	// Decide what indexes we can use
+	if userID == 0 {
+		// Missing, invalid, expired access token (JWT)
+		index.Prefix.Index.Value = "corteza-public-"
+	} else {
+		// Authenticated user
+		index.Prefix.Index.Value = "corteza-private-"
+
+		if !allowedRoleExist {
+			// Skip all documents that do not have baring roles in to allow list
+			query.Query.Bool.Filter = append(query.Query.Bool.Filter, map[string]map[string]interface{}{
+				"terms": {"security.allowedRoles": roles},
+			})
+
+			// Skip all documents that have baring roles in to deny list
+			query.Query.Bool.MustNot = append(query.Query.Bool.MustNot, map[string]map[string]interface{}{
+				"terms": {"security.deniedRoles": roles},
+			})
+		}
+	}
+
+	// Aggregations V1.0
+	// if len(p.aggregations) > 0 {
+	//	query.Aggregations = make(map[string]esSearchAggr)
+	//
+	//	for _, a := range p.aggregations {
+	//		query.Aggregations[a] = esSearchAggr{esSearchAggrTerm{Field: a + ".keyword"}}
+	//	}
+	// }
+
+	// Search string filter
+	if !noQ {
+		applyTraditionalSearch(query, index, sqs.Wrap.Query)
+	}
+
+	// add the highlight for the matched fields.
+	query.Highlight = EsSearchHighlight{
+		RequireFieldMatch: false,
+		Fields: map[string]highlightSetting{
+			"values.*": {
+				PreTags:           []string{"*"},
+				PostTags:          []string{"*"},
+				NumberOfFragments: 0,
+			},
+		},
+	}
+
+	var (
+		mm   = esMultiMatch{}
+		mdd  esNestedDisMax
+		nsdd esNestedDisMax
+	)
+	for _, mAggs := range p.moduleAggs {
+		mm.Wrap.Query = mAggs
+		mm.Wrap.Type = "cross_fields"
+		mm.Wrap.MinimumShouldMatch = "100%"
+		mm.Wrap.Fields = []string{"module.name"}
+		// query.Query.Bool.Must = append(query.Query.Bool.Must, mm)
+		// query.Query.DisMax.Queries = append(query.Query.DisMax.Queries, mm)
+
+		// dd.Wrap.Queries = append(dd.Wrap.Queries, mm)
+		mdd.Wrap.Path = "module"
+		nsdd.Wrap.IgnoreUnmapped = true
+		mdd.Wrap.Query.Wrap.Queries = append(mdd.Wrap.Query.Wrap.Queries, mm)
+	}
+
+	if len(mdd.Wrap.Query.Wrap.Queries) > 0 {
+		query.Query.Bool.Must = append(query.Query.Bool.Must, mdd)
+	}
+
+	// no need now since we are adding below as filter
+	// if p.aggOnly {
+	for _, nAggs := range p.namespaceAggs {
+		mm.Wrap.Query = nAggs
+		mm.Wrap.Type = "cross_fields"
+		mm.Wrap.MinimumShouldMatch = "100%"
+		mm.Wrap.Fields = []string{"namespace.name"}
+		// query.Query.Bool.Must = append(query.Query.Bool.Must, mm)
+		// query.Query.DisMax.Queries = append(query.Query.DisMax.Queries, mm)
+
+		// dd.Wrap.Queries = append(dd.Wrap.Queries, mm)
+		nsdd.Wrap.Path = "namespace"
+		nsdd.Wrap.IgnoreUnmapped = true
+		nsdd.Wrap.Query.Wrap.Queries = append(nsdd.Wrap.Query.Wrap.Queries, mm)
+	}
+	// }
+
+	if len(nsdd.Wrap.Query.Wrap.Queries) > 0 {
+		query.Query.Bool.Must = append(query.Query.Bool.Must, nsdd)
+	}
+
+	// if !p.aggOnly && !noNSFilter {
+	// 	nsf := make(map[string]interface{})
+	// 	nsf["terms"] = map[string][]string{
+	// 		"namespace.name.keyword": p.namespaceAggs,
+	// 	}
+	// 	query.Query.Bool.Filter = append(query.Query.Bool.Filter, nsf)
+	// }
+
+	// Aggregations V1.0 Improved
+	// if len(p.aggregations) > 0 {
+	//	for _, a := range p.aggregations {
+	//		if len(a) > 0 {
+	//			sqs = esSimpleQueryString{}
+	//			sqs.Wrap.Query = a
+	//			query.Query.Bool.Must = append(query.Query.Bool.Must, sqs)
+	//		}
+	//	}
+	// }
+
+	// if noQ == 0 && len(p.moduleAggs) == 0 && len(p.namespaceAggs) == 0 {
+	//	query.Query.DisMax.Queries = append(query.Query.DisMax.Queries, index)
+	// }
+	query.Aggregations = make(map[string]esSearchNestedAggr)
+	query.Aggregations["namespace"] = esSearchNestedAggr{
+		Nested: esSearchNestedAggs{Path: "namespace"},
+		Aggregations: map[string]esSearchNestedAggrTerm{
+			"namespace": {
+				Terms: esSearchAggrTerm{
+					// Field: "namespace.name.keyword",
+					Field: "namespace.handle",
+					Size:  999,
+				},
+			},
+		},
+	}
+
+	query.Source = SourceFilter{
+		Excludes: []string{"vectorsValue"},
+	}
+
+	if !noQ || !noNSFilter {
+		query.Aggregations["module"] = esSearchNestedAggr{
+			Nested: esSearchNestedAggs{Path: "module"},
+			Aggregations: map[string]esSearchNestedAggrTerm{
+				"module": {
+					Terms: esSearchAggrTerm{
+						// Field: "module.name.keyword",
+						Field: "module.handle",
+						Size:  999,
+					},
+				},
+			},
+		}
+	}
+
+	// query.Aggregations["resource"] = esSearchAggr{
+	//	Terms: esSearchAggrTerm{
+	//		Field: "resourceType.keyword",
+	//		Size:  999,
+	//	},
+	//	Aggregations: EsSearchAggrTerms{
+	//		"resourceName": esSearchAggr{
+	//			Terms: esSearchAggrTerm{
+	//				Field: "name.keyword",
+	//				Size:  999,
+	//			},
+	//		},
+	//		"modules": esSearchAggr{
+	//			Terms: esSearchAggrTerm{
+	//				Field: "module.name.keyword",
+	//				Size:  999,
+	//			},
+	//		},
+	//		"namespaces": esSearchAggr{
+	//			Terms: esSearchAggrTerm{
+	//				Field: "namespace.name.keyword",
+	//				Size:  999,
+	//			},
+	//		},
+	//	},
+	// }
+
+	// Aggregations V2.0
+	// if len(p.aggregations) > 0 {
+	//	query.Aggregations = (Aggregations{}).encodeTerms(p.aggregations)
+	// }
+
+	// spew.Dump("query: ", query)
+	if err = json.NewEncoder(&buf).Encode(query); err != nil {
+		err = fmt.Errorf("could not encode query: %q", err)
+		return
+	}
+	spew.Dump("buf: ", buf)
+
+	log.Debug("searching ",
+		zap.String("for", p.title),
+		// zap.String("open search body", buf.String()),
+	)
+
+	// Why set size to 999? default value for size is 10,
+	// so we needed to set value till we add (@todo) pagination to search result
+	if p.from > 0 && p.from >= p.size {
+		p.size = p.from + 100
+	}
+	if p.size == 0 {
+		p.size = 999
+	}
+	page.From = p.from
+	page.Size = p.size
+
+	// In case of requesting specific resource types, use only the appropriate indexes
+	indexes := []string{}
+	if len(p.resourceType) > 0 {
+		for _, rt := range p.resourceType {
+			if _, ok := rtIndexMapping[rt]; !ok {
+				err = fmt.Errorf("invalid resource type: %s", rt)
+				return
+			}
+
+			indexes = append(indexes, fmt.Sprintf("%s%s", index.Prefix.Index.Value, rtIndexMapping[rt]))
+		}
+	}
+
+	sReqArgs := []func(*esapi.SearchRequest){
+		esc.Search.WithContext(ctx),
+		esc.Search.WithBody(&buf),
+		esc.Search.WithTrackTotalHits(true),
+		// esc.Search.WithScroll(),
+		esc.Search.WithSize(p.size),
+		esc.Search.WithFrom(p.from),
+		esc.Search.WithIndex(indexes...),
+		// esc.Search.WithExplain(true), // debug
+	}
+
+	if p.dumpRaw {
+		sReqArgs = append(
+			sReqArgs,
+			esc.Search.WithSourceExcludes("security"),
+			esc.Search.WithPretty(),
+		)
+	}
+
+	// Perform the search request.
+	res, err := esc.Search(sReqArgs...)
+
+	if err != nil {
+		return
+	}
+
+	if err = validElasticResponse(res, err); err != nil {
+		err = fmt.Errorf("invalid search response: %w", err)
+		return
+	}
+
+	defer res.Body.Close()
+
+	if p.dumpRaw {
+		// Copy body buf and then restore it
+		bodyBytes, _ := ioutil.ReadAll(res.Body)
+		res.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
+		os.Stdout.Write(bodyBytes)
+	}
+
+	sr = &esSearchResponse{}
+	if err = json.NewDecoder(res.Body).Decode(sr); err != nil {
+		return
+	}
+
+	// Print the response status, number of results, and request duration.
+	log.Debug("search completed",
+		zap.String("for", p.title),
+		zap.String("query", sqs.Wrap.Query),
+		zap.String("indexPrefix", index.Prefix.Index.Value),
+		zap.Int("from", p.from),
+		zap.Int("size", p.size),
+		zap.String("status", res.Status()),
+		zap.Int("took", sr.Took),
+		zap.Bool("timedOut", sr.TimedOut),
+		zap.Int("hits", sr.Hits.Total.Value),
+		zap.String("hitsRelation", sr.Hits.Total.Relation),
+		zap.Int("namespaceAggs", len(sr.Aggregations.Namespace.Namespace.Buckets)),
+		zap.Int("moduleAggs", len(sr.Aggregations.Module.Module.Buckets)),
+	)
+
+	return
+}
+
+// @todo move this to es service
+func validElasticResponse(res *esapi.Response, err error) error {
+	if err != nil {
+		return fmt.Errorf("failed to get response from search backend: %w", err)
+	}
+
+	if res.IsError() {
+		defer res.Body.Close()
+		var rsp struct {
+			Error struct {
+				Type   string
+				Reason string
+			}
+		}
+
+		if err := json.NewDecoder(res.Body).Decode(&rsp); err != nil {
+			return fmt.Errorf("could not parse response body: %w", err)
+		} else {
+			return fmt.Errorf("search backend responded with an error: %s (type: %s, status: %s)", rsp.Error.Reason, rsp.Error.Type, res.Status())
+		}
+	}
+
+	return nil
+}
+
+func extractFromSubClaim(sub string) (userID uint64, rr []uint64) {
+	parts := strings.Split(sub, " ")
+	rr = make([]uint64, len(parts)-1)
+	for p := range parts {
+		id, _ := strconv.ParseUint(parts[p], 10, 64)
+		if p == 0 {
+			userID = id
+		} else {
+			rr[p-1] = id
+		}
+	}
+
+	return
+}
+
+func applyTraditionalSearch(query *esSearchParams, index interface{}, searchQuery string) {
+	query.Query.Bool.Must = []interface{}{index}
+	query.Query.Bool.Must = append(query.Query.Bool.Must, buildWildcardQuery(searchQuery))
+}
+
+func buildWildcardQuery(searchQuery string) map[string]interface{} {
+	return map[string]interface{}{
+		"wildcard": map[string]interface{}{
+			"catch_all": map[string]interface{}{
+				"value": fmt.Sprintf("*%s*", searchQuery),
+			},
+		},
+	}
+}
+
+func buildKNNQuery(vector interface{}) map[string]interface{} {
+	return map[string]interface{}{
+		"knn": map[string]interface{}{
+			"vectorsValue": map[string]interface{}{
+				"vector": vector,
+				"k":      10,
+			},
+		},
+	}
+}
+
+func buildMultiMatchQuery(searchQuery string) map[string]interface{} {
+	return map[string]interface{}{
+		"multi_match": map[string]interface{}{
+			"query":  searchQuery,
+			"fields": []string{"catch_all"},
+			"boost":  1.0,
+		},
+	}
+}

extra/server-discovery/searcher/rest/handlers/search.go

@@ -0,0 +1,49 @@
+package handlers
+
+import (
+	"context"
+	"github.com/cortezaproject/corteza/extra/server-discovery/searcher/rest/request"
+	"github.com/cortezaproject/corteza/server/pkg/api"
+	"github.com/go-chi/chi/v5"
+	"net/http"
+)
+
+type (
+	// SearchAPI Internal API interface
+	SearchAPI interface {
+		SearchResources(context.Context, *request.SearchResources) (interface{}, error)
+	}
+
+	// Search HTTP API interface
+	Search struct {
+		SearchResources func(http.ResponseWriter, *http.Request)
+	}
+)
+
+func NewSearch(h SearchAPI) *Search {
+	return &Search{
+		SearchResources: func(w http.ResponseWriter, r *http.Request) {
+			defer r.Body.Close()
+			params := request.NewSearchListResources()
+			if err := params.Fill(r); err != nil {
+				api.Send(w, r, err)
+				return
+			}
+
+			value, err := h.SearchResources(r.Context(), params)
+			if err != nil {
+				api.Send(w, r, err)
+				return
+			}
+
+			api.Send(w, r, value)
+		},
+	}
+}
+
+func (h Search) MountRoutes(r chi.Router, middlewares ...func(http.Handler) http.Handler) {
+	r.Group(func(r chi.Router) {
+		r.Use(middlewares...)
+		r.Get("/", h.SearchResources)
+	})
+}

extra/server-discovery/searcher/rest/request/request.go

@@ -0,0 +1,139 @@
+package request
+
+import (
+	"net/http"
+
+	"github.com/cortezaproject/corteza/server/pkg/payload"
+)
+
+type (
+	SearchResources struct {
+		Q             string
+		From          int
+		Size          int
+		Rt            []string
+		NamespaceAggs []string
+		ModuleAggs    []string
+		DumpRaw       string
+	}
+)
+
+// NewSearchListResources request
+func NewSearchListResources() *SearchResources {
+	return &SearchResources{}
+}
+
+// Auditable returns all auditable/loggable parameters
+func (r SearchResources) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"q":             r.Q,
+		"from":          r.From,
+		"size":          r.Size,
+		"namespaceAggs": r.NamespaceAggs,
+		"resourceTypes": r.Rt,
+		"moduleAggs":    r.ModuleAggs,
+		"dump":          r.DumpRaw,
+	}
+}
+
+func (r SearchResources) GetQuery() string {
+	return r.Q
+}
+
+func (r SearchResources) GetSize() int {
+	return r.Size
+}
+
+func (r SearchResources) GetFrom() int {
+	return r.From
+}
+
+func (r SearchResources) GetResourceTypes() []string {
+	return r.Rt
+}
+
+func (r SearchResources) GetNamespaceAggs() []string {
+	return r.NamespaceAggs
+}
+
+func (r SearchResources) GetModuleAggs() []string {
+	return r.ModuleAggs
+}
+
+func (r SearchResources) GetDumpRaw() string {
+	return r.DumpRaw
+}
+
+// Fill processes request and fills internal variables
+func (r *SearchResources) Fill(req *http.Request) (err error) {
+	{
+		// GET params
+		tmp := req.URL.Query()
+
+		if val, ok := tmp["q"]; ok && len(val) > 0 {
+			r.Q, err = val[0], nil
+			if err != nil {
+				return err
+			}
+		}
+
+		if val, ok := tmp["size"]; ok && len(val) > 0 {
+			r.Size, err = payload.ParseInt(val[0]), nil
+			if err != nil {
+				return err
+			}
+		}
+
+		if val, ok := tmp["from"]; ok && len(val) > 0 {
+			r.From, err = payload.ParseInt(val[0]), nil
+			if err != nil {
+				return err
+			}
+		}
+
+		if val, ok := tmp["resourceTypes[]"]; ok {
+			r.Rt, err = val, nil
+			if err != nil {
+				return err
+			}
+		} else if val, ok = tmp["resourceTypes"]; ok {
+			r.Rt, err = val, nil
+			if err != nil {
+				return err
+			}
+		}
+
+		if val, ok := tmp["namespaceAggs[]"]; ok {
+			r.NamespaceAggs, err = val, nil
+			if err != nil {
+				return err
+			}
+		} else if val, ok = tmp["namespaceAggs"]; ok {
+			r.NamespaceAggs, err = val, nil
+			if err != nil {
+				return err
+			}
+		}
+
+		if val, ok := tmp["moduleAggs[]"]; ok {
+			r.ModuleAggs, err = val, nil
+			if err != nil {
+				return err
+			}
+		} else if val, ok = tmp["moduleAggs"]; ok {
+			r.ModuleAggs, err = val, nil
+			if err != nil {
+				return err
+			}
+		}
+
+		if val, ok := tmp["dump"]; ok && len(val) > 0 {
+			r.DumpRaw, err = val[0], nil
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return err
+}

extra/server-discovery/searcher/rest/router.go

@@ -0,0 +1,21 @@
+package rest
+
+import (
+	"github.com/cortezaproject/corteza/extra/server-discovery/pkg/auth"
+	"github.com/cortezaproject/corteza/extra/server-discovery/searcher/rest/handlers"
+	"github.com/go-chi/chi/v5"
+	"go.uber.org/zap"
+)
+
+func MountRoutes(log *zap.Logger) func(r chi.Router) {
+	return func(r chi.Router) {
+		r.Group(func(r chi.Router) {
+			search := Search()
+
+			r.Use(auth.HttpTokenValidator("discovery"))
+
+			// traditional search endpoint for Traditional and vectorsearch
+			handlers.NewSearch(search).MountRoutes(r)
+		})
+	}
+}

extra/server-discovery/searcher/rest/search.go

@@ -0,0 +1,336 @@
+package rest
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/cortezaproject/corteza/extra/server-discovery/searcher"
+	"github.com/cortezaproject/corteza/extra/server-discovery/searcher/rest/request"
+	types2 "github.com/cortezaproject/corteza/server/compose/types"
+)
+
+type (
+	search struct{}
+
+	cResponse struct {
+		Response struct {
+			Set []struct {
+				NamespaceID uint64 `json:",string"`
+				Slug        string `json:"slug"`
+
+				Name     string              `json:"name"`
+				ModuleID uint64              `json:",string"`
+				Handle   string              `json:"handle"`
+				Config   types2.ModuleConfig `json:"config"`
+			} `json:"set,omitempty"`
+		} `json:"response,omitempty"`
+	}
+
+	moduleMeta struct {
+		Discovery ModuleMeta `json:"discovery"`
+	}
+
+	ModuleMeta struct {
+		Public struct {
+			Result []Result `json:"result"`
+		} `json:"public"`
+		Private struct {
+			Result []Result `json:"result"`
+		} `json:"private"`
+		Protected struct {
+			Result []Result `json:"result"`
+		} `json:"protected"`
+	}
+
+	Result struct {
+		Lang   string   `json:"lang"`
+		Fields []string `json:"fields"`
+		// @todo? TBD? excludeModuleFields, includeModuleFields <- if passed filter module field accordingly.
+	}
+
+	nsMeta struct {
+		Name   string
+		Handle string
+	}
+	mMeta struct {
+		Name   string
+		Handle string
+	}
+)
+
+func Search() *search {
+	return &search{}
+}
+
+func (s search) SearchResources(ctx context.Context, r *request.SearchResources) (out interface{}, err error) {
+	var (
+		log           = searcher.DefaultLogger
+		allowedRoles  = searcher.DefaultConfig.Searcher.AllowedRole
+		searchString  = r.GetQuery()
+		size          = r.GetSize()
+		from          = r.GetFrom()
+		namespaceAggs = r.GetNamespaceAggs()
+		moduleAggs    = r.GetModuleAggs()
+		resourceTypes = r.GetResourceTypes()
+		validDumpRaw  = r.GetDumpRaw() != ""
+
+		page          pagination
+		results       *esSearchResponse
+		aggregation   *esSearchResponse
+		nsAggregation *esSearchResponse
+		mAggregation  *esSearchResponse
+
+		nsReq      *http.Request
+		nsRes      *http.Response
+		mReq       *http.Request
+		mRes       *http.Response
+		nsResponse cResponse
+		mResponse  cResponse
+		moduleMap  = make(map[string][]string)
+
+		nsHandleMap = make(map[string]nsMeta)
+		mHandleMap  = make(map[string]mMeta)
+	)
+
+	esc := searcher.DefaultEsClient
+
+	results, page, err = esSearch(ctx, log, esc, searchParams{
+		title:         "results",
+		query:         searchString,
+		resourceType:  resourceTypes,
+		from:          from,
+		size:          size,
+		moduleAggs:    moduleAggs,
+		namespaceAggs: namespaceAggs,
+		dumpRaw:       validDumpRaw,
+		allowedRoles:  allowedRoles,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("could not execute search: %w", err)
+	}
+
+	if len(searchString) == 0 {
+		aggregation, _, err = esSearch(ctx, log, esc, searchParams{
+			title: "aggregation",
+			// size:          size,
+			dumpRaw:       validDumpRaw,
+			namespaceAggs: namespaceAggs,
+			aggOnly:       true,
+			allowedRoles:  allowedRoles,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("could not execute aggregation search: %w", err)
+		}
+	}
+
+	// append all namespace agg with counts no matter what
+	nsAggregation, _, err = esSearch(ctx, log, esc, searchParams{
+		title: "nsAggregation",
+		// size:    size,
+		dumpRaw:      validDumpRaw,
+		aggOnly:      true,
+		allowedRoles: allowedRoles,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("could not execute namespace aggregation search: %w", err)
+	}
+
+	if len(searchString) == 0 {
+		if aggregation != nil && nsAggregation != nil {
+			aggregation.Aggregations.Namespace = nsAggregation.Aggregations.Namespace
+		}
+	} else {
+		if results != nil && nsAggregation != nil {
+			nsMap := make(map[string]struct {
+				Key      string `json:"key"`
+				DocCount int    `json:"doc_count"`
+			})
+			for _, bucket := range results.Aggregations.Namespace.Namespace.Buckets {
+				nsMap[bucket.Key] = bucket
+			}
+
+			var buckets []struct {
+				Key      string `json:"key"`
+				DocCount int    `json:"doc_count"`
+			}
+			for _, bucket := range nsAggregation.Aggregations.Namespace.Namespace.Buckets {
+				val, ok := nsMap[bucket.Key]
+				if ok {
+					val.DocCount = nsMap[bucket.Key].DocCount
+				} else {
+					val.Key = bucket.Key
+					val.DocCount = 0
+				}
+				buckets = append(buckets, val)
+			}
+
+			results.Aggregations.Namespace.Namespace.Buckets = buckets
+		}
+	}
+	// append namespace agg response which are not in es response
+	if results != nil && len(namespaceAggs) > 0 {
+		nsMap := make(map[string]struct {
+			Key      string `json:"key"`
+			DocCount int    `json:"doc_count"`
+		})
+		var bb []struct {
+			Key      string `json:"key"`
+			DocCount int    `json:"doc_count"`
+		}
+		for _, b := range results.Aggregations.Namespace.Namespace.Buckets {
+			nsMap = map[string]struct {
+				Key      string `json:"key"`
+				DocCount int    `json:"doc_count"`
+			}{
+				b.Key: b,
+			}
+			bb = append(bb, b)
+		}
+
+		for _, agg := range namespaceAggs {
+			if _, ok := nsMap[agg]; !ok {
+				nsMap = map[string]struct {
+					Key      string `json:"key"`
+					DocCount int    `json:"doc_count"`
+				}{
+					agg: {Key: agg, DocCount: 0},
+				}
+				bb = append(bb, struct {
+					Key      string `json:"key"`
+					DocCount int    `json:"doc_count"`
+				}{Key: agg, DocCount: 0})
+			}
+		}
+
+		if len(bb) > 0 {
+			results.Aggregations.Namespace.Namespace.Buckets = bb
+		}
+	}
+
+	mAggregation, _, err = esSearch(ctx, log, esc, searchParams{
+		title: "mAggregation",
+		// size:          size,
+		dumpRaw:       validDumpRaw,
+		query:         searchString,
+		namespaceAggs: namespaceAggs,
+		aggOnly:       true,
+		mAggOnly:      true,
+		allowedRoles:  allowedRoles,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("could not execute module aggregation search: %w", err)
+	}
+
+	if len(searchString) > 0 {
+		if results != nil && mAggregation != nil {
+			results.Aggregations.Module = mAggregation.Aggregations.Module
+		}
+	}
+
+	// append module agg response which are not in es response
+	if results != nil && len(moduleAggs) > 0 {
+		mMap := make(map[string]struct {
+			Key      string `json:"key"`
+			DocCount int    `json:"doc_count"`
+		})
+		var bb []struct {
+			Key      string `json:"key"`
+			DocCount int    `json:"doc_count"`
+		}
+		for _, b := range results.Aggregations.Module.Module.Buckets {
+			mMap = map[string]struct {
+				Key      string `json:"key"`
+				DocCount int    `json:"doc_count"`
+			}{
+				b.Key: b,
+			}
+			bb = append(bb, b)
+		}
+
+		for _, agg := range moduleAggs {
+			if _, ok := mMap[agg]; !ok {
+				mMap = map[string]struct {
+					Key      string `json:"key"`
+					DocCount int    `json:"doc_count"`
+				}{
+					agg: {Key: agg, DocCount: 0},
+				}
+				bb = append(bb, struct {
+					Key      string `json:"key"`
+					DocCount int    `json:"doc_count"`
+				}{Key: agg, DocCount: 0})
+			}
+		}
+
+		if len(bb) > 0 {
+			results.Aggregations.Module.Module.Buckets = bb
+		}
+	}
+
+	noHits := len(searchString) == 0 && len(moduleAggs) == 0 && len(namespaceAggs) == 0
+	// if !noHits {
+	// @todo only fetch module from result but that requires another loop to fetch module Id from es response
+	// 			TEMP fix, I have solution use elastic for the same but different index
+	nsReq, err = searcher.DefaultApiClient.Namespaces()
+	if err != nil {
+		return nil, fmt.Errorf("failed to prepare namespace request: %w", err)
+	} else {
+		if nsRes, err = searcher.DefaultApiClient.HttpClient().Do(nsReq.WithContext(ctx)); err != nil {
+			return nil, fmt.Errorf("failed to send namespace request: %w", err)
+		}
+		if nsRes.StatusCode != http.StatusOK {
+			fmt.Println("err: ", err)
+			return nil, fmt.Errorf("request resulted in an unexpected status: %s: %w", err)
+		}
+		if err = json.NewDecoder(nsRes.Body).Decode(&nsResponse); err != nil {
+			return nil, fmt.Errorf("failed to decode namespace response: %w", err)
+		}
+
+		if err = nsRes.Body.Close(); err != nil {
+			return nil, fmt.Errorf("failed to close namespace response body: %w", err)
+		}
+
+		for _, s := range nsResponse.Response.Set {
+			// Get the module handles for aggs response
+			nsHandleMap[s.Slug] = nsMeta{
+				Name:   s.Name,
+				Handle: s.Slug,
+			}
+			if mReq, err = searcher.DefaultApiClient.Modules(s.NamespaceID); err != nil {
+				return nil, fmt.Errorf("failed to prepare module meta request: %w", err)
+			}
+			if mRes, err = searcher.DefaultApiClient.HttpClient().Do(mReq.WithContext(ctx)); err != nil {
+				return nil, fmt.Errorf("failed to send module request: %w", err)
+			}
+			if mRes.StatusCode != http.StatusOK {
+				return nil, fmt.Errorf("request resulted in an unexpected status: %s: %w", err)
+			}
+			if err = json.NewDecoder(mRes.Body).Decode(&mResponse); err != nil {
+				return nil, fmt.Errorf("failed to decode response: %w", err)
+			}
+			if err = mRes.Body.Close(); err != nil {
+				return nil, fmt.Errorf("failed to close response body: %w", err)
+			}
+
+			for _, m := range mResponse.Response.Set {
+				// Get the module handles for aggs response
+				mHandleMap[m.Handle] = mMeta{
+					Name:   m.Name,
+					Handle: m.Slug,
+				}
+
+				var (
+					key = fmt.Sprintf("%d-%d", s.NamespaceID, m.ModuleID)
+				)
+				if len(m.Config.Discovery.Private.Result) > 0 && len(m.Config.Discovery.Private.Result[0].Fields) > 0 {
+					moduleMap[key] = m.Config.Discovery.Private.Result[0].Fields
+				}
+			}
+		}
+	}
+	// }
+
+	return conv(results, aggregation, noHits, moduleMap, nsHandleMap, mHandleMap, page)
+}