snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

extra/server-discovery/pkg/es/connect.go

@@ -0,0 +1,52 @@
+package es
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"net/http"
+	"os"
+
+	"github.com/cortezaproject/corteza/extra/server-discovery/pkg/options"
+	"github.com/elastic/go-elasticsearch/v7"
+)
+
+func Connect(opt options.EsOpt) (*elasticsearch.Client, error) {
+	config := elasticsearch.Config{
+		Addresses:            opt.Addresses,
+		EnableRetryOnTimeout: opt.EnableRetryOnTimeout,
+		MaxRetries:           opt.MaxRetries,
+	}
+	if len(opt.Username) > 0 {
+		config.Username = opt.Username
+		config.Password = opt.Password
+	}
+
+	// if the user provided a CA certificate file, we need to load it
+	// and set it in the TLS config
+	if opt.CertFile != "" {
+		rootCACert, err := os.ReadFile(opt.CertFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read CA certificate file: %w", err)
+		}
+
+		caCertPool := x509.NewCertPool()
+		if !caCertPool.AppendCertsFromPEM(rootCACert) {
+			return nil, fmt.Errorf("failed parse root certificate: %w", err)
+		}
+
+		config.Transport = &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs: caCertPool,
+			},
+		}
+	} else if !opt.Secure {
+		config.Transport = &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		}
+	}
+
+	return elasticsearch.NewClient(config)
+}

extra/server-discovery/pkg/es/es.go

@@ -0,0 +1,70 @@
+package es
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/elastic/go-elasticsearch/v7/esapi"
+	"github.com/elastic/go-elasticsearch/v7/esutil"
+	"go.uber.org/zap"
+)
+
+type (
+	es struct {
+		log *zap.Logger
+		esc *elasticsearch.Client
+		//esb esutil.BulkIndexer
+	}
+
+	Service interface {
+		EsClient() *elasticsearch.Client
+		EsBulk() esutil.BulkIndexer
+		Watch(ctx context.Context)
+	}
+
+	//apiClientService interface {
+	//	HttpClient() *http.Client
+	//	Mappings() (*http.Request, error)
+	//	Resources(string, url.Values) (*http.Request, error)
+	//	Request(string) (*http.Request, error)
+	//	Authenticate() error
+	//}
+)
+
+func ES(log *zap.Logger, esc *elasticsearch.Client) (out *es, err error) {
+	out = &es{log: log, esc: esc}
+	return
+}
+
+func (es *es) BulkIndexer() (esutil.BulkIndexer, error) {
+	return esutil.NewBulkIndexer(esutil.BulkIndexerConfig{
+		Client:     es.esc,
+		FlushBytes: 5e+5,
+	})
+}
+
+func ValidElasticResponse(res *esapi.Response, err error) error {
+	if err != nil {
+		return fmt.Errorf("failed to get response from search backend: %w", err)
+	}
+
+	if res.IsError() {
+		defer res.Body.Close()
+		var rsp struct {
+			Error struct {
+				Type   string
+				Reason string
+			}
+		}
+
+		if err := json.NewDecoder(res.Body).Decode(&rsp); err != nil {
+			return fmt.Errorf("could not parse response body: %w", err)
+		} else {
+			return fmt.Errorf("search backend responded with an error: %s (type: %s, status: %s)", rsp.Error.Reason, rsp.Error.Type, res.Status())
+		}
+	}
+
+	return nil
+}

extra/server-discovery/pkg/es/mapping/mapping.go

@@ -0,0 +1,287 @@
+package mapping
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"github.com/cortezaproject/corteza/extra/server-discovery/pkg/es/reindex"
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/elastic/go-elasticsearch/v7/esapi"
+	"go.uber.org/zap"
+)
+
+type (
+	esIndex struct {
+		Name      string `json:"index"`
+		Health    string `json:"health"`
+		Status    string `json:"status"`
+		DocsCount string `json:"docs.count"`
+		StoreSize string `json:"store.size"`
+	}
+
+	rspDiscoveryMappings struct {
+		Response []*mapping
+	}
+
+	reqMapping struct {
+		// @todo structure
+		Settings map[string]any `json:"settings,omitempty"`
+
+		Mappings struct {
+			Properties map[string]*property `json:"properties,omitempty"`
+		} `json:"mappings,omitempty"`
+	}
+
+	mapping struct {
+		Index        string               `json:"index"`
+		Properties   map[string]*property `json:"properties"`
+		Mapping      map[string]*property `json:"mapping"`
+		DocumentsURL string               `json:"documentsURL"`
+	}
+
+	property struct {
+		// https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html
+		Type string `json:"type,omitempty"`
+
+		Analyzer string `json:"analyzer,omitempty"`
+
+		// Boost factor
+		// https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-boost.html
+		Boost float32 `json:"boost,omitempty"`
+
+		Properties map[string]*property `json:"properties,omitempty"`
+
+		// for vector search
+		Dimension int            `json:"dimension,omitempty"`
+		Method    map[string]any `json:"method,omitempty"`
+	}
+
+	apiClientService interface {
+		HttpClient() *http.Client
+		Mappings() (*http.Request, error)
+		Resources(string, url.Values) (*http.Request, error)
+		Request(string) (*http.Request, error)
+		Authenticate() error
+	}
+
+	mapper struct {
+		log      *zap.Logger
+		esClient *elasticsearch.Client
+		api      apiClientService
+	}
+)
+
+func Mapper(log *zap.Logger, esClient *elasticsearch.Client, api apiClientService) *mapper {
+	return &mapper{
+		log:      log,
+		esClient: esClient,
+		api:      api,
+	}
+}
+
+// Mappings fetches mappings from discovery server and update elastic search indexes
+func (m *mapper) Mappings(ctx context.Context, esc *elasticsearch.Client, indexPrefix string) (err error) {
+	var (
+		req             *http.Request
+		rsp             *http.Response
+		rspPayload      = &rspDiscoveryMappings{}
+		buf             = &bytes.Buffer{}
+		esRsp           *esapi.Response
+		existingIndexes []*esIndex
+		index           string
+	)
+
+	if req, err = m.api.Mappings(); err != nil {
+		return fmt.Errorf("failed to prepare mapping request: %w", err)
+	}
+
+	//d, _ := httputil.DumpRequest(req, true)
+	//println(string(d))
+
+	if rsp, err = m.api.HttpClient().Do(req.WithContext(ctx)); err != nil {
+		return fmt.Errorf("failed to send request: %w", err)
+	}
+
+	if rsp.StatusCode != http.StatusOK {
+		return fmt.Errorf("request resulted in an unexpected status: %s", rsp.Status)
+	}
+
+	//d, _ := httputil.DumpResponse(rsp, true)
+	//println(string(d))
+
+	if err = json.NewDecoder(rsp.Body).Decode(rspPayload); err != nil {
+		return fmt.Errorf("failed to decode mapping response: %w", err)
+	}
+	if err = rsp.Body.Close(); err != nil {
+		return fmt.Errorf("failed to close mapping response body: %w", err)
+	}
+
+	if existingIndexes, err = m.getExistingIndexes(ctx); err != nil {
+		return fmt.Errorf("failed to fetch existing indexes: %w", err)
+	}
+
+	indexMap := m.mapExistingIndexes(existingIndexes)
+
+	//esc, err := m.es.Client()
+	//if err != nil {
+	//	return fmt.Errorf("failed to prepare es client: %w", err)
+	//}
+
+	for _, im := range rspPayload.Response {
+		buf.Reset()
+		esReq := reqMapping{}
+		//esReq.Mappings.Properties = im.Properties
+		esReq.Settings = map[string]any{
+			"analysis": map[string]any{
+				"analyzer": map[string]any{
+					"edge_ngram_analyzer": map[string]any{
+						"tokenizer": "edge_ngram_tokenizer",
+						"filter":    []any{"lowercase"},
+					},
+				},
+				"tokenizer": map[string]any{
+					"edge_ngram_tokenizer": map[string]any{
+						"type":        "edge_ngram",
+						"min_gram":    2,
+						"max_gram":    15,
+						"token_chars": []any{"letter", "digit"},
+					},
+				},
+			},
+			"index": map[string]any{
+				"knn": true,
+			},
+		}
+		esReq.Mappings.Properties = im.Mapping
+
+		if err = json.NewEncoder(buf).Encode(esReq); err != nil {
+			return
+		}
+
+		index = fmt.Sprintf(reindex.IndexTpl, indexPrefix, im.Index)
+		iLog := m.log.With(zap.String("name", index))
+
+		if e := indexMap[index]; e != nil {
+			// @todo upsert settings if not the same?
+			iLog.Info("index exists",
+				zap.String("health", e.Health),
+				zap.String("status", e.Status),
+				zap.String("size", e.StoreSize),
+				zap.String("documents", e.DocsCount),
+			)
+
+			continue
+		}
+
+		if esRsp, err = esc.Indices.Create(index, esc.Indices.Create.WithBody(buf)); esRsp.IsError() || err != nil {
+			if err != nil {
+				iLog.Error("index creation failed", zap.Error(err))
+			}
+			if len(esRsp.String()) > 0 {
+				iLog.Error(fmt.Sprintf("index creation failed due to %s", esRsp.String()))
+			}
+			continue
+		}
+
+		if err = esRsp.Body.Close(); err != nil {
+			return
+		}
+
+		iLog.Info("index created")
+	}
+
+	return
+}
+
+func (m *mapper) mapExistingIndexes(ii []*esIndex) (im map[string]*esIndex) {
+	im = make(map[string]*esIndex)
+	for _, i := range ii {
+		im[i.Name] = i
+	}
+
+	return
+}
+
+func (m *mapper) getExistingIndexes(ctx context.Context) (ii []*esIndex, err error) {
+	var (
+		esRsp *esapi.Response
+	)
+
+	ii = make([]*esIndex, 100)
+
+	esc := m.esClient
+
+	esRsp, err = esc.Cat.Indices(
+		esc.Cat.Indices.WithContext(ctx),
+		esc.Cat.Indices.WithFormat("json"),
+	)
+	if err != nil {
+		return
+	}
+	defer esRsp.Body.Close()
+
+	return ii, json.NewDecoder(esRsp.Body).Decode(&ii)
+}
+
+// ConfigurationMapping ...
+func (m *mapper) ConfigurationMapping(ctx context.Context) (err error) {
+	var (
+		buf = &bytes.Buffer{}
+
+		// @todo better naming and ref please parent structs
+		configMap = reqMapping{
+			Mappings: struct {
+				Properties map[string]*property `json:"properties,omitempty"`
+			}(struct{ Properties map[string]*property }{Properties: map[string]*property{
+				"incrementalIndex": {
+					//Type: "object",
+					Properties: map[string]*property{
+						"updatedAt": {Type: "date"},
+					},
+				},
+			}}),
+		}
+	)
+	const (
+		CortezaConfigurationIndex = "corteza-configuration-test"
+	)
+
+	index := fmt.Sprintf(CortezaConfigurationIndex)
+	esc := m.esClient
+
+	if err = json.NewEncoder(buf).Encode(configMap); err != nil {
+		return
+	}
+
+	// Check whether a particular configuration index exists
+	res, err := esc.Indices.Exists([]string{index}, esc.Indices.Exists.WithContext(ctx))
+	if res != nil && err != nil && !res.IsError() {
+		if err != nil {
+			m.log.Error("Configuration index exist check failed", zap.Error(err))
+		} else {
+			// Put mapping if exist otherwise create it
+			if res, err := esc.Indices.Create(index, esc.Indices.Create.WithBody(buf)); res.IsError() || err != nil {
+				if err != nil {
+					m.log.Error("Configuration index creation failed", zap.Error(err))
+				}
+				if len(res.String()) > 0 {
+					m.log.Error(fmt.Sprintf("Configuration index creation failed due to %s", res.String()))
+				}
+			}
+
+			m.log.Info("Configuration index created")
+			return
+		}
+	}
+
+	m.log.Info("Configuration index exist")
+
+	return
+}
+
+// @todo: maybe add const for all es mapping property type
+// @todo: add helper methods for all Indices & use it instead of esc.Indices.

extra/server-discovery/pkg/es/mapping/mapping_test.go

@@ -0,0 +1,106 @@
+package mapping
+
+import (
+	"fmt"
+	"github.com/cortezaproject/corteza/server/pkg/options"
+	"os"
+	"strings"
+	"testing"
+)
+
+type (
+	config struct {
+		httpAddr string
+		es       struct {
+			addresses []string
+		}
+		cortezaAuth         string
+		cortezaDiscoveryAPI string
+		schemas             []*schema
+	}
+
+	schema struct {
+		indexPrefix  string
+		clientKey    string
+		clientSecret string
+	}
+)
+
+const (
+	envKeyHttpAddr = "HTTP_ADDR"
+)
+
+// @todo use it properly
+func TestMappings(t *testing.T) {
+	//var (
+	//	ctx = context.Background()
+	//	log = logger.Default()
+	//	req = require.New(t)
+	//)
+	//
+	//cfg, err := GetConfig()
+	//req.NoError(err)
+
+	//client, err := Es(cfg.es.addresses)
+	//req.NoError(err)
+	//
+	//api, err := indexer.ApiClient(cfg.cortezaDiscoveryAPI, cfg.cortezaAuth, cfg.schemas[0].clientKey, cfg.schemas[0].clientSecret)
+	//req.NoError(err)
+	//
+	//err = Mappings(ctx, log, client, api, "private")
+	//req.NoError(err)
+}
+
+// @todo reuse existing one
+func GetConfig() (*config, error) {
+	c := &config{}
+	return c, func() error {
+		baseUrl := options.EnvString("CORTEZA_SERVER_BASE_URL", "http://server:80")
+
+		c.httpAddr = options.EnvString(envKeyHttpAddr, "127.0.0.1:3201")
+
+		c.cortezaAuth = options.EnvString("CORTEZA_SERVER_AUTH", baseUrl+"/auth")
+		if c.cortezaAuth == "" {
+			return fmt.Errorf("corteza Auth endpoint value empty, set it directly with CORTEZA_SERVER_AUTH or indirectly with CORTEZA_SERVER_BASE_URL")
+		}
+
+		c.cortezaDiscoveryAPI = options.EnvString("CORTEZA_SERVER_API_DISCOVERY", baseUrl+"/api/discovery")
+		if c.cortezaDiscoveryAPI == "" {
+			return fmt.Errorf("corteza Discovery API endpoint value empty, set it directly with CORTEZA_SERVER_AUTH or indirectly with CORTEZA_SERVER_API_DISCOVERY")
+		}
+
+		for _, ar := range []string{"public", "protected", "private"} {
+			var (
+				has  bool
+				ucAr = strings.ToUpper(ar)
+				s    = &schema{indexPrefix: ar}
+
+				keyEnv = ucAr + "_INDEX_CLIENT_KEY"
+				secEnv = ucAr + "_INDEX_CLIENT_SECRET"
+			)
+
+			if s.clientKey, has = os.LookupEnv(keyEnv); !has {
+				continue
+			} else if s.clientKey == "" {
+				return fmt.Errorf("client key (%s) for '%s' is empty or missing", keyEnv, s.indexPrefix)
+			}
+
+			if s.clientSecret = os.Getenv(secEnv); s.clientSecret == "" {
+				return fmt.Errorf("client secret (%s) for '%s' is empty or missing", secEnv, s.indexPrefix)
+			}
+
+			c.schemas = append(c.schemas, s)
+		}
+
+		if len(c.schemas) == 0 {
+			return fmt.Errorf("set at least one client secret pair using <PREFIX>_INDEX_CLIENT_KEY and <PREFIX>_INDEX_CLIENT_SECRET where prefix is one of 'public', 'protected' or 'private'")
+		}
+
+		for _, a := range strings.Split(options.EnvString("ES_ADDRESS", "http://es:9200"), " ") {
+			if a = strings.TrimSpace(a); a != "" {
+				c.es.addresses = append(c.es.addresses, a)
+			}
+		}
+		return nil
+	}()
+}

extra/server-discovery/pkg/es/reindex/reindex.go

@@ -0,0 +1,972 @@
+package reindex
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/cortezaproject/corteza/extra/server-discovery/pkg/options"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/elastic/go-elasticsearch/v7/esutil"
+	"github.com/jmoiron/sqlx/types"
+	"go.uber.org/zap"
+)
+
+type (
+	docsSources struct {
+		endpoint string
+		index    string
+		action   string
+		params   map[string]string
+		callback func(*document)
+	}
+
+	rspDiscoveryDocuments struct {
+		Error *struct {
+			Message string
+		}
+		Response *struct {
+			Filter struct {
+				NextPage string
+			}
+
+			Documents []*document
+		}
+	}
+
+	// auxiliary struct for parsing indexable documents from Discovery API
+	document struct {
+		ID     string
+		Index  string
+		Source json.RawMessage
+	}
+
+	feedSources struct {
+		endpoint string
+		index    string
+		action   string
+		params   map[string]string
+		callback func(*document)
+	}
+
+	feedResponse struct {
+		Error *struct {
+			Message string
+		}
+		Response *struct {
+			Filter struct {
+				NextPage string
+			}
+
+			ActivityLogs []ActivityLog `json:"activityLogs"`
+		}
+	}
+
+	ActivityLog struct {
+		ID             uint64         `json:"activityID,string"`
+		ResourceID     uint64         `json:"resourceID,string"`
+		ResourceType   string         `json:"resourceType"`
+		ResourceAction string         `json:"resourceAction"`
+		Timestamp      time.Time      `json:"timestamp"`
+		Meta           types.JSONText `json:"meta"`
+	}
+
+	ActivityLogMeta struct {
+		NamespaceID uint64 `json:"namespaceID,string"`
+		ModuleID    uint64 `json:"moduleID,string"`
+	}
+
+	esService interface {
+		BulkIndexer() (esutil.BulkIndexer, error)
+	}
+
+	embedderService interface {
+		GenerateEmbeddings(input string) ([]float64, error)
+	}
+
+	apiClientService interface {
+		HttpClient() *http.Client
+		Mappings() (*http.Request, error)
+		Feed(url.Values) (*http.Request, error)
+		Resources(string, url.Values) (*http.Request, error)
+		Request(string) (*http.Request, error)
+		Authenticate() error
+	}
+
+	reIndexer struct {
+		log            *zap.Logger
+		esOpt          options.EsOpt
+		es             esService
+		esClient       *elasticsearch.Client
+		api            apiClientService
+		assureMappings func(context.Context) error
+	}
+
+	NamespaceRef struct {
+		NamespaceID string `json:"namespaceID,omitempty"`
+		Name        string `json:"name,omitempty"`
+	}
+
+	ModuleRef struct {
+		ModuleID string `json:"moduleID,omitempty"`
+		Name     string `json:"name,omitempty"`
+		Handle   string `json:"handle,omitempty"`
+	}
+
+	TimestampInfo struct {
+		At time.Time `json:"at,omitempty"`
+		By *User     `json:"by,omitempty"`
+	}
+
+	User struct {
+		UserID string `json:"userID,omitempty"`
+		Email  string `json:"email,omitempty"`
+		Name   string `json:"name,omitempty"`
+		Handle string `json:"handle,omitempty"`
+	}
+
+	SecurityRule struct {
+		AllowedRoles []string `json:"allowedRoles,omitempty"`
+		DeniedRoles  []string `json:"deniedRoles,omitempty"`
+	}
+
+	ComposeRecord struct {
+		ResourceType string            `json:"resourceType"`
+		RecordID     string            `json:"recordID"`
+		ValueLabels  map[string]string `json:"valueLabels,omitempty"`
+		Values       map[string]any    `json:"values,omitempty"`
+		CatchAll     []any             `json:"catch_all,omitempty"`
+		Updated      *TimestampInfo    `json:"updated,omitempty"`
+		Created      *TimestampInfo    `json:"created,omitempty"`
+		Deleted      *TimestampInfo    `json:"deleted,omitempty"`
+		Security     []SecurityRule    `json:"security,omitempty"`
+		Namespace    *NamespaceRef     `json:"namespace,omitempty"`
+		Module       *ModuleRef        `json:"module,omitempty"`
+		VectorsValue []float64         `json:"vectorsValue,omitempty"`
+	}
+)
+
+const (
+	IndexTpl    = "corteza-%s-%s"
+	MarkerIndex = "corteza_indexer_state"
+	MarkerID    = "last_index_time"
+)
+
+func ReIndexer(log *zap.Logger, es esService, esc *elasticsearch.Client, api apiClientService, esOpt options.EsOpt, assureMappings func(context.Context) error) *reIndexer {
+	return &reIndexer{
+		log:            log,
+		esOpt:          esOpt,
+		es:             es,
+		esClient:       esc,
+		api:            api,
+		assureMappings: assureMappings,
+	}
+}
+
+func (ri *reIndexer) ReindexAll(ctx context.Context, esb esutil.BulkIndexer, indexPrefix string) error {
+	var (
+		srcQueue = make(chan *docsSources, 100)
+		bErr     = ri.reindexManager(ctx, esb, indexPrefix, srcQueue)
+	)
+
+	srcQueue <- &docsSources{
+		endpoint: "/system/users",
+		index:    "system-users",
+	}
+
+	postProcModules := func(namespaceID string) func(d *document) {
+		return func(d *document) {
+			srcQueue <- &docsSources{
+				endpoint: fmt.Sprintf("/compose/namespaces/%s/modules/%s/records", namespaceID, d.ID),
+				index:    fmt.Sprintf("compose-records-%s-%s", namespaceID, d.ID),
+			}
+		}
+	}
+
+	postProcNamespaces := func(d *document) {
+		srcQueue <- &docsSources{
+			endpoint: fmt.Sprintf("/compose/namespaces/%s/modules", d.ID),
+			index:    "compose-modules",
+			callback: postProcModules(d.ID),
+		}
+	}
+
+	_ = postProcModules
+	_ = postProcNamespaces
+
+	srcQueue <- &docsSources{
+		endpoint: "/compose/namespaces",
+		index:    "compose-namespaces",
+		callback: postProcNamespaces,
+	}
+	_ = fmt.Errorf("blocking error")
+	return <-bErr
+}
+
+func (ri *reIndexer) reindexManager(ctx context.Context, esb esutil.BulkIndexer, indexPrefix string, srcQueue chan *docsSources) chan error {
+	var qErr = make(chan error)
+	const maxQueueLen = 3
+
+	go func(esb esutil.BulkIndexer) {
+		var (
+			pQueueLen        = -1
+			pQueueStaleCount int
+
+			ticker = time.NewTicker(time.Second)
+		)
+
+		defer ticker.Stop()
+		defer func() {
+			qErr <- nil
+		}()
+
+		for {
+			select {
+			case <-ctx.Done():
+				if ctx.Err() != context.Canceled {
+					ri.log.Error(ctx.Err().Error())
+				} else {
+					ri.log.Info("stopped")
+				}
+				return
+
+			case ds := <-srcQueue:
+				if ds == nil {
+					// graceful termination
+					ri.log.Info("done")
+					return
+				}
+
+				err := ri.reindex(ctx, esb, indexPrefix, ds)
+				if err != nil {
+					ri.log.Error("failed to reindex", zap.Error(err), zap.String("endpoint", ds.endpoint))
+					return
+				}
+
+			case <-ticker.C:
+				if pQueueLen != len(srcQueue) {
+					pQueueStaleCount = maxQueueLen
+				} else {
+					pQueueStaleCount--
+				}
+
+				if pQueueStaleCount <= 0 {
+					ri.log.Info("idle")
+					return
+				}
+
+				pQueueLen = len(srcQueue)
+
+				//esb, err := ri.es.BulkIndexer()
+				//if err != nil {
+				//	qErr <- err
+				//}
+
+				s := esb.Stats()
+				ri.log.Debug("batch indexing stats",
+					zap.Uint64("added", s.NumAdded),
+					zap.Uint64("created", s.NumCreated),
+					zap.Uint64("updated", s.NumUpdated),
+					zap.Uint64("deleted", s.NumDeleted),
+					zap.Uint64("flushed", s.NumFlushed),
+					zap.Uint64("failed", s.NumFailed),
+					zap.Uint64("indexed", s.NumIndexed),
+					zap.Uint64("requests", s.NumRequests),
+					zap.Int("queue length", pQueueLen),
+				)
+			}
+		}
+	}(esb)
+
+	println("returning")
+	return qErr
+}
+
+func (ri *reIndexer) reindex(ctx context.Context, esb esutil.BulkIndexer, indexPrefix string, ds *docsSources) (err error) {
+	var (
+		qs     = url.Values{"limit": []string{"500"}}
+		req    *http.Request
+		rsp    *http.Response
+		cursor string
+	)
+
+	//esb, err := ri.es.BulkIndexer()
+	//if err != nil {
+	//	return fmt.Errorf("failed to prepare bulk indexer: %w", err)
+	//}
+
+	for {
+		rspPayload := &rspDiscoveryDocuments{}
+
+		if cursor != "" {
+			// set new cursor and update source URL
+			qs.Set("pageCursor", cursor)
+		}
+
+		if req, err = ri.api.Resources(ds.endpoint, qs); err != nil {
+			return fmt.Errorf("failed to prepare resource request: %w", err)
+		}
+
+		if rsp, err = ri.api.HttpClient().Do(req.WithContext(ctx)); err != nil {
+			return fmt.Errorf("failed to send request: %w", err)
+		}
+
+		if rsp.StatusCode != http.StatusOK {
+			return fmt.Errorf("request resulted in an unexpected status '%s' for url '%s'", rsp.Status, req.URL)
+		}
+
+		//{
+		//	d, err := httputil.DumpRequestOut(req, true)
+		//	println(string(d))
+		//	spew.Dump(err)
+		//}
+		//{
+		//	d, err := httputil.DumpResponse(rsp, true)
+		//	println(string(d))
+		//	spew.Dump(err)
+		//}
+
+		if err = json.NewDecoder(rsp.Body).Decode(rspPayload); err != nil {
+			return fmt.Errorf("failed to decode reindexing response: %w", err)
+		}
+
+		if err = rsp.Body.Close(); err != nil {
+			return fmt.Errorf("failed to close reindexing response body: %w", err)
+		}
+
+		var docs int
+		if rspPayload.Error != nil {
+			ri.log.Debug("skipping",
+				zap.String("index", fmt.Sprintf(IndexTpl, indexPrefix, ds.index)),
+				zap.String("error", rspPayload.Error.Message),
+			)
+			return
+		} else if rspPayload.Response != nil {
+			docs = len(rspPayload.Response.Documents)
+		}
+
+		ri.log.Debug("reindexing",
+			zap.Int("docs", docs),
+			zap.String("index", fmt.Sprintf(IndexTpl, indexPrefix, ds.index)),
+		)
+
+		if docs == 0 {
+			return
+		}
+
+		for _, doc := range rspPayload.Response.Documents {
+			body, err := ri.processResource(doc.Source)
+			if err != nil {
+				ri.log.Error("failed to process record's for embeddings: ", zap.Error(err))
+			}
+
+			err = esb.Add(ctx, esutil.BulkIndexerItem{
+				Index:      fmt.Sprintf(IndexTpl, indexPrefix, ds.index),
+				Action:     "index",
+				DocumentID: doc.ID,
+				Body:       body,
+				OnFailure: func(ctx context.Context, req esutil.BulkIndexerItem, rsp esutil.BulkIndexerResponseItem, err error) {
+					spew.Dump(req)
+					spew.Dump(rsp)
+					spew.Dump(err)
+				},
+			})
+
+			if err != nil {
+				return err
+			}
+
+			if ds.callback != nil {
+				go ds.callback(doc)
+			}
+		}
+
+		cursor = rspPayload.Response.Filter.NextPage
+		if rspPayload.Response.Filter.NextPage == "" {
+			break
+		}
+	}
+
+	//if err = esb.Close(ctx); err != nil {
+	//	return fmt.Errorf("failed to close bulk indexer: %w", err)
+	//}
+
+	return nil
+}
+
+func (ri *reIndexer) feedReindex(ctx context.Context, esb esutil.BulkIndexer, indexPrefix string, ds *feedSources) (err error) {
+	var (
+		qs     = url.Values{"limit": []string{"500"}}
+		req    *http.Request
+		rsp    *http.Response
+		cursor string
+	)
+
+	if ds == nil {
+		ri.log.Debug("invalid resource for feed update")
+		return
+	}
+
+	err = ri.assureMappings(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to assure mappings: %w", err)
+	}
+
+	for {
+		rspPayload := &rspDiscoveryDocuments{}
+
+		if cursor != "" {
+			// set new cursor and update source URL
+			qs.Set("pageCursor", cursor)
+		}
+
+		if ds.params != nil {
+			var (
+				val string
+				ok  bool
+			)
+
+			val, ok = ds.params["userID"]
+			if ok {
+				qs.Set("userID", val)
+			}
+
+			val, ok = ds.params["namespaceID"]
+			if ok {
+				qs.Set("namespaceID", val)
+			}
+
+			val, ok = ds.params["moduleID"]
+			if ok {
+				qs.Set("moduleID", val)
+			}
+
+			val, ok = ds.params["recordID"]
+			if ok {
+				qs.Set("recordID", val)
+			}
+
+			val, ok = ds.params["deleted"]
+			if ok {
+				qs.Set("deleted", val)
+			}
+		}
+
+		if req, err = ri.api.Resources(ds.endpoint, qs); err != nil {
+			return fmt.Errorf("failed to prepare resource request: %w", err)
+		}
+
+		if rsp, err = ri.api.HttpClient().Do(req.WithContext(ctx)); err != nil {
+			return fmt.Errorf("failed to send request: %w", err)
+		}
+
+		if rsp.StatusCode != http.StatusOK {
+			return fmt.Errorf("request resulted in an unexpected status '%s' for url '%s'", rsp.Status, req.URL)
+		}
+
+		//{
+		//	d, err := httputil.DumpRequestOut(req, true)
+		//	println(string(d))
+		//	spew.Dump(err)
+		//}
+		//{
+		//	d, err := httputil.DumpResponse(rsp, true)
+		//	println(string(d))
+		//	spew.Dump(err)
+		//}
+
+		if err = json.NewDecoder(rsp.Body).Decode(rspPayload); err != nil {
+			return fmt.Errorf("failed to decode reindexing response: %w", err)
+		}
+
+		if err = rsp.Body.Close(); err != nil {
+			return fmt.Errorf("failed to close reindexing response body: %w", err)
+		}
+
+		var (
+			docs     int
+			docIndex = fmt.Sprintf(IndexTpl, indexPrefix, ds.index)
+		)
+		if rspPayload.Error != nil {
+			ri.log.Debug("skipping",
+				zap.String("index", docIndex),
+				zap.String("error", rspPayload.Error.Message),
+			)
+			return
+		} else if rspPayload.Response != nil {
+			docs = len(rspPayload.Response.Documents)
+		}
+
+		ri.log.Debug("feed reindexing",
+			zap.Int("docs", docs),
+			zap.String("index", docIndex),
+		)
+
+		if docs == 0 {
+			return
+		}
+
+		action := "index"
+		if len(ds.action) > 0 {
+			action = ds.action
+		}
+
+		for _, doc := range rspPayload.Response.Documents {
+			esbItem := esutil.BulkIndexerItem{
+				Index:      docIndex,
+				Action:     action,
+				DocumentID: doc.ID,
+				OnFailure: func(ctx context.Context, req esutil.BulkIndexerItem, rsp esutil.BulkIndexerResponseItem, err error) {
+					spew.Dump(req)
+					spew.Dump(rsp)
+					spew.Dump(err)
+				},
+			}
+			if action != "delete" {
+				esbItem.Action = "index"
+				body, err := ri.processResource(doc.Source)
+				if err != nil {
+					return err
+				}
+
+				esbItem.Body = body
+			}
+
+			err = esb.Add(ctx, esbItem)
+			if err != nil {
+				return err
+			}
+
+			if ds.callback != nil {
+				go ds.callback(doc)
+			}
+		}
+
+		cursor = rspPayload.Response.Filter.NextPage
+		if rspPayload.Response.Filter.NextPage == "" {
+			break
+		}
+	}
+
+	return nil
+}
+
+func (ri *reIndexer) feedReindexManager(ctx context.Context, esb esutil.BulkIndexer, indexPrefix string, feedQueue chan *feedSources) chan error {
+	var qErr = make(chan error)
+	const maxQueueLen = 3
+
+	go func(esb esutil.BulkIndexer) {
+		var (
+			pQueueLen        = -1
+			pQueueStaleCount int
+
+			ticker = time.NewTicker(time.Second)
+		)
+
+		defer ticker.Stop()
+		defer func() {
+			qErr <- nil
+		}()
+
+		for {
+			select {
+			case <-ctx.Done():
+				if ctx.Err() != context.Canceled {
+					ri.log.Error(ctx.Err().Error())
+				} else {
+					ri.log.Info("feed changes stopped")
+				}
+				return
+
+			case ds := <-feedQueue:
+				if ds == nil {
+					// graceful termination
+					ri.log.Info("feed changes done")
+					return
+				}
+
+				err := ri.feedReindex(ctx, esb, indexPrefix, ds)
+				if err != nil {
+					ri.log.Error("failed to reindex", zap.Error(err), zap.String("endpoint", ds.endpoint))
+					return
+				}
+
+			case <-ticker.C:
+				if pQueueLen != len(feedQueue) {
+					pQueueStaleCount = maxQueueLen
+				} else {
+					pQueueStaleCount--
+				}
+
+				if pQueueStaleCount <= 0 {
+					ri.log.Info("feed changes idle")
+					return
+				}
+
+				pQueueLen = len(feedQueue)
+
+				s := esb.Stats()
+				ri.log.Debug("feed batch indexing stats",
+					zap.Uint64("added", s.NumAdded),
+					zap.Uint64("created", s.NumCreated),
+					zap.Uint64("updated", s.NumUpdated),
+					zap.Uint64("deleted", s.NumDeleted),
+					zap.Uint64("flushed", s.NumFlushed),
+					zap.Uint64("failed", s.NumFailed),
+					zap.Uint64("indexed", s.NumIndexed),
+					zap.Uint64("requests", s.NumRequests),
+					zap.Int("queue length", pQueueLen),
+				)
+			}
+		}
+	}(esb)
+
+	println("feed changes returning")
+	return qErr
+}
+
+func (ri *reIndexer) feedReindexChanges(ctx context.Context, esb esutil.BulkIndexer, indexPrefix string, als []ActivityLog) error {
+	var (
+		updateQueue = make(chan *feedSources, 100)
+		bErr        = ri.feedReindexManager(ctx, esb, indexPrefix, updateQueue)
+
+		duplicateMap = make(map[string]ActivityLog)
+	)
+
+	for _, al := range als {
+		activityType := fmt.Sprintf("%d-%s", al.ResourceID, al.ResourceAction)
+		if val, ok := duplicateMap[activityType]; ok {
+			continue
+		} else {
+			duplicateMap[activityType] = val
+		}
+
+		action := al.ResourceAction
+		switch al.ResourceType {
+		case "system:user":
+			updateQueue <- &feedSources{
+				endpoint: "/system/users",
+				index:    "system-users",
+				action:   action,
+				params: map[string]string{
+					"userID":  fmt.Sprintf("%d", al.ResourceID),
+					"deleted": "1",
+				},
+			}
+			break
+
+		case "compose:namespace":
+			updateQueue <- &feedSources{
+				endpoint: "/compose/namespaces",
+				index:    "compose-namespaces",
+				action:   action,
+				params: map[string]string{
+					"namespaceID": fmt.Sprintf("%d", al.ResourceID),
+					"deleted":     "1",
+				},
+			}
+			break
+
+		case "compose:module":
+			var meta ActivityLogMeta
+			err := al.Meta.Unmarshal(&meta)
+			if err != nil {
+				return err
+			}
+
+			updateQueue <- &feedSources{
+				endpoint: fmt.Sprintf("/compose/namespaces/%d/modules", meta.NamespaceID),
+				index:    "compose-modules",
+				action:   action,
+				params: map[string]string{
+					"moduleID": fmt.Sprintf("%d", al.ResourceID),
+					"deleted":  "1",
+				},
+			}
+			break
+
+		case "compose:record":
+			var meta ActivityLogMeta
+			err := al.Meta.Unmarshal(&meta)
+			if err != nil {
+				return err
+			}
+
+			updateQueue <- &feedSources{
+				endpoint: fmt.Sprintf("/compose/namespaces/%d/modules/%d/records", meta.NamespaceID, meta.ModuleID),
+				index:    fmt.Sprintf("compose-records-%d-%d", meta.NamespaceID, meta.ModuleID),
+				action:   action,
+				params: map[string]string{
+					"recordID": fmt.Sprintf("%d", al.ResourceID),
+					"deleted":  "1",
+				},
+			}
+			break
+
+		default:
+			break
+		}
+
+	}
+
+	return <-bErr
+}
+
+func (ri *reIndexer) Watch(ctx context.Context) {
+	var now time.Time
+
+	isFirst, err := ri.IsFirstRunWithMarker(ctx)
+	if err != nil {
+		ri.log.Warn(fmt.Sprintf("failed to check first run indexes marker: %s", err))
+		isFirst = false
+	}
+
+	if isFirst {
+		startTime := time.Date(2018, time.January, 1, 0, 0, 0, 0, time.UTC) // Corteza start year
+		if ri.esOpt.IndexBackFillMonths != 0 {
+			startTime = time.Now().AddDate(0, -ri.esOpt.IndexBackFillMonths, 0)
+		}
+
+		ri.log.Info(fmt.Sprintf("first run detected: starting from (%s)", startTime.UTC().Format(time.RFC3339)))
+
+		if err := ri.processBacklogInChunks(ctx, startTime, &now); err != nil {
+			ri.log.Error(fmt.Sprintf("backlog processing failed: %s", err))
+		}
+
+		defer func() {
+			if err := ri.SaveLastIndexTime(ctx); err != nil {
+				ri.log.Error(fmt.Sprintf("failed to save indexer state to [corteza_indexer_state]: %s", err))
+			}
+		}()
+	}
+
+	now = time.Now()
+	ri.log.Info("continuing reindexing from current time")
+
+	timeOut := ri.esOpt.IndexInterval
+	ticker := time.NewTicker(time.Second * time.Duration(timeOut))
+
+	go func() {
+		defer ticker.Stop()
+		processing := false
+
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				if processing {
+					ri.log.Warn("skipping feed changes reindexing: already processing")
+					continue
+				}
+
+				processing = true
+				go func() {
+					defer func() {
+						processing = false
+					}()
+
+					tmpTime := time.Now()
+
+					if err := ri.fetchAndIndex(ctx, now, nil); err != nil {
+						ri.log.Error(fmt.Sprintf("reindex error: %s", err))
+						return
+					}
+
+					now = tmpTime
+				}()
+			}
+		}
+	}()
+}
+
+func (ri *reIndexer) processResource(source []byte) (*bytes.Reader, error) {
+	var peekResource struct {
+		ResourceType string `json:"resourceType"`
+	}
+
+	if err := json.Unmarshal(source, &peekResource); err != nil {
+		return nil, fmt.Errorf("error unmarshaling: %v", err)
+	}
+
+	if peekResource.ResourceType != "compose:record" {
+		return bytes.NewReader(source), nil
+	}
+
+	// process record's resource
+	var record ComposeRecord
+	if err := json.Unmarshal(source, &record); err != nil {
+		return nil, fmt.Errorf("error unmarshaling record: %v", err)
+	}
+
+	updatedJSON, err := json.Marshal(record)
+	if err != nil {
+		return nil, fmt.Errorf("error marshaling: %v", err)
+	}
+
+	return bytes.NewReader(updatedJSON), nil
+}
+
+func (ri *reIndexer) IsFirstRunWithMarker(ctx context.Context) (bool, error) {
+	res, err := ri.esClient.Get(
+		MarkerIndex,
+		MarkerID,
+		ri.esClient.Get.WithContext(ctx),
+	)
+
+	// create the index and the fields.
+
+	if err != nil {
+		return false, err
+	}
+
+	defer res.Body.Close()
+
+	if res.IsError() {
+		// marker index does'nt exist
+		if res.StatusCode == http.StatusNotFound {
+			return true, err
+		}
+	}
+
+	return false, nil
+}
+
+func (ri *reIndexer) SaveLastIndexTime(ctx context.Context) error {
+	doc := map[string]interface{}{
+		"last_run":    time.Now().UTC().Format(time.RFC3339),
+		"initialized": true,
+	}
+
+	docBytes, err := json.Marshal(doc)
+	if err != nil {
+		return fmt.Errorf("failed to marshal marker document: %w", err)
+	}
+
+	// create/update the marker document
+	res, err := ri.esClient.Index(
+		MarkerIndex,
+		bytes.NewReader(docBytes),
+		ri.esClient.Index.WithContext(ctx),
+		ri.esClient.Index.WithDocumentID(MarkerID),
+		ri.esClient.Index.WithRefresh("true"),
+	)
+
+	if err != nil {
+		return fmt.Errorf("failed to save marker index document: %w", err)
+	}
+
+	defer res.Body.Close()
+
+	if res.IsError() {
+		return fmt.Errorf("failed to save marker index document: %s", res.Status())
+	}
+
+	return nil
+}
+
+func (ri *reIndexer) fetchAndIndex(ctx context.Context, fromTime time.Time, toTime *time.Time) error {
+	esb, err := ri.es.BulkIndexer()
+	if err != nil {
+		return err
+	}
+	defer esb.Close(ctx)
+
+	feeds, err := ri.fetchSinglePage(ctx, fromTime, toTime, "")
+	if err != nil {
+		return err
+	}
+
+	if feeds != nil && feeds.Response != nil && len(feeds.Response.ActivityLogs) > 0 {
+		ri.log.Info(fmt.Sprintf("indexing %d records", len(feeds.Response.ActivityLogs)))
+		return ri.feedReindexChanges(ctx, esb, "private", feeds.Response.ActivityLogs)
+	} else {
+		ri.log.Debug(fmt.Sprintf("no feed changes since %s; current time: %s", fromTime.UTC().String(), time.Now().UTC().String()))
+	}
+
+	return nil
+}
+
+func (ri *reIndexer) fetchSinglePage(ctx context.Context, fromTime time.Time, toTime *time.Time, cursor string) (*feedResponse, error) {
+	qs := url.Values{
+		"from":  []string{fromTime.UTC().Format(time.RFC3339)},
+		"limit": []string{"500"},
+	}
+	if toTime != nil {
+		qs.Set("to", toTime.UTC().Format(time.RFC3339))
+	}
+	if cursor != "" {
+		qs.Set("cursor", cursor)
+	}
+
+	req, err := ri.api.Feed(qs)
+	if err != nil {
+		ri.log.Error(fmt.Sprintf("failed to prepare feed request: %s", err))
+		return nil, err
+	}
+
+	rsp, err := ri.api.HttpClient().Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer rsp.Body.Close()
+
+	if rsp.StatusCode != http.StatusOK {
+		ri.log.Error(fmt.Sprintf("request resulted in an unexpected status '%s' for feed", rsp.Status))
+		return nil, fmt.Errorf("unexpected status: %s", rsp.Status)
+	}
+
+	feeds := &feedResponse{}
+	if err = json.NewDecoder(rsp.Body).Decode(feeds); err != nil {
+		ri.log.Error(fmt.Sprintf("failed to decode feed response: %s", err))
+		return nil, err
+	}
+
+	return feeds, nil
+}
+
+func (ri *reIndexer) processBacklogInChunks(ctx context.Context, fromTime time.Time, toTime *time.Time) error {
+	esb, err := ri.es.BulkIndexer()
+	if err != nil {
+		return err
+	}
+	defer esb.Close(ctx)
+
+	cursor := ""
+	totalIndexed := 0
+
+	for {
+		feeds, err := ri.fetchSinglePage(ctx, fromTime, toTime, cursor)
+		if err != nil {
+			return err
+		}
+
+		if feeds != nil && feeds.Response != nil && len(feeds.Response.ActivityLogs) > 0 {
+			ri.log.Info(fmt.Sprintf("indexing %d records (page)", len(feeds.Response.ActivityLogs)))
+			if err := ri.feedReindexChanges(ctx, esb, "private", feeds.Response.ActivityLogs); err != nil {
+				return fmt.Errorf("failed to index records: %w", err)
+			}
+			totalIndexed += len(feeds.Response.ActivityLogs)
+		}
+
+		if feeds == nil || feeds.Response == nil || feeds.Response.Filter.NextPage == "" {
+			break
+		}
+
+		cursor = feeds.Response.Filter.NextPage
+		ri.log.Debug(fmt.Sprintf("fetching next page with cursor: %s", cursor))
+	}
+
+	if totalIndexed > 0 {
+		ri.log.Info(fmt.Sprintf("completed indexing %d total records for time chunk", totalIndexed))
+	} else {
+		ri.log.Debug(fmt.Sprintf("no feed changes since %s; current time: %s", fromTime.UTC().String(), time.Now().UTC().String()))
+	}
+
+	return nil
+}