snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

extra/server-discovery/pkg/api/client.go

@@ -0,0 +1,205 @@
+package api
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/cortezaproject/corteza/extra/server-discovery/pkg/options"
+)
+
+type (
+	credentials struct {
+		AccessToken string `json:"access_token"`
+		ExpiresIn   int    `json:"expires_in"`
+		expiresAt   time.Time
+
+		authBaseUri string
+		key         string
+		secret      string
+	}
+
+	client struct {
+		baseUri          string
+		discoveryBaseUrl string
+		credentials      *credentials
+	}
+
+	ClientService interface {
+		HttpClient() *http.Client
+		Mappings() (*http.Request, error)
+		Feed(url.Values) (*http.Request, error)
+		Resources(string, url.Values) (*http.Request, error)
+		Namespaces() (*http.Request, error)
+		Modules(uint64) (*http.Request, error)
+		Request(string) (*http.Request, error)
+		Authenticate() error
+	}
+)
+
+func Client(opt options.CortezaOpt, key, secret string) (c *client, err error) {
+	c = &client{baseUri: opt.BaseUrl, discoveryBaseUrl: opt.DiscoveryUrl}
+	c.credentials = &credentials{
+		authBaseUri: opt.AuthUrl,
+		key:         key,
+		secret:      secret,
+	}
+	return c, err
+}
+
+func (*client) HttpClient() *http.Client {
+	return http.DefaultClient
+}
+
+func (c *client) Mappings() (*http.Request, error) {
+	return c.Request(fmt.Sprintf("%s/mappings/", c.discoveryBaseUrl))
+}
+
+func (c *client) Feed(qs url.Values) (*http.Request, error) {
+	query := ""
+	if len(qs.Encode()) > 0 {
+		query = fmt.Sprintf("from=%s", qs.Get("from"))
+	}
+	return c.Request(fmt.Sprintf("%s/feed/?%s", c.discoveryBaseUrl, query))
+}
+
+func (c *client) Resources(endpoint string, qs url.Values) (*http.Request, error) {
+	return c.Request(fmt.Sprintf("%s/resources/%s?%s", c.discoveryBaseUrl, strings.TrimLeft(endpoint, "/"), qs.Encode()))
+}
+
+func (c *client) Namespaces() (*http.Request, error) {
+	return c.Request(fmt.Sprintf("%s/api/compose/namespace/", c.baseUri))
+}
+
+func (c *client) Modules(namespaceID uint64) (*http.Request, error) {
+	return c.Request(fmt.Sprintf("%s/api/compose/namespace/%d/module/?sort=name+ASC", c.baseUri, namespaceID))
+}
+
+func (c *client) Request(endpoint string) (req *http.Request, err error) {
+	if err = c.Authenticate(); err != nil {
+		return
+	}
+
+	if req, err = http.NewRequest(http.MethodGet, endpoint, nil); err != nil {
+		return
+	}
+
+	req.Header.Set("User-Agent", "corteza-server-discovery/0.1")
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", c.credentials.AccessToken))
+	return
+}
+
+func (c *client) Authenticate() (err error) {
+	if c.credentials == nil {
+		return fmt.Errorf("missing credentials")
+	}
+
+	tokenNeedsRefresh := c.credentials.expiresAt.IsZero() || time.Now().After(c.credentials.expiresAt.Add(-60*time.Second))
+
+	if tokenNeedsRefresh {
+		newCredentials, err := c.authToken()
+		if err != nil {
+			return fmt.Errorf("failed to refresh token: %w", err)
+		}
+
+		c.credentials = newCredentials
+	}
+
+	return nil
+}
+
+func (c *client) authToken() (crd *credentials, err error) {
+	var (
+		req         *http.Request
+		rsp         *http.Response
+		form        = url.Values{}
+		authBaseUri = c.credentials.authBaseUri
+		key         = c.credentials.key
+		secret      = c.credentials.secret
+	)
+
+	form.Set("grant_type", "client_credentials")
+	form.Set("scope", "profile api discovery")
+
+	req, err = http.NewRequest(http.MethodPost, authBaseUri+"/oauth2/token", strings.NewReader(form.Encode()))
+	if err != nil {
+		return
+	}
+
+	req.SetBasicAuth(key, secret)
+	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+
+	//d, _ := httputil.DumpRequest(req, true)
+	//println(string(d))
+
+	rsp, err = c.HttpClient().Do(req)
+	if err != nil {
+		return
+	}
+
+	defer rsp.Body.Close()
+	crd = &credentials{
+		authBaseUri: authBaseUri,
+		key:         key,
+		secret:      secret,
+	}
+
+	if rsp.StatusCode != http.StatusOK {
+		aux := struct{ Error string }{}
+		if err = json.NewDecoder(rsp.Body).Decode(&aux); err != nil {
+			return
+		} else if aux.Error != "" {
+			return nil, fmt.Errorf(aux.Error)
+		} else {
+			return nil, fmt.Errorf("can not authenticate, unexpected error")
+		}
+
+	}
+
+	//d, _ := httputil.DumpResponse(rsp, true)
+	//println(string(d))
+
+	err = json.NewDecoder(rsp.Body).Decode(crd)
+	if err != nil {
+		return
+	}
+
+	crd.expiresAt, err = parseJWTExpiry(crd.AccessToken)
+	if err != nil {
+		crd.expiresAt = time.Now().Add(time.Second * time.Duration(crd.ExpiresIn))
+	}
+
+	return
+}
+
+func parseJWTExpiry(tokenString string) (time.Time, error) {
+	parts := strings.Split(tokenString, ".")
+	if len(parts) != 3 {
+		return time.Time{}, fmt.Errorf("invalid JWT format: expected 3 parts, got %d", len(parts))
+	}
+
+	payload, err := base64.RawURLEncoding.DecodeString(parts[1])
+	if err != nil {
+		return time.Time{}, fmt.Errorf("failed to decode JWT payload: %w", err)
+	}
+
+	// Parse exp claim
+	var claims struct {
+		Exp int64 `json:"exp"`
+	}
+
+	if err := json.Unmarshal(payload, &claims); err != nil {
+		return time.Time{}, fmt.Errorf("failed to unmarshal JWT claims: %w", err)
+	}
+
+	if claims.Exp == 0 {
+		return time.Time{}, fmt.Errorf("JWT missing exp claim")
+	}
+
+	return time.Unix(claims.Exp, 0), nil
+}

extra/server-discovery/pkg/api/server/cors.go

@@ -0,0 +1,33 @@
+package server
+
+import (
+	"net/http"
+
+	"github.com/go-chi/cors"
+)
+
+// Sets up default CORS rules to use as a middleware
+func handleCORS(next http.Handler) http.Handler {
+	return cors.New(cors.Options{
+		AllowedOrigins: []string{
+			"http://*",
+			"https://*",
+		},
+		AllowedMethods: []string{
+			http.MethodHead,
+			http.MethodGet,
+			http.MethodPost,
+			http.MethodPut,
+			http.MethodPatch,
+			http.MethodDelete,
+		},
+		AllowedHeaders: []string{
+			"Accept",
+			"Authorization",
+			"Content-Type",
+			"X-CSRF-ID",
+		},
+		AllowCredentials: true,
+		MaxAge:           300, // Maximum value not ignored by any of major browsers
+	}).Handler(next)
+}

extra/server-discovery/pkg/api/server/demux.go

@@ -0,0 +1,53 @@
+package server
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/go-chi/chi/v5"
+	"go.uber.org/atomic"
+)
+
+type (
+	// demux (demultiplexer) routes request to one of the underlying routers
+	// according to current state
+	demux struct {
+		state   *atomic.Uint32
+		routers map[uint32]chi.Router
+	}
+)
+
+var _ http.Handler = &demux{}
+
+func Demux(state uint32, r chi.Router) *demux {
+	return &demux{
+		state:   atomic.NewUint32(state),
+		routers: map[uint32]chi.Router{state: r},
+	}
+}
+
+func (d *demux) State(s uint32) {
+	d.state.Store(s)
+}
+
+func (d *demux) Router(s uint32, r chi.Router) {
+	d.routers[s] = r
+}
+
+func (d *demux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	defer panicRecovery(r.Context(), w)
+
+	var (
+		state          = d.state.Load()
+		router, exists = d.routers[state]
+	)
+
+	if !exists {
+		_, _ = fmt.Fprintf(w, "unconfigured request demultiplexor state (%d)", state)
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	router.ServeHTTP(w, r)
+
+}

extra/server-discovery/pkg/api/server/handlers.go

@@ -0,0 +1,106 @@
+package server
+
+import (
+	"github.com/cortezaproject/corteza/extra/server-discovery/pkg/auth"
+	"github.com/cortezaproject/corteza/extra/server-discovery/pkg/healthcheck"
+	"github.com/cortezaproject/corteza/extra/server-discovery/pkg/options"
+	"github.com/cortezaproject/corteza/server/pkg/api"
+	"github.com/cortezaproject/corteza/server/pkg/version"
+	"github.com/go-chi/chi/v5"
+	"go.uber.org/zap"
+	"net/http"
+	"path"
+	"strings"
+)
+
+// routes used when server is in waiting mode
+func waitingRoutes(log *zap.Logger, httpOpt options.HttpServerOpt) (r chi.Router) {
+	r = chi.NewRouter()
+	r.Use(handleCORS)
+
+	mountServiceHandlers(r, log, httpOpt, waiting)
+
+	return
+}
+
+// routes used when server in shutdown mode
+func shutdownRoutes() (r chi.Router) {
+	r = chi.NewRouter()
+
+	return
+}
+
+// routes used when in active mode
+func activeRoutes(log *zap.Logger, mountable []func(r chi.Router), envOpt options.EnvironmentOpt, httpOpt options.HttpServerOpt, searcherOpt options.SearcherOpt) (r chi.Router) {
+	r = chi.NewRouter()
+	r.Use(handleCORS)
+
+	r.Route("/"+strings.TrimPrefix(httpOpt.BaseUrl, "/"), func(r chi.Router) {
+		// Base middleware, CORS, RealIP, RequestID, context-logger
+		r.Use(BaseMiddleware(envOpt.IsProduction(), log)...)
+
+		// Verifies JWT in headers, cookies, ...
+		r.Use(auth.HttpTokenVerifier)
+
+		for _, mount := range mountable {
+			mount(r)
+		}
+
+	})
+
+	if httpOpt.BaseUrl != "/" {
+		r.Handle("/", http.RedirectHandler(httpOpt.BaseUrl, http.StatusTemporaryRedirect))
+	}
+
+	mountServiceHandlers(r, log, httpOpt, active)
+
+	return
+}
+
+func mountServiceHandlers(r chi.Router, log *zap.Logger, opt options.HttpServerOpt, state uint32) {
+	if opt.EnableVersionRoute {
+		mountVersionHandler(r, log, opt.BaseUrl)
+	}
+
+	if opt.EnableHealthcheckRoute {
+		mountHealthCheckHandler(r, log, opt.BaseUrl)
+	}
+}
+
+func mountVersionHandler(r chi.Router, log *zap.Logger, basePath string) {
+	var (
+		dPath   = "/version"
+		sPath   = path.Join(basePath, dPath)
+		handler = func(w http.ResponseWriter, r *http.Request) {
+			api.Send(w, r, struct {
+				BuildTime string `json:"buildTime"`
+				Version   string `json:"version"`
+			}{version.BuildTime, version.Version})
+		}
+	)
+
+	r.Get(dPath, handler)
+	log.Debug("version route enabled: " + dPath)
+
+	if dPath != sPath {
+		r.Get(sPath, handler)
+		log.Debug("version route enabled: " + sPath)
+	}
+
+}
+
+func mountHealthCheckHandler(r chi.Router, log *zap.Logger, basePath string) {
+	// default & sub path for health-check endpoint
+	var (
+		dPath = "/healthcheck"
+		sPath = path.Join(basePath, dPath)
+	)
+
+	r.Get(dPath, healthcheck.HttpHandler())
+	log.Debug("health check route enabled: " + sPath)
+
+	if dPath != sPath {
+		r.Get(sPath, healthcheck.HttpHandler())
+		log.Debug("health check route enabled: " + sPath)
+	}
+}

extra/server-discovery/pkg/api/server/logger.go

@@ -0,0 +1,76 @@
+package server
+
+import (
+	"github.com/cortezaproject/corteza/server/pkg/logger"
+	"github.com/go-chi/chi/v5/middleware"
+	"go.uber.org/zap"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// contextLogger middleware binds logger to request's context.
+//
+// This allows us to use logger from context (with requestID)
+// inside our (generated) handlers and controllers
+func contextLogger(log *zap.Logger) func(next http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			var requestID = middleware.GetReqID(req.Context())
+
+			w.Header().Add("X-Request-Id", requestID)
+
+			req = req.WithContext(logger.ContextWithValue(
+				req.Context(),
+				log.With(zap.String("requestID", requestID)).Named("rest"),
+			))
+
+			next.ServeHTTP(w, req)
+		})
+	}
+}
+
+// LogRequest middleware logs request details
+//
+// It uses logger from context, see contextLogger()
+func LogRequest(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		var remote = req.RemoteAddr
+		if l := strings.LastIndex(remote, ":"); l > -1 {
+			remote = remote[:l]
+		}
+
+		logger.ContextValue(req.Context()).Info(
+			"HTTP request "+req.Method+" "+req.URL.Path,
+			zap.String("method", req.Method),
+			zap.String("path", req.URL.Path),
+			zap.Int64("size", req.ContentLength),
+			zap.String("remote", remote),
+		)
+		next.ServeHTTP(w, req)
+	})
+}
+
+// LogResponse middleware logs response details
+//
+// It uses logger from context, see contextLogger()
+func LogResponse(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		wrapped := middleware.NewWrapResponseWriter(w, req.ProtoMajor)
+		t := time.Now()
+
+		defer func() {
+			logger.ContextValue(req.Context()).Info(
+				"HTTP response "+req.Method+" "+req.URL.Path,
+				zap.String("method", req.Method),
+				zap.String("path", req.URL.Path),
+				zap.Int("status", wrapped.Status()),
+				zap.Int("size", wrapped.BytesWritten()),
+				zap.Float64("duration", time.Since(t).Seconds()),
+			)
+		}()
+
+		next.ServeHTTP(wrapped, req)
+	})
+
+}

extra/server-discovery/pkg/api/server/middleware.go

@@ -0,0 +1,49 @@
+package server
+
+import (
+	"context"
+	"net/http"
+	"os"
+	"runtime/debug"
+
+	"github.com/cortezaproject/corteza/server/pkg/api"
+	"github.com/cortezaproject/corteza/server/pkg/locale"
+	"github.com/cortezaproject/corteza/server/pkg/logger"
+	"github.com/go-chi/chi/v5/middleware"
+	"go.uber.org/zap"
+)
+
+func BaseMiddleware(isProduction bool, log *zap.Logger) []func(http.Handler) http.Handler {
+	return []func(http.Handler) http.Handler{
+		handleCORS,
+		locale.DetectLanguage(locale.Global()),
+		middleware.RealIP,
+		api.RemoteAddrToContext,
+		middleware.RequestID,
+		api.DebugToContext(isProduction),
+		contextLogger(log),
+	}
+}
+
+func panicRecovery(ctx context.Context, w http.ResponseWriter) {
+	if err := recover(); err != nil {
+		log := logger.ContextValue(ctx)
+		if err, ok := err.(error); ok {
+			log = log.With(zap.Error(err))
+		} else {
+			log = log.With(zap.Any("recover-value", err))
+		}
+
+		log.Debug("crashed on http request", zap.ByteString("stack", debug.Stack()))
+
+		w.WriteHeader(500)
+
+		if _, has := os.LookupEnv("DEBUG_DUMP_STACK_IN_RESPONSE"); has {
+			// Provide nice call stack on endpoint when
+			// we crash
+			_, _ = w.Write(debug.Stack())
+		}
+
+		return
+	}
+}

extra/server-discovery/pkg/api/server/server.go

@@ -0,0 +1,107 @@
+package server
+
+import (
+	"context"
+	"net"
+	"net/http"
+
+	"github.com/cortezaproject/corteza/extra/server-discovery/pkg/options"
+	"github.com/go-chi/chi/v5"
+	"go.uber.org/zap"
+)
+
+type (
+	server struct {
+		log            *zap.Logger
+		httpOpt        options.HttpServerOpt
+		waitForOpt     options.WaitForOpt
+		environmentOpt options.EnvironmentOpt
+		searcherOpt    options.SearcherOpt
+		endpoints      []func(r chi.Router)
+
+		demux *demux
+	}
+)
+
+const (
+	waiting uint32 = iota
+	active
+	shutdown
+)
+
+// New initializes new HTTP server with special powers
+// Server is started as early as possible and with a special request handler
+// that demultiplexes request to one of the configured routers according to the server state.
+//
+// Waiting state
+// This is initial state that with some simple route handlers:
+//  - /version
+//  - /healthcheck
+//  - /healthcheck
+func New(log *zap.Logger, envOpt options.EnvironmentOpt, httpOpt options.HttpServerOpt, waitForOpt options.WaitForOpt, searcherOpt options.SearcherOpt) *server {
+	s := &server{
+		endpoints: make([]func(r chi.Router), 0),
+		log:       log.Named("http"),
+
+		environmentOpt: envOpt,
+		httpOpt:        httpOpt,
+		searcherOpt:    searcherOpt,
+		waitForOpt:     waitForOpt,
+	}
+
+	s.demux = Demux(waiting, waitingRoutes(s.log.Named("waiting"), s.httpOpt))
+	s.demux.Router(shutdown, shutdownRoutes())
+
+	return s
+}
+
+// Activate reconfigures server to use active routes
+func (s *server) Activate(mm ...func(chi.Router)) {
+	s.demux.Router(active, activeRoutes(s.log, mm, s.environmentOpt, s.httpOpt, s.searcherOpt))
+
+	s.log.Debug("entering active state")
+	s.demux.State(active)
+}
+
+// Shutdown reconfigures server to use shutdown routes
+func (s *server) Shutdown() {
+	s.log.Debug("entering shutdown state")
+	s.demux.State(shutdown)
+}
+
+func (s server) Serve(ctx context.Context) {
+	s.log.Info(
+		"starting HTTP server",
+
+		zap.String("path-prefix", s.httpOpt.BaseUrl),
+		zap.String("address", s.httpOpt.Addr),
+	)
+
+	listener, err := net.Listen("tcp", s.httpOpt.Addr)
+	if err != nil {
+		s.log.Error("cannot start server", zap.Error(err))
+		return
+	}
+
+	go func() {
+		srv := http.Server{
+			Handler: s.demux,
+
+			// use root context as server's base context and as a basis for
+			// context for all requests
+			// this enables us to send cancellation down to every request
+			BaseContext: func(listener net.Listener) context.Context { return ctx },
+		}
+		err = srv.Serve(listener)
+	}()
+	<-ctx.Done()
+
+	if err == nil {
+		err = ctx.Err()
+		if err == context.Canceled {
+			err = nil
+		}
+	}
+
+	s.log.Info("HTTP server stopped", zap.Error(err))
+}

extra/server-discovery/pkg/api/server/waitfor.go

@@ -0,0 +1,34 @@
+package server
+
+import (
+	"context"
+	"github.com/go-chi/chi/v5"
+	"go.uber.org/zap"
+	"net"
+	"net/http"
+	"os"
+)
+
+// WaitFor sets up a simple status page, delays execution and probes services
+func (s server) WaitFor(ctx context.Context) {
+	var (
+		err error
+	)
+
+	// Set up a simple HTTP server that will inform the impatient users
+	listener, err := net.Listen("tcp", s.httpOpt.Addr)
+	if err != nil {
+		s.log.Error("cannot start server", zap.Error(err))
+		os.Exit(1)
+	}
+	defer listener.Close()
+	go func() {
+		router := chi.NewRouter()
+		router.Get("/*", func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusPreconditionFailed)
+			w.Write([]byte("waiting for services..."))
+		})
+		_ = http.Serve(listener, router)
+	}()
+	<-ctx.Done()
+}

extra/server-discovery/pkg/auth/identity.go

@@ -0,0 +1,60 @@
+package auth
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+type (
+	identity struct {
+		id       uint64
+		memberOf []uint64
+	}
+)
+
+// Anonymous constructs and returns new anonymous identity with system anonymous roles
+func Anonymous() *identity {
+	return &identity{
+		// memberOf: AnonymousRoles().IDs(),
+	}
+}
+
+// Authenticated constructs and returns new authenticated identity with assigned roles + system authenticated roles
+func Authenticated(id uint64, rr ...uint64) *identity {
+	return &identity{
+		id: id,
+		// memberOf: append(rr, AuthenticatedRoles().IDs()...),
+	}
+}
+
+func (i identity) Identity() uint64 {
+	return i.id
+}
+
+func (i identity) Roles() []uint64 {
+	return i.memberOf
+}
+
+func (i identity) Valid() bool {
+	return i.id > 0
+}
+
+func (i identity) String() string {
+	return fmt.Sprintf("%d", i.Identity())
+}
+
+func ExtractFromSubClaim(sub string) (userID uint64, rr []uint64) {
+	parts := strings.Split(sub, " ")
+	rr = make([]uint64, len(parts)-1)
+	for p := range parts {
+		id, _ := strconv.ParseUint(parts[p], 10, 64)
+		if p == 0 {
+			userID = id
+		} else {
+			rr[p-1] = id
+		}
+	}
+
+	return
+}

extra/server-discovery/pkg/auth/identity_context.go

@@ -0,0 +1,13 @@
+package auth
+
+import (
+	"context"
+)
+
+type (
+	identityCtxKey struct{}
+)
+
+func SetIdentityToContext(ctx context.Context, identity Identifiable) context.Context {
+	return context.WithValue(ctx, identityCtxKey{}, identity)
+}

extra/server-discovery/pkg/auth/interfaces.go

@@ -0,0 +1,26 @@
+package auth
+
+type (
+	Identifiable interface {
+		Identity() uint64
+		Roles() []uint64
+		Valid() bool
+		String() string
+	}
+
+	//TokenGenerator interface {
+	//	Encode(i Identifiable, clientID uint64, scope ...string) (token []byte, err error)
+	//	Generate(ctx context.Context, i Identifiable, clientID uint64, scope ...string) (token []byte, err error)
+	//}
+
+	//TokenHandler interface {
+	//	TokenGenerator
+	//	HttpVerifier() func(http.Handler) http.Handler
+	//	HttpAuthenticator() func(http.Handler) http.Handler
+	//}
+
+	Signer interface {
+		Sign(userID uint64, pp ...interface{}) string
+		Verify(signature string, userID uint64, pp ...interface{}) bool
+	}
+)

extra/server-discovery/pkg/auth/token_issuer.go

@@ -0,0 +1,394 @@
+package auth
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/pkg/payload"
+	"github.com/go-oauth2/oauth2/v4"
+	"github.com/go-oauth2/oauth2/v4/generates"
+	"github.com/go-oauth2/oauth2/v4/models"
+	"github.com/lestrrat-go/jwx/jwa"
+	"github.com/lestrrat-go/jwx/jwk"
+	"github.com/lestrrat-go/jwx/jwt"
+	"github.com/spf13/cast"
+)
+
+type (
+	tokenIssuer struct {
+		defaultRequest *TokenRequest
+
+		// store issued tokens
+		store tokenIssuerStore
+
+		// lookup for issued tokens
+		lookup tokenIssuerLookup
+
+		// generator for issued tokens
+		generator tokenIssuerGenerator
+
+		// signer for issued tokens
+		signer tokenIssuerSigner
+	}
+
+	// IssuerOptFn modify toeknIssuer
+	IssuerOptFn func(*tokenIssuer) error
+
+	TokenRequest struct {
+		AccessToken  string
+		RefreshToken string
+		Expiration   time.Duration
+		Audience     string
+		Issuer       string
+		IssuedAt     time.Time
+		ClientID     uint64
+		UserID       uint64
+		Roles        []uint64
+		Scope        []string
+	}
+
+	// IssueOptFn functions modify TokenRequest
+	IssueOptFn func(*TokenRequest) error
+
+	tokenIssuerStore     func(context.Context, TokenRequest) error
+	tokenIssuerLookup    func(context.Context, string) error
+	tokenIssuerGenerator func(context.Context, TokenRequest) (string, string, error)
+	tokenIssuerSigner    func(token jwt.Token) ([]byte, error)
+)
+
+var (
+	TokenIssuer *tokenIssuer
+
+	// wrapper around time.Now() that will aid service testing
+	now = func() *time.Time {
+		c := time.Now().Truncate(time.Second)
+		return &c
+	}
+)
+
+// NewTokenIssuer initializes and returns new instance of JWT manager
+func NewTokenIssuer(opt ...IssuerOptFn) (issuer *tokenIssuer, err error) {
+	issuer = &tokenIssuer{
+		defaultRequest: &TokenRequest{Issuer: "cortezaproject.org"},
+
+		store: func(ctx context.Context, request TokenRequest) error {
+			// elegantly handle unconfigured store
+			return fmt.Errorf("token issuer store not configured")
+		},
+
+		lookup: func(context.Context, string) error {
+			// elegantly handle unconfigured lookup
+			return fmt.Errorf("token issuer lookup not configured")
+		},
+
+		signer: func(token jwt.Token) ([]byte, error) {
+			// elegantly handle unconfigured signer
+			return nil, fmt.Errorf("token issuer signer not configured")
+		},
+
+		generator: DefaultAccessTokenGenerator,
+	}
+
+	return issuer, applyToIssuer(issuer, opt...)
+}
+
+// Issue issues new access token, stores it and returns signed JWT.
+func (i *tokenIssuer) Issue(ctx context.Context, opt ...IssueOptFn) (_ []byte, err error) {
+	var req = i.newTokenRequest()
+	if err = req.apply(opt...); err != nil {
+		return
+	}
+
+	if len(req.AccessToken+req.RefreshToken) > 0 {
+		panic("can not issue new token with preset access and refresh tokens, " +
+			"this is most likely an implementation mistake")
+	}
+
+	if req.AccessToken, req.RefreshToken, err = i.generator(ctx, *req); err != nil {
+		return
+	}
+
+	if err = i.store(ctx, *req); err != nil {
+		return
+	}
+
+	return i.sign(req)
+}
+
+func (i *tokenIssuer) Sign(opt ...IssueOptFn) (_ []byte, err error) {
+	var req = i.newTokenRequest()
+	if err = req.apply(opt...); err != nil {
+		return
+	}
+
+	return i.sign(req)
+}
+
+func (i *tokenIssuer) sign(req *TokenRequest) ([]byte, error) {
+	if token, err := makeToken(req); err != nil {
+		return nil, err
+	} else {
+		return i.signer(token)
+	}
+}
+
+// Returns new token request and copies relevant values from the default token request on the issuer
+func (i *tokenIssuer) newTokenRequest() *TokenRequest {
+	return &TokenRequest{
+		Issuer:     i.defaultRequest.Issuer,
+		ClientID:   i.defaultRequest.ClientID,
+		Expiration: i.defaultRequest.Expiration,
+		IssuedAt:   *now(),
+	}
+}
+
+// Validate performs token validation by checking existence of access-token in the store
+func (i *tokenIssuer) Validate(ctx context.Context, token jwt.Token) (err error) {
+	if err = i.lookup(ctx, token.JwtID()); err != nil {
+		return fmt.Errorf("unauthorized")
+	}
+
+	return nil
+}
+
+func makeToken(req *TokenRequest) (_ jwt.Token, err error) {
+	var (
+		roles = make([]string, len(req.Roles))
+		token = jwt.New()
+
+		toString = func(i uint64) string {
+			return strconv.FormatUint(i, 10)
+		}
+	)
+
+	if len(req.Scope) == 0 {
+		// for backward compatibility we default
+		// unset scope to profile & api
+		req.Scope = []string{"profile", "api"}
+	}
+
+	if req.IssuedAt.IsZero() {
+		req.IssuedAt = *now()
+	}
+
+	for i, r := range req.Roles {
+		roles[i] = toString(r)
+	}
+
+	// The key part: store access token as JWT ID claim.
+	// Claim will be extracted when JWT is validated and checked
+	if err = token.Set(jwt.JwtIDKey, req.AccessToken); err != nil {
+		return
+	}
+
+	if err = token.Set(jwt.SubjectKey, toString(req.UserID)); err != nil {
+		return
+	}
+
+	if err = token.Set(jwt.ExpirationKey, now().Add(req.Expiration).Unix()); err != nil {
+		return
+	}
+
+	if req.Audience != "" {
+		if err = token.Set(jwt.AudienceKey, req.Audience); err != nil {
+			return
+		}
+	}
+
+	if err = token.Set(jwt.IssuerKey, req.Issuer); err != nil {
+		return
+	}
+
+	if err = token.Set(jwt.IssuedAtKey, req.IssuedAt.Unix()); err != nil {
+		return
+	}
+
+	if err = token.Set("clientID", toString(req.ClientID)); err != nil {
+		return
+	}
+
+	if err = token.Set("scope", strings.Join(req.Scope, " ")); err != nil {
+		return
+	}
+
+	if err = token.Set("roles", roles); err != nil {
+		return
+	}
+
+	return token, nil
+}
+
+// IdentityFromToken decodes sub & roles claims into identity
+func IdentityFromToken(token jwt.Token) *identity {
+	if token == nil {
+		return Anonymous()
+	}
+
+	var (
+		roles, _ = token.Get("roles")
+	)
+
+	return Authenticated(
+		cast.ToUint64(token.Subject()),
+		payload.ParseUint64s(cast.ToStringSlice(roles))...,
+	)
+}
+
+// DefaultAccessTokenGenerator uses token generator from oauth2 lib
+func DefaultAccessTokenGenerator(ctx context.Context, req TokenRequest) (string, string, error) {
+	return generates.NewAccessGenerate().Token(
+		ctx,
+		&oauth2.GenerateBasic{
+			Client: &models.Client{
+				ID: strconv.FormatUint(req.ClientID, 10),
+			},
+			UserID:    strconv.FormatUint(req.UserID, 10),
+			CreateAt:  *now(),
+			TokenInfo: nil,
+			Request:   nil,
+		},
+		true,
+	)
+}
+
+// WithSecretSigner configures token issuer with
+func WithSecretSigner(secret string) IssuerOptFn {
+	return func(tm *tokenIssuer) (err error) {
+		if len(secret) == 0 {
+			return fmt.Errorf("JWK missing")
+		}
+
+		var key jwk.Key
+		if key, err = jwk.New([]byte(secret)); err != nil {
+			return fmt.Errorf("could not parse JWK: %w", err)
+		}
+
+		tm.signer = func(token jwt.Token) ([]byte, error) {
+			return jwt.Sign(token, jwa.HS512, key)
+		}
+
+		return nil
+	}
+}
+
+// WithDefaultClientID configures ID of the default auth client
+func WithDefaultClientID(ID uint64) IssuerOptFn {
+	return func(tm *tokenIssuer) (err error) {
+		tm.defaultRequest.ClientID = ID
+		return
+	}
+}
+
+// WithDefaultExpiration configures default token expiration time
+func WithDefaultExpiration(exp time.Duration) IssuerOptFn {
+	return func(tm *tokenIssuer) (err error) {
+		tm.defaultRequest.Expiration = exp
+		return
+	}
+}
+
+// WithDefaultIssuer configures default issuer claim
+func WithDefaultIssuer(iss string) IssuerOptFn {
+	return func(tm *tokenIssuer) (err error) {
+		tm.defaultRequest.Issuer = iss
+		return
+	}
+}
+
+// WithStore configures store function
+func WithStore(fn tokenIssuerStore) IssuerOptFn {
+	return func(tm *tokenIssuer) (err error) {
+		tm.store = fn
+		return
+	}
+}
+
+// WithLookup configures lookup function
+func WithLookup(fn tokenIssuerLookup) IssuerOptFn {
+	return func(tm *tokenIssuer) (err error) {
+		tm.lookup = fn
+		return
+	}
+}
+
+// WithGenerator configures generator function
+func WithGenerator(fn tokenIssuerGenerator) IssuerOptFn {
+	return func(tm *tokenIssuer) (err error) {
+		tm.generator = fn
+		return
+	}
+}
+
+// WithSigner configures signer function
+func WithSigner(fn tokenIssuerSigner) IssuerOptFn {
+	return func(tm *tokenIssuer) (err error) {
+		tm.signer = fn
+		return
+	}
+}
+
+func (req *TokenRequest) apply(opt ...IssueOptFn) (err error) {
+	for _, fn := range opt {
+		if err = fn(req); err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func WithExpiration(e time.Duration) IssueOptFn {
+	return func(t *TokenRequest) (err error) {
+		t.Expiration = e
+		return
+	}
+}
+
+func WithIdentity(i Identifiable) IssueOptFn {
+	return func(t *TokenRequest) (err error) {
+		t.UserID = i.Identity()
+		t.Roles = i.Roles()
+		return
+	}
+}
+
+func WithAccessToken(access string) IssueOptFn {
+	return func(t *TokenRequest) (err error) {
+		t.AccessToken = access
+		return
+	}
+}
+
+func WithScope(ss ...string) IssueOptFn {
+	return func(t *TokenRequest) (err error) {
+		t.Scope = ss
+		return
+	}
+}
+
+func WithAudience(aud string) IssueOptFn {
+	return func(t *TokenRequest) (err error) {
+		t.Audience = aud
+		return
+	}
+}
+
+func WithClientID(id uint64) IssueOptFn {
+	return func(t *TokenRequest) (err error) {
+		t.ClientID = id
+		return
+	}
+}
+
+func applyToIssuer(tm *tokenIssuer, opt ...IssuerOptFn) (err error) {
+	for _, fn := range opt {
+		if err = fn(tm); err != nil {
+			return
+		}
+	}
+
+	return
+}

extra/server-discovery/pkg/auth/token_middleware.go

@@ -0,0 +1,94 @@
+package auth
+
+import (
+	"context"
+	"fmt"
+	"github.com/lestrrat-go/jwx/jwa"
+	"net/http"
+
+	"github.com/cortezaproject/corteza/server/pkg/errors"
+	"github.com/go-chi/jwtauth"
+	"github.com/lestrrat-go/jwx/jwk"
+	"github.com/lestrrat-go/jwx/jwt"
+)
+
+var (
+	HttpTokenVerifier func(http.Handler) http.Handler
+)
+
+// verifier returns a jwt verification middleware
+//
+// Tasks
+// 1. picks token from header, query or cookie
+// 2. extracts identity (if any) and adds it into request context
+//
+// In there is no token
+func verifier(ja *jwtauth.JWTAuth) func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			token, err := jwtauth.VerifyRequest(ja, r, jwtauth.TokenFromHeader, jwtauth.TokenFromQuery, jwtauth.TokenFromCookie)
+			ctx := r.Context()
+
+			ctx = jwtauth.NewContext(ctx, token, err)
+			ctx = SetIdentityToContext(ctx, IdentityFromToken(token))
+
+			next.ServeHTTP(w, r.WithContext(ctx))
+		})
+	}
+}
+
+// TokenVerifierMiddlewareWithSecretSigner returns HTTP handler with simple jwa.HS512 + secret verifier
+//
+// This should be 1:1 with token issuer!
+func TokenVerifierMiddlewareWithSecretSigner(secret string) (_ func(http.Handler) http.Handler, err error) {
+	if len(secret) == 0 {
+		return nil, fmt.Errorf("JWK missing")
+	}
+
+	var key jwk.Key
+	if key, err = jwk.New([]byte(secret)); err != nil {
+		return nil, fmt.Errorf("could not parse JWK: %w", err)
+	}
+
+	return verifier(jwtauth.New(jwa.HS512.String(), key, nil)), nil
+}
+
+// HttpTokenValidator checks if there is a token with identity and matching scope claim
+//
+// Empty scope defaults to "api"!
+func HttpTokenValidator(scope ...string) func(http.Handler) http.Handler {
+	if len(scope) == 0 {
+		// ensure that scope is not empty
+		scope = []string{"api"}
+	}
+
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			err := verifyToken(r.Context(), scope...)
+			if err != nil && !errors.Is(err, jwtauth.ErrNoTokenFound) {
+				errors.ProperlyServeHTTP(w, r, err, false)
+				return
+			}
+
+			next.ServeHTTP(w, r)
+		})
+	}
+}
+
+// pulls token from context and validates scope & access-token
+func verifyToken(ctx context.Context, scope ...string) (err error) {
+	var token jwt.Token
+	if token, _, err = jwtauth.FromContext(ctx); err != nil {
+		return
+	}
+
+	if token == nil {
+		return fmt.Errorf("unauthorized")
+	}
+
+	// if len(scope) > 0 && !CheckJwtScope(token, scope...) {
+	// 	return errUnauthorizedScope()
+	// }
+
+	return
+}

extra/server-discovery/pkg/es/connect.go

@@ -0,0 +1,52 @@
+package es
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"net/http"
+	"os"
+
+	"github.com/cortezaproject/corteza/extra/server-discovery/pkg/options"
+	"github.com/elastic/go-elasticsearch/v7"
+)
+
+func Connect(opt options.EsOpt) (*elasticsearch.Client, error) {
+	config := elasticsearch.Config{
+		Addresses:            opt.Addresses,
+		EnableRetryOnTimeout: opt.EnableRetryOnTimeout,
+		MaxRetries:           opt.MaxRetries,
+	}
+	if len(opt.Username) > 0 {
+		config.Username = opt.Username
+		config.Password = opt.Password
+	}
+
+	// if the user provided a CA certificate file, we need to load it
+	// and set it in the TLS config
+	if opt.CertFile != "" {
+		rootCACert, err := os.ReadFile(opt.CertFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read CA certificate file: %w", err)
+		}
+
+		caCertPool := x509.NewCertPool()
+		if !caCertPool.AppendCertsFromPEM(rootCACert) {
+			return nil, fmt.Errorf("failed parse root certificate: %w", err)
+		}
+
+		config.Transport = &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs: caCertPool,
+			},
+		}
+	} else if !opt.Secure {
+		config.Transport = &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		}
+	}
+
+	return elasticsearch.NewClient(config)
+}

extra/server-discovery/pkg/es/es.go

@@ -0,0 +1,70 @@
+package es
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/elastic/go-elasticsearch/v7/esapi"
+	"github.com/elastic/go-elasticsearch/v7/esutil"
+	"go.uber.org/zap"
+)
+
+type (
+	es struct {
+		log *zap.Logger
+		esc *elasticsearch.Client
+		//esb esutil.BulkIndexer
+	}
+
+	Service interface {
+		EsClient() *elasticsearch.Client
+		EsBulk() esutil.BulkIndexer
+		Watch(ctx context.Context)
+	}
+
+	//apiClientService interface {
+	//	HttpClient() *http.Client
+	//	Mappings() (*http.Request, error)
+	//	Resources(string, url.Values) (*http.Request, error)
+	//	Request(string) (*http.Request, error)
+	//	Authenticate() error
+	//}
+)
+
+func ES(log *zap.Logger, esc *elasticsearch.Client) (out *es, err error) {
+	out = &es{log: log, esc: esc}
+	return
+}
+
+func (es *es) BulkIndexer() (esutil.BulkIndexer, error) {
+	return esutil.NewBulkIndexer(esutil.BulkIndexerConfig{
+		Client:     es.esc,
+		FlushBytes: 5e+5,
+	})
+}
+
+func ValidElasticResponse(res *esapi.Response, err error) error {
+	if err != nil {
+		return fmt.Errorf("failed to get response from search backend: %w", err)
+	}
+
+	if res.IsError() {
+		defer res.Body.Close()
+		var rsp struct {
+			Error struct {
+				Type   string
+				Reason string
+			}
+		}
+
+		if err := json.NewDecoder(res.Body).Decode(&rsp); err != nil {
+			return fmt.Errorf("could not parse response body: %w", err)
+		} else {
+			return fmt.Errorf("search backend responded with an error: %s (type: %s, status: %s)", rsp.Error.Reason, rsp.Error.Type, res.Status())
+		}
+	}
+
+	return nil
+}

extra/server-discovery/pkg/es/mapping/mapping.go

@@ -0,0 +1,287 @@
+package mapping
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"github.com/cortezaproject/corteza/extra/server-discovery/pkg/es/reindex"
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/elastic/go-elasticsearch/v7/esapi"
+	"go.uber.org/zap"
+)
+
+type (
+	esIndex struct {
+		Name      string `json:"index"`
+		Health    string `json:"health"`
+		Status    string `json:"status"`
+		DocsCount string `json:"docs.count"`
+		StoreSize string `json:"store.size"`
+	}
+
+	rspDiscoveryMappings struct {
+		Response []*mapping
+	}
+
+	reqMapping struct {
+		// @todo structure
+		Settings map[string]any `json:"settings,omitempty"`
+
+		Mappings struct {
+			Properties map[string]*property `json:"properties,omitempty"`
+		} `json:"mappings,omitempty"`
+	}
+
+	mapping struct {
+		Index        string               `json:"index"`
+		Properties   map[string]*property `json:"properties"`
+		Mapping      map[string]*property `json:"mapping"`
+		DocumentsURL string               `json:"documentsURL"`
+	}
+
+	property struct {
+		// https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html
+		Type string `json:"type,omitempty"`
+
+		Analyzer string `json:"analyzer,omitempty"`
+
+		// Boost factor
+		// https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-boost.html
+		Boost float32 `json:"boost,omitempty"`
+
+		Properties map[string]*property `json:"properties,omitempty"`
+
+		// for vector search
+		Dimension int            `json:"dimension,omitempty"`
+		Method    map[string]any `json:"method,omitempty"`
+	}
+
+	apiClientService interface {
+		HttpClient() *http.Client
+		Mappings() (*http.Request, error)
+		Resources(string, url.Values) (*http.Request, error)
+		Request(string) (*http.Request, error)
+		Authenticate() error
+	}
+
+	mapper struct {
+		log      *zap.Logger
+		esClient *elasticsearch.Client
+		api      apiClientService
+	}
+)
+
+func Mapper(log *zap.Logger, esClient *elasticsearch.Client, api apiClientService) *mapper {
+	return &mapper{
+		log:      log,
+		esClient: esClient,
+		api:      api,
+	}
+}
+
+// Mappings fetches mappings from discovery server and update elastic search indexes
+func (m *mapper) Mappings(ctx context.Context, esc *elasticsearch.Client, indexPrefix string) (err error) {
+	var (
+		req             *http.Request
+		rsp             *http.Response
+		rspPayload      = &rspDiscoveryMappings{}
+		buf             = &bytes.Buffer{}
+		esRsp           *esapi.Response
+		existingIndexes []*esIndex
+		index           string
+	)
+
+	if req, err = m.api.Mappings(); err != nil {
+		return fmt.Errorf("failed to prepare mapping request: %w", err)
+	}
+
+	//d, _ := httputil.DumpRequest(req, true)
+	//println(string(d))
+
+	if rsp, err = m.api.HttpClient().Do(req.WithContext(ctx)); err != nil {
+		return fmt.Errorf("failed to send request: %w", err)
+	}
+
+	if rsp.StatusCode != http.StatusOK {
+		return fmt.Errorf("request resulted in an unexpected status: %s", rsp.Status)
+	}
+
+	//d, _ := httputil.DumpResponse(rsp, true)
+	//println(string(d))
+
+	if err = json.NewDecoder(rsp.Body).Decode(rspPayload); err != nil {
+		return fmt.Errorf("failed to decode mapping response: %w", err)
+	}
+	if err = rsp.Body.Close(); err != nil {
+		return fmt.Errorf("failed to close mapping response body: %w", err)
+	}
+
+	if existingIndexes, err = m.getExistingIndexes(ctx); err != nil {
+		return fmt.Errorf("failed to fetch existing indexes: %w", err)
+	}
+
+	indexMap := m.mapExistingIndexes(existingIndexes)
+
+	//esc, err := m.es.Client()
+	//if err != nil {
+	//	return fmt.Errorf("failed to prepare es client: %w", err)
+	//}
+
+	for _, im := range rspPayload.Response {
+		buf.Reset()
+		esReq := reqMapping{}
+		//esReq.Mappings.Properties = im.Properties
+		esReq.Settings = map[string]any{
+			"analysis": map[string]any{
+				"analyzer": map[string]any{
+					"edge_ngram_analyzer": map[string]any{
+						"tokenizer": "edge_ngram_tokenizer",
+						"filter":    []any{"lowercase"},
+					},
+				},
+				"tokenizer": map[string]any{
+					"edge_ngram_tokenizer": map[string]any{
+						"type":        "edge_ngram",
+						"min_gram":    2,
+						"max_gram":    15,
+						"token_chars": []any{"letter", "digit"},
+					},
+				},
+			},
+			"index": map[string]any{
+				"knn": true,
+			},
+		}
+		esReq.Mappings.Properties = im.Mapping
+
+		if err = json.NewEncoder(buf).Encode(esReq); err != nil {
+			return
+		}
+
+		index = fmt.Sprintf(reindex.IndexTpl, indexPrefix, im.Index)
+		iLog := m.log.With(zap.String("name", index))
+
+		if e := indexMap[index]; e != nil {
+			// @todo upsert settings if not the same?
+			iLog.Info("index exists",
+				zap.String("health", e.Health),
+				zap.String("status", e.Status),
+				zap.String("size", e.StoreSize),
+				zap.String("documents", e.DocsCount),
+			)
+
+			continue
+		}
+
+		if esRsp, err = esc.Indices.Create(index, esc.Indices.Create.WithBody(buf)); esRsp.IsError() || err != nil {
+			if err != nil {
+				iLog.Error("index creation failed", zap.Error(err))
+			}
+			if len(esRsp.String()) > 0 {
+				iLog.Error(fmt.Sprintf("index creation failed due to %s", esRsp.String()))
+			}
+			continue
+		}
+
+		if err = esRsp.Body.Close(); err != nil {
+			return
+		}
+
+		iLog.Info("index created")
+	}
+
+	return
+}
+
+func (m *mapper) mapExistingIndexes(ii []*esIndex) (im map[string]*esIndex) {
+	im = make(map[string]*esIndex)
+	for _, i := range ii {
+		im[i.Name] = i
+	}
+
+	return
+}
+
+func (m *mapper) getExistingIndexes(ctx context.Context) (ii []*esIndex, err error) {
+	var (
+		esRsp *esapi.Response
+	)
+
+	ii = make([]*esIndex, 100)
+
+	esc := m.esClient
+
+	esRsp, err = esc.Cat.Indices(
+		esc.Cat.Indices.WithContext(ctx),
+		esc.Cat.Indices.WithFormat("json"),
+	)
+	if err != nil {
+		return
+	}
+	defer esRsp.Body.Close()
+
+	return ii, json.NewDecoder(esRsp.Body).Decode(&ii)
+}
+
+// ConfigurationMapping ...
+func (m *mapper) ConfigurationMapping(ctx context.Context) (err error) {
+	var (
+		buf = &bytes.Buffer{}
+
+		// @todo better naming and ref please parent structs
+		configMap = reqMapping{
+			Mappings: struct {
+				Properties map[string]*property `json:"properties,omitempty"`
+			}(struct{ Properties map[string]*property }{Properties: map[string]*property{
+				"incrementalIndex": {
+					//Type: "object",
+					Properties: map[string]*property{
+						"updatedAt": {Type: "date"},
+					},
+				},
+			}}),
+		}
+	)
+	const (
+		CortezaConfigurationIndex = "corteza-configuration-test"
+	)
+
+	index := fmt.Sprintf(CortezaConfigurationIndex)
+	esc := m.esClient
+
+	if err = json.NewEncoder(buf).Encode(configMap); err != nil {
+		return
+	}
+
+	// Check whether a particular configuration index exists
+	res, err := esc.Indices.Exists([]string{index}, esc.Indices.Exists.WithContext(ctx))
+	if res != nil && err != nil && !res.IsError() {
+		if err != nil {
+			m.log.Error("Configuration index exist check failed", zap.Error(err))
+		} else {
+			// Put mapping if exist otherwise create it
+			if res, err := esc.Indices.Create(index, esc.Indices.Create.WithBody(buf)); res.IsError() || err != nil {
+				if err != nil {
+					m.log.Error("Configuration index creation failed", zap.Error(err))
+				}
+				if len(res.String()) > 0 {
+					m.log.Error(fmt.Sprintf("Configuration index creation failed due to %s", res.String()))
+				}
+			}
+
+			m.log.Info("Configuration index created")
+			return
+		}
+	}
+
+	m.log.Info("Configuration index exist")
+
+	return
+}
+
+// @todo: maybe add const for all es mapping property type
+// @todo: add helper methods for all Indices & use it instead of esc.Indices.

extra/server-discovery/pkg/es/mapping/mapping_test.go

@@ -0,0 +1,106 @@
+package mapping
+
+import (
+	"fmt"
+	"github.com/cortezaproject/corteza/server/pkg/options"
+	"os"
+	"strings"
+	"testing"
+)
+
+type (
+	config struct {
+		httpAddr string
+		es       struct {
+			addresses []string
+		}
+		cortezaAuth         string
+		cortezaDiscoveryAPI string
+		schemas             []*schema
+	}
+
+	schema struct {
+		indexPrefix  string
+		clientKey    string
+		clientSecret string
+	}
+)
+
+const (
+	envKeyHttpAddr = "HTTP_ADDR"
+)
+
+// @todo use it properly
+func TestMappings(t *testing.T) {
+	//var (
+	//	ctx = context.Background()
+	//	log = logger.Default()
+	//	req = require.New(t)
+	//)
+	//
+	//cfg, err := GetConfig()
+	//req.NoError(err)
+
+	//client, err := Es(cfg.es.addresses)
+	//req.NoError(err)
+	//
+	//api, err := indexer.ApiClient(cfg.cortezaDiscoveryAPI, cfg.cortezaAuth, cfg.schemas[0].clientKey, cfg.schemas[0].clientSecret)
+	//req.NoError(err)
+	//
+	//err = Mappings(ctx, log, client, api, "private")
+	//req.NoError(err)
+}
+
+// @todo reuse existing one
+func GetConfig() (*config, error) {
+	c := &config{}
+	return c, func() error {
+		baseUrl := options.EnvString("CORTEZA_SERVER_BASE_URL", "http://server:80")
+
+		c.httpAddr = options.EnvString(envKeyHttpAddr, "127.0.0.1:3201")
+
+		c.cortezaAuth = options.EnvString("CORTEZA_SERVER_AUTH", baseUrl+"/auth")
+		if c.cortezaAuth == "" {
+			return fmt.Errorf("corteza Auth endpoint value empty, set it directly with CORTEZA_SERVER_AUTH or indirectly with CORTEZA_SERVER_BASE_URL")
+		}
+
+		c.cortezaDiscoveryAPI = options.EnvString("CORTEZA_SERVER_API_DISCOVERY", baseUrl+"/api/discovery")
+		if c.cortezaDiscoveryAPI == "" {
+			return fmt.Errorf("corteza Discovery API endpoint value empty, set it directly with CORTEZA_SERVER_AUTH or indirectly with CORTEZA_SERVER_API_DISCOVERY")
+		}
+
+		for _, ar := range []string{"public", "protected", "private"} {
+			var (
+				has  bool
+				ucAr = strings.ToUpper(ar)
+				s    = &schema{indexPrefix: ar}
+
+				keyEnv = ucAr + "_INDEX_CLIENT_KEY"
+				secEnv = ucAr + "_INDEX_CLIENT_SECRET"
+			)
+
+			if s.clientKey, has = os.LookupEnv(keyEnv); !has {
+				continue
+			} else if s.clientKey == "" {
+				return fmt.Errorf("client key (%s) for '%s' is empty or missing", keyEnv, s.indexPrefix)
+			}
+
+			if s.clientSecret = os.Getenv(secEnv); s.clientSecret == "" {
+				return fmt.Errorf("client secret (%s) for '%s' is empty or missing", secEnv, s.indexPrefix)
+			}
+
+			c.schemas = append(c.schemas, s)
+		}
+
+		if len(c.schemas) == 0 {
+			return fmt.Errorf("set at least one client secret pair using <PREFIX>_INDEX_CLIENT_KEY and <PREFIX>_INDEX_CLIENT_SECRET where prefix is one of 'public', 'protected' or 'private'")
+		}
+
+		for _, a := range strings.Split(options.EnvString("ES_ADDRESS", "http://es:9200"), " ") {
+			if a = strings.TrimSpace(a); a != "" {
+				c.es.addresses = append(c.es.addresses, a)
+			}
+		}
+		return nil
+	}()
+}

extra/server-discovery/pkg/es/reindex/reindex.go

@@ -0,0 +1,972 @@
+package reindex
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/cortezaproject/corteza/extra/server-discovery/pkg/options"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/elastic/go-elasticsearch/v7/esutil"
+	"github.com/jmoiron/sqlx/types"
+	"go.uber.org/zap"
+)
+
+type (
+	docsSources struct {
+		endpoint string
+		index    string
+		action   string
+		params   map[string]string
+		callback func(*document)
+	}
+
+	rspDiscoveryDocuments struct {
+		Error *struct {
+			Message string
+		}
+		Response *struct {
+			Filter struct {
+				NextPage string
+			}
+
+			Documents []*document
+		}
+	}
+
+	// auxiliary struct for parsing indexable documents from Discovery API
+	document struct {
+		ID     string
+		Index  string
+		Source json.RawMessage
+	}
+
+	feedSources struct {
+		endpoint string
+		index    string
+		action   string
+		params   map[string]string
+		callback func(*document)
+	}
+
+	feedResponse struct {
+		Error *struct {
+			Message string
+		}
+		Response *struct {
+			Filter struct {
+				NextPage string
+			}
+
+			ActivityLogs []ActivityLog `json:"activityLogs"`
+		}
+	}
+
+	ActivityLog struct {
+		ID             uint64         `json:"activityID,string"`
+		ResourceID     uint64         `json:"resourceID,string"`
+		ResourceType   string         `json:"resourceType"`
+		ResourceAction string         `json:"resourceAction"`
+		Timestamp      time.Time      `json:"timestamp"`
+		Meta           types.JSONText `json:"meta"`
+	}
+
+	ActivityLogMeta struct {
+		NamespaceID uint64 `json:"namespaceID,string"`
+		ModuleID    uint64 `json:"moduleID,string"`
+	}
+
+	esService interface {
+		BulkIndexer() (esutil.BulkIndexer, error)
+	}
+
+	embedderService interface {
+		GenerateEmbeddings(input string) ([]float64, error)
+	}
+
+	apiClientService interface {
+		HttpClient() *http.Client
+		Mappings() (*http.Request, error)
+		Feed(url.Values) (*http.Request, error)
+		Resources(string, url.Values) (*http.Request, error)
+		Request(string) (*http.Request, error)
+		Authenticate() error
+	}
+
+	reIndexer struct {
+		log            *zap.Logger
+		esOpt          options.EsOpt
+		es             esService
+		esClient       *elasticsearch.Client
+		api            apiClientService
+		assureMappings func(context.Context) error
+	}
+
+	NamespaceRef struct {
+		NamespaceID string `json:"namespaceID,omitempty"`
+		Name        string `json:"name,omitempty"`
+	}
+
+	ModuleRef struct {
+		ModuleID string `json:"moduleID,omitempty"`
+		Name     string `json:"name,omitempty"`
+		Handle   string `json:"handle,omitempty"`
+	}
+
+	TimestampInfo struct {
+		At time.Time `json:"at,omitempty"`
+		By *User     `json:"by,omitempty"`
+	}
+
+	User struct {
+		UserID string `json:"userID,omitempty"`
+		Email  string `json:"email,omitempty"`
+		Name   string `json:"name,omitempty"`
+		Handle string `json:"handle,omitempty"`
+	}
+
+	SecurityRule struct {
+		AllowedRoles []string `json:"allowedRoles,omitempty"`
+		DeniedRoles  []string `json:"deniedRoles,omitempty"`
+	}
+
+	ComposeRecord struct {
+		ResourceType string            `json:"resourceType"`
+		RecordID     string            `json:"recordID"`
+		ValueLabels  map[string]string `json:"valueLabels,omitempty"`
+		Values       map[string]any    `json:"values,omitempty"`
+		CatchAll     []any             `json:"catch_all,omitempty"`
+		Updated      *TimestampInfo    `json:"updated,omitempty"`
+		Created      *TimestampInfo    `json:"created,omitempty"`
+		Deleted      *TimestampInfo    `json:"deleted,omitempty"`
+		Security     []SecurityRule    `json:"security,omitempty"`
+		Namespace    *NamespaceRef     `json:"namespace,omitempty"`
+		Module       *ModuleRef        `json:"module,omitempty"`
+		VectorsValue []float64         `json:"vectorsValue,omitempty"`
+	}
+)
+
+const (
+	IndexTpl    = "corteza-%s-%s"
+	MarkerIndex = "corteza_indexer_state"
+	MarkerID    = "last_index_time"
+)
+
+func ReIndexer(log *zap.Logger, es esService, esc *elasticsearch.Client, api apiClientService, esOpt options.EsOpt, assureMappings func(context.Context) error) *reIndexer {
+	return &reIndexer{
+		log:            log,
+		esOpt:          esOpt,
+		es:             es,
+		esClient:       esc,
+		api:            api,
+		assureMappings: assureMappings,
+	}
+}
+
+func (ri *reIndexer) ReindexAll(ctx context.Context, esb esutil.BulkIndexer, indexPrefix string) error {
+	var (
+		srcQueue = make(chan *docsSources, 100)
+		bErr     = ri.reindexManager(ctx, esb, indexPrefix, srcQueue)
+	)
+
+	srcQueue <- &docsSources{
+		endpoint: "/system/users",
+		index:    "system-users",
+	}
+
+	postProcModules := func(namespaceID string) func(d *document) {
+		return func(d *document) {
+			srcQueue <- &docsSources{
+				endpoint: fmt.Sprintf("/compose/namespaces/%s/modules/%s/records", namespaceID, d.ID),
+				index:    fmt.Sprintf("compose-records-%s-%s", namespaceID, d.ID),
+			}
+		}
+	}
+
+	postProcNamespaces := func(d *document) {
+		srcQueue <- &docsSources{
+			endpoint: fmt.Sprintf("/compose/namespaces/%s/modules", d.ID),
+			index:    "compose-modules",
+			callback: postProcModules(d.ID),
+		}
+	}
+
+	_ = postProcModules
+	_ = postProcNamespaces
+
+	srcQueue <- &docsSources{
+		endpoint: "/compose/namespaces",
+		index:    "compose-namespaces",
+		callback: postProcNamespaces,
+	}
+	_ = fmt.Errorf("blocking error")
+	return <-bErr
+}
+
+func (ri *reIndexer) reindexManager(ctx context.Context, esb esutil.BulkIndexer, indexPrefix string, srcQueue chan *docsSources) chan error {
+	var qErr = make(chan error)
+	const maxQueueLen = 3
+
+	go func(esb esutil.BulkIndexer) {
+		var (
+			pQueueLen        = -1
+			pQueueStaleCount int
+
+			ticker = time.NewTicker(time.Second)
+		)
+
+		defer ticker.Stop()
+		defer func() {
+			qErr <- nil
+		}()
+
+		for {
+			select {
+			case <-ctx.Done():
+				if ctx.Err() != context.Canceled {
+					ri.log.Error(ctx.Err().Error())
+				} else {
+					ri.log.Info("stopped")
+				}
+				return
+
+			case ds := <-srcQueue:
+				if ds == nil {
+					// graceful termination
+					ri.log.Info("done")
+					return
+				}
+
+				err := ri.reindex(ctx, esb, indexPrefix, ds)
+				if err != nil {
+					ri.log.Error("failed to reindex", zap.Error(err), zap.String("endpoint", ds.endpoint))
+					return
+				}
+
+			case <-ticker.C:
+				if pQueueLen != len(srcQueue) {
+					pQueueStaleCount = maxQueueLen
+				} else {
+					pQueueStaleCount--
+				}
+
+				if pQueueStaleCount <= 0 {
+					ri.log.Info("idle")
+					return
+				}
+
+				pQueueLen = len(srcQueue)
+
+				//esb, err := ri.es.BulkIndexer()
+				//if err != nil {
+				//	qErr <- err
+				//}
+
+				s := esb.Stats()
+				ri.log.Debug("batch indexing stats",
+					zap.Uint64("added", s.NumAdded),
+					zap.Uint64("created", s.NumCreated),
+					zap.Uint64("updated", s.NumUpdated),
+					zap.Uint64("deleted", s.NumDeleted),
+					zap.Uint64("flushed", s.NumFlushed),
+					zap.Uint64("failed", s.NumFailed),
+					zap.Uint64("indexed", s.NumIndexed),
+					zap.Uint64("requests", s.NumRequests),
+					zap.Int("queue length", pQueueLen),
+				)
+			}
+		}
+	}(esb)
+
+	println("returning")
+	return qErr
+}
+
+func (ri *reIndexer) reindex(ctx context.Context, esb esutil.BulkIndexer, indexPrefix string, ds *docsSources) (err error) {
+	var (
+		qs     = url.Values{"limit": []string{"500"}}
+		req    *http.Request
+		rsp    *http.Response
+		cursor string
+	)
+
+	//esb, err := ri.es.BulkIndexer()
+	//if err != nil {
+	//	return fmt.Errorf("failed to prepare bulk indexer: %w", err)
+	//}
+
+	for {
+		rspPayload := &rspDiscoveryDocuments{}
+
+		if cursor != "" {
+			// set new cursor and update source URL
+			qs.Set("pageCursor", cursor)
+		}
+
+		if req, err = ri.api.Resources(ds.endpoint, qs); err != nil {
+			return fmt.Errorf("failed to prepare resource request: %w", err)
+		}
+
+		if rsp, err = ri.api.HttpClient().Do(req.WithContext(ctx)); err != nil {
+			return fmt.Errorf("failed to send request: %w", err)
+		}
+
+		if rsp.StatusCode != http.StatusOK {
+			return fmt.Errorf("request resulted in an unexpected status '%s' for url '%s'", rsp.Status, req.URL)
+		}
+
+		//{
+		//	d, err := httputil.DumpRequestOut(req, true)
+		//	println(string(d))
+		//	spew.Dump(err)
+		//}
+		//{
+		//	d, err := httputil.DumpResponse(rsp, true)
+		//	println(string(d))
+		//	spew.Dump(err)
+		//}
+
+		if err = json.NewDecoder(rsp.Body).Decode(rspPayload); err != nil {
+			return fmt.Errorf("failed to decode reindexing response: %w", err)
+		}
+
+		if err = rsp.Body.Close(); err != nil {
+			return fmt.Errorf("failed to close reindexing response body: %w", err)
+		}
+
+		var docs int
+		if rspPayload.Error != nil {
+			ri.log.Debug("skipping",
+				zap.String("index", fmt.Sprintf(IndexTpl, indexPrefix, ds.index)),
+				zap.String("error", rspPayload.Error.Message),
+			)
+			return
+		} else if rspPayload.Response != nil {
+			docs = len(rspPayload.Response.Documents)
+		}
+
+		ri.log.Debug("reindexing",
+			zap.Int("docs", docs),
+			zap.String("index", fmt.Sprintf(IndexTpl, indexPrefix, ds.index)),
+		)
+
+		if docs == 0 {
+			return
+		}
+
+		for _, doc := range rspPayload.Response.Documents {
+			body, err := ri.processResource(doc.Source)
+			if err != nil {
+				ri.log.Error("failed to process record's for embeddings: ", zap.Error(err))
+			}
+
+			err = esb.Add(ctx, esutil.BulkIndexerItem{
+				Index:      fmt.Sprintf(IndexTpl, indexPrefix, ds.index),
+				Action:     "index",
+				DocumentID: doc.ID,
+				Body:       body,
+				OnFailure: func(ctx context.Context, req esutil.BulkIndexerItem, rsp esutil.BulkIndexerResponseItem, err error) {
+					spew.Dump(req)
+					spew.Dump(rsp)
+					spew.Dump(err)
+				},
+			})
+
+			if err != nil {
+				return err
+			}
+
+			if ds.callback != nil {
+				go ds.callback(doc)
+			}
+		}
+
+		cursor = rspPayload.Response.Filter.NextPage
+		if rspPayload.Response.Filter.NextPage == "" {
+			break
+		}
+	}
+
+	//if err = esb.Close(ctx); err != nil {
+	//	return fmt.Errorf("failed to close bulk indexer: %w", err)
+	//}
+
+	return nil
+}
+
+func (ri *reIndexer) feedReindex(ctx context.Context, esb esutil.BulkIndexer, indexPrefix string, ds *feedSources) (err error) {
+	var (
+		qs     = url.Values{"limit": []string{"500"}}
+		req    *http.Request
+		rsp    *http.Response
+		cursor string
+	)
+
+	if ds == nil {
+		ri.log.Debug("invalid resource for feed update")
+		return
+	}
+
+	err = ri.assureMappings(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to assure mappings: %w", err)
+	}
+
+	for {
+		rspPayload := &rspDiscoveryDocuments{}
+
+		if cursor != "" {
+			// set new cursor and update source URL
+			qs.Set("pageCursor", cursor)
+		}
+
+		if ds.params != nil {
+			var (
+				val string
+				ok  bool
+			)
+
+			val, ok = ds.params["userID"]
+			if ok {
+				qs.Set("userID", val)
+			}
+
+			val, ok = ds.params["namespaceID"]
+			if ok {
+				qs.Set("namespaceID", val)
+			}
+
+			val, ok = ds.params["moduleID"]
+			if ok {
+				qs.Set("moduleID", val)
+			}
+
+			val, ok = ds.params["recordID"]
+			if ok {
+				qs.Set("recordID", val)
+			}
+
+			val, ok = ds.params["deleted"]
+			if ok {
+				qs.Set("deleted", val)
+			}
+		}
+
+		if req, err = ri.api.Resources(ds.endpoint, qs); err != nil {
+			return fmt.Errorf("failed to prepare resource request: %w", err)
+		}
+
+		if rsp, err = ri.api.HttpClient().Do(req.WithContext(ctx)); err != nil {
+			return fmt.Errorf("failed to send request: %w", err)
+		}
+
+		if rsp.StatusCode != http.StatusOK {
+			return fmt.Errorf("request resulted in an unexpected status '%s' for url '%s'", rsp.Status, req.URL)
+		}
+
+		//{
+		//	d, err := httputil.DumpRequestOut(req, true)
+		//	println(string(d))
+		//	spew.Dump(err)
+		//}
+		//{
+		//	d, err := httputil.DumpResponse(rsp, true)
+		//	println(string(d))
+		//	spew.Dump(err)
+		//}
+
+		if err = json.NewDecoder(rsp.Body).Decode(rspPayload); err != nil {
+			return fmt.Errorf("failed to decode reindexing response: %w", err)
+		}
+
+		if err = rsp.Body.Close(); err != nil {
+			return fmt.Errorf("failed to close reindexing response body: %w", err)
+		}
+
+		var (
+			docs     int
+			docIndex = fmt.Sprintf(IndexTpl, indexPrefix, ds.index)
+		)
+		if rspPayload.Error != nil {
+			ri.log.Debug("skipping",
+				zap.String("index", docIndex),
+				zap.String("error", rspPayload.Error.Message),
+			)
+			return
+		} else if rspPayload.Response != nil {
+			docs = len(rspPayload.Response.Documents)
+		}
+
+		ri.log.Debug("feed reindexing",
+			zap.Int("docs", docs),
+			zap.String("index", docIndex),
+		)
+
+		if docs == 0 {
+			return
+		}
+
+		action := "index"
+		if len(ds.action) > 0 {
+			action = ds.action
+		}
+
+		for _, doc := range rspPayload.Response.Documents {
+			esbItem := esutil.BulkIndexerItem{
+				Index:      docIndex,
+				Action:     action,
+				DocumentID: doc.ID,
+				OnFailure: func(ctx context.Context, req esutil.BulkIndexerItem, rsp esutil.BulkIndexerResponseItem, err error) {
+					spew.Dump(req)
+					spew.Dump(rsp)
+					spew.Dump(err)
+				},
+			}
+			if action != "delete" {
+				esbItem.Action = "index"
+				body, err := ri.processResource(doc.Source)
+				if err != nil {
+					return err
+				}
+
+				esbItem.Body = body
+			}
+
+			err = esb.Add(ctx, esbItem)
+			if err != nil {
+				return err
+			}
+
+			if ds.callback != nil {
+				go ds.callback(doc)
+			}
+		}
+
+		cursor = rspPayload.Response.Filter.NextPage
+		if rspPayload.Response.Filter.NextPage == "" {
+			break
+		}
+	}
+
+	return nil
+}
+
+func (ri *reIndexer) feedReindexManager(ctx context.Context, esb esutil.BulkIndexer, indexPrefix string, feedQueue chan *feedSources) chan error {
+	var qErr = make(chan error)
+	const maxQueueLen = 3
+
+	go func(esb esutil.BulkIndexer) {
+		var (
+			pQueueLen        = -1
+			pQueueStaleCount int
+
+			ticker = time.NewTicker(time.Second)
+		)
+
+		defer ticker.Stop()
+		defer func() {
+			qErr <- nil
+		}()
+
+		for {
+			select {
+			case <-ctx.Done():
+				if ctx.Err() != context.Canceled {
+					ri.log.Error(ctx.Err().Error())
+				} else {
+					ri.log.Info("feed changes stopped")
+				}
+				return
+
+			case ds := <-feedQueue:
+				if ds == nil {
+					// graceful termination
+					ri.log.Info("feed changes done")
+					return
+				}
+
+				err := ri.feedReindex(ctx, esb, indexPrefix, ds)
+				if err != nil {
+					ri.log.Error("failed to reindex", zap.Error(err), zap.String("endpoint", ds.endpoint))
+					return
+				}
+
+			case <-ticker.C:
+				if pQueueLen != len(feedQueue) {
+					pQueueStaleCount = maxQueueLen
+				} else {
+					pQueueStaleCount--
+				}
+
+				if pQueueStaleCount <= 0 {
+					ri.log.Info("feed changes idle")
+					return
+				}
+
+				pQueueLen = len(feedQueue)
+
+				s := esb.Stats()
+				ri.log.Debug("feed batch indexing stats",
+					zap.Uint64("added", s.NumAdded),
+					zap.Uint64("created", s.NumCreated),
+					zap.Uint64("updated", s.NumUpdated),
+					zap.Uint64("deleted", s.NumDeleted),
+					zap.Uint64("flushed", s.NumFlushed),
+					zap.Uint64("failed", s.NumFailed),
+					zap.Uint64("indexed", s.NumIndexed),
+					zap.Uint64("requests", s.NumRequests),
+					zap.Int("queue length", pQueueLen),
+				)
+			}
+		}
+	}(esb)
+
+	println("feed changes returning")
+	return qErr
+}
+
+func (ri *reIndexer) feedReindexChanges(ctx context.Context, esb esutil.BulkIndexer, indexPrefix string, als []ActivityLog) error {
+	var (
+		updateQueue = make(chan *feedSources, 100)
+		bErr        = ri.feedReindexManager(ctx, esb, indexPrefix, updateQueue)
+
+		duplicateMap = make(map[string]ActivityLog)
+	)
+
+	for _, al := range als {
+		activityType := fmt.Sprintf("%d-%s", al.ResourceID, al.ResourceAction)
+		if val, ok := duplicateMap[activityType]; ok {
+			continue
+		} else {
+			duplicateMap[activityType] = val
+		}
+
+		action := al.ResourceAction
+		switch al.ResourceType {
+		case "system:user":
+			updateQueue <- &feedSources{
+				endpoint: "/system/users",
+				index:    "system-users",
+				action:   action,
+				params: map[string]string{
+					"userID":  fmt.Sprintf("%d", al.ResourceID),
+					"deleted": "1",
+				},
+			}
+			break
+
+		case "compose:namespace":
+			updateQueue <- &feedSources{
+				endpoint: "/compose/namespaces",
+				index:    "compose-namespaces",
+				action:   action,
+				params: map[string]string{
+					"namespaceID": fmt.Sprintf("%d", al.ResourceID),
+					"deleted":     "1",
+				},
+			}
+			break
+
+		case "compose:module":
+			var meta ActivityLogMeta
+			err := al.Meta.Unmarshal(&meta)
+			if err != nil {
+				return err
+			}
+
+			updateQueue <- &feedSources{
+				endpoint: fmt.Sprintf("/compose/namespaces/%d/modules", meta.NamespaceID),
+				index:    "compose-modules",
+				action:   action,
+				params: map[string]string{
+					"moduleID": fmt.Sprintf("%d", al.ResourceID),
+					"deleted":  "1",
+				},
+			}
+			break
+
+		case "compose:record":
+			var meta ActivityLogMeta
+			err := al.Meta.Unmarshal(&meta)
+			if err != nil {
+				return err
+			}
+
+			updateQueue <- &feedSources{
+				endpoint: fmt.Sprintf("/compose/namespaces/%d/modules/%d/records", meta.NamespaceID, meta.ModuleID),
+				index:    fmt.Sprintf("compose-records-%d-%d", meta.NamespaceID, meta.ModuleID),
+				action:   action,
+				params: map[string]string{
+					"recordID": fmt.Sprintf("%d", al.ResourceID),
+					"deleted":  "1",
+				},
+			}
+			break
+
+		default:
+			break
+		}
+
+	}
+
+	return <-bErr
+}
+
+func (ri *reIndexer) Watch(ctx context.Context) {
+	var now time.Time
+
+	isFirst, err := ri.IsFirstRunWithMarker(ctx)
+	if err != nil {
+		ri.log.Warn(fmt.Sprintf("failed to check first run indexes marker: %s", err))
+		isFirst = false
+	}
+
+	if isFirst {
+		startTime := time.Date(2018, time.January, 1, 0, 0, 0, 0, time.UTC) // Corteza start year
+		if ri.esOpt.IndexBackFillMonths != 0 {
+			startTime = time.Now().AddDate(0, -ri.esOpt.IndexBackFillMonths, 0)
+		}
+
+		ri.log.Info(fmt.Sprintf("first run detected: starting from (%s)", startTime.UTC().Format(time.RFC3339)))
+
+		if err := ri.processBacklogInChunks(ctx, startTime, &now); err != nil {
+			ri.log.Error(fmt.Sprintf("backlog processing failed: %s", err))
+		}
+
+		defer func() {
+			if err := ri.SaveLastIndexTime(ctx); err != nil {
+				ri.log.Error(fmt.Sprintf("failed to save indexer state to [corteza_indexer_state]: %s", err))
+			}
+		}()
+	}
+
+	now = time.Now()
+	ri.log.Info("continuing reindexing from current time")
+
+	timeOut := ri.esOpt.IndexInterval
+	ticker := time.NewTicker(time.Second * time.Duration(timeOut))
+
+	go func() {
+		defer ticker.Stop()
+		processing := false
+
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				if processing {
+					ri.log.Warn("skipping feed changes reindexing: already processing")
+					continue
+				}
+
+				processing = true
+				go func() {
+					defer func() {
+						processing = false
+					}()
+
+					tmpTime := time.Now()
+
+					if err := ri.fetchAndIndex(ctx, now, nil); err != nil {
+						ri.log.Error(fmt.Sprintf("reindex error: %s", err))
+						return
+					}
+
+					now = tmpTime
+				}()
+			}
+		}
+	}()
+}
+
+func (ri *reIndexer) processResource(source []byte) (*bytes.Reader, error) {
+	var peekResource struct {
+		ResourceType string `json:"resourceType"`
+	}
+
+	if err := json.Unmarshal(source, &peekResource); err != nil {
+		return nil, fmt.Errorf("error unmarshaling: %v", err)
+	}
+
+	if peekResource.ResourceType != "compose:record" {
+		return bytes.NewReader(source), nil
+	}
+
+	// process record's resource
+	var record ComposeRecord
+	if err := json.Unmarshal(source, &record); err != nil {
+		return nil, fmt.Errorf("error unmarshaling record: %v", err)
+	}
+
+	updatedJSON, err := json.Marshal(record)
+	if err != nil {
+		return nil, fmt.Errorf("error marshaling: %v", err)
+	}
+
+	return bytes.NewReader(updatedJSON), nil
+}
+
+func (ri *reIndexer) IsFirstRunWithMarker(ctx context.Context) (bool, error) {
+	res, err := ri.esClient.Get(
+		MarkerIndex,
+		MarkerID,
+		ri.esClient.Get.WithContext(ctx),
+	)
+
+	// create the index and the fields.
+
+	if err != nil {
+		return false, err
+	}
+
+	defer res.Body.Close()
+
+	if res.IsError() {
+		// marker index does'nt exist
+		if res.StatusCode == http.StatusNotFound {
+			return true, err
+		}
+	}
+
+	return false, nil
+}
+
+func (ri *reIndexer) SaveLastIndexTime(ctx context.Context) error {
+	doc := map[string]interface{}{
+		"last_run":    time.Now().UTC().Format(time.RFC3339),
+		"initialized": true,
+	}
+
+	docBytes, err := json.Marshal(doc)
+	if err != nil {
+		return fmt.Errorf("failed to marshal marker document: %w", err)
+	}
+
+	// create/update the marker document
+	res, err := ri.esClient.Index(
+		MarkerIndex,
+		bytes.NewReader(docBytes),
+		ri.esClient.Index.WithContext(ctx),
+		ri.esClient.Index.WithDocumentID(MarkerID),
+		ri.esClient.Index.WithRefresh("true"),
+	)
+
+	if err != nil {
+		return fmt.Errorf("failed to save marker index document: %w", err)
+	}
+
+	defer res.Body.Close()
+
+	if res.IsError() {
+		return fmt.Errorf("failed to save marker index document: %s", res.Status())
+	}
+
+	return nil
+}
+
+func (ri *reIndexer) fetchAndIndex(ctx context.Context, fromTime time.Time, toTime *time.Time) error {
+	esb, err := ri.es.BulkIndexer()
+	if err != nil {
+		return err
+	}
+	defer esb.Close(ctx)
+
+	feeds, err := ri.fetchSinglePage(ctx, fromTime, toTime, "")
+	if err != nil {
+		return err
+	}
+
+	if feeds != nil && feeds.Response != nil && len(feeds.Response.ActivityLogs) > 0 {
+		ri.log.Info(fmt.Sprintf("indexing %d records", len(feeds.Response.ActivityLogs)))
+		return ri.feedReindexChanges(ctx, esb, "private", feeds.Response.ActivityLogs)
+	} else {
+		ri.log.Debug(fmt.Sprintf("no feed changes since %s; current time: %s", fromTime.UTC().String(), time.Now().UTC().String()))
+	}
+
+	return nil
+}
+
+func (ri *reIndexer) fetchSinglePage(ctx context.Context, fromTime time.Time, toTime *time.Time, cursor string) (*feedResponse, error) {
+	qs := url.Values{
+		"from":  []string{fromTime.UTC().Format(time.RFC3339)},
+		"limit": []string{"500"},
+	}
+	if toTime != nil {
+		qs.Set("to", toTime.UTC().Format(time.RFC3339))
+	}
+	if cursor != "" {
+		qs.Set("cursor", cursor)
+	}
+
+	req, err := ri.api.Feed(qs)
+	if err != nil {
+		ri.log.Error(fmt.Sprintf("failed to prepare feed request: %s", err))
+		return nil, err
+	}
+
+	rsp, err := ri.api.HttpClient().Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer rsp.Body.Close()
+
+	if rsp.StatusCode != http.StatusOK {
+		ri.log.Error(fmt.Sprintf("request resulted in an unexpected status '%s' for feed", rsp.Status))
+		return nil, fmt.Errorf("unexpected status: %s", rsp.Status)
+	}
+
+	feeds := &feedResponse{}
+	if err = json.NewDecoder(rsp.Body).Decode(feeds); err != nil {
+		ri.log.Error(fmt.Sprintf("failed to decode feed response: %s", err))
+		return nil, err
+	}
+
+	return feeds, nil
+}
+
+func (ri *reIndexer) processBacklogInChunks(ctx context.Context, fromTime time.Time, toTime *time.Time) error {
+	esb, err := ri.es.BulkIndexer()
+	if err != nil {
+		return err
+	}
+	defer esb.Close(ctx)
+
+	cursor := ""
+	totalIndexed := 0
+
+	for {
+		feeds, err := ri.fetchSinglePage(ctx, fromTime, toTime, cursor)
+		if err != nil {
+			return err
+		}
+
+		if feeds != nil && feeds.Response != nil && len(feeds.Response.ActivityLogs) > 0 {
+			ri.log.Info(fmt.Sprintf("indexing %d records (page)", len(feeds.Response.ActivityLogs)))
+			if err := ri.feedReindexChanges(ctx, esb, "private", feeds.Response.ActivityLogs); err != nil {
+				return fmt.Errorf("failed to index records: %w", err)
+			}
+			totalIndexed += len(feeds.Response.ActivityLogs)
+		}
+
+		if feeds == nil || feeds.Response == nil || feeds.Response.Filter.NextPage == "" {
+			break
+		}
+
+		cursor = feeds.Response.Filter.NextPage
+		ri.log.Debug(fmt.Sprintf("fetching next page with cursor: %s", cursor))
+	}
+
+	if totalIndexed > 0 {
+		ri.log.Info(fmt.Sprintf("completed indexing %d total records for time chunk", totalIndexed))
+	} else {
+		ri.log.Debug(fmt.Sprintf("no feed changes since %s; current time: %s", fromTime.UTC().String(), time.Now().UTC().String()))
+	}
+
+	return nil
+}

extra/server-discovery/pkg/healthcheck/check.go

@@ -0,0 +1,120 @@
+package healthcheck
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"strings"
+)
+
+type (
+	checkFn func(ctx context.Context) error
+
+	Meta struct {
+		Label       string
+		Description string
+	}
+
+	check struct {
+		fn checkFn
+		*Meta
+	}
+
+	result struct {
+		err error
+		*Meta
+	}
+
+	results []*result
+	checks  struct {
+		cc []*check
+	}
+)
+
+var (
+	defaults *checks
+)
+
+func init() {
+	defaults = New()
+}
+
+func Defaults() *checks {
+	return defaults
+}
+
+func New() *checks {
+	return &checks{cc: []*check{}}
+}
+
+// Add appends new check
+func (c *checks) Add(fn checkFn, label string, description ...string) {
+	c.cc = append(c.cc, &check{fn, &Meta{Label: label, Description: strings.Join(description, "")}})
+}
+
+func (c checks) Run(ctx context.Context) results {
+	var rr = make([]*result, len(c.cc))
+
+	for i, c := range c.cc {
+		rr[i] = &result{c.fn(ctx), c.Meta}
+	}
+
+	return rr
+}
+
+func (rr results) Healthy() bool {
+	for _, c := range rr {
+		if c.err != nil {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (rr results) String() string {
+	buf := &bytes.Buffer{}
+
+	rr.WriteTo(buf)
+
+	return buf.String()
+}
+
+func (rr results) WriteTo(w io.Writer) (int64, error) {
+	var (
+		p = func(f string, aa ...interface{}) {
+			_, _ = fmt.Fprintf(w, f, aa...)
+		}
+	)
+
+	for _, r := range rr {
+		if r.IsHealthy() {
+			p("PASS")
+		} else {
+			p("FAIL")
+		}
+
+		p(" %s", r.Label)
+
+		if !r.IsHealthy() {
+			p(": %v", r.Error())
+		}
+
+		p("\n")
+	}
+
+	return 0, nil
+}
+
+func (r *result) IsHealthy() bool {
+	return r != nil && r.err == nil
+}
+
+func (r *result) Error() string {
+	if r == nil || r.err == nil {
+		return ""
+	}
+
+	return r.err.Error()
+}

extra/server-discovery/pkg/healthcheck/check_test.go

@@ -0,0 +1,68 @@
+package healthcheck
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_Healthy(t *testing.T) {
+	tests := []struct {
+		name    string
+		checks  []*check
+		healthy bool
+		string  string
+	}{
+		{
+			"should be healthy with handle for stringer output",
+			[]*check{{func(ctx context.Context) error { return nil }, &Meta{Label: "check01"}}},
+			true,
+			"PASS check01\n",
+		},
+		{
+			"should handle multiple healthy checks",
+			[]*check{
+				{func(ctx context.Context) error { return nil }, &Meta{Label: "check01"}},
+				{func(ctx context.Context) error { return nil }, &Meta{Label: "check02"}},
+			},
+			true,
+			"PASS check01\nPASS check02\n",
+		},
+		{
+			"should handle healthy and unhealthy checks",
+			[]*check{
+				{func(ctx context.Context) error { return nil }, &Meta{Label: "check01"}},
+				{func(ctx context.Context) error { return fmt.Errorf("x") }, &Meta{Label: "check02"}},
+				{func(ctx context.Context) error { return nil }, &Meta{Label: "check03"}},
+			},
+			false,
+			"PASS check01\nFAIL check02: x\nPASS check03\n",
+		},
+		{
+			"should handle labels",
+			[]*check{
+				{func(ctx context.Context) error { return nil }, &Meta{Label: "check01"}},
+				{func(ctx context.Context) error { return nil }, &Meta{Label: "Pretty check"}},
+			},
+			true,
+			"PASS check01\nPASS Pretty check\n",
+		},
+		{
+			"should handle empty check list",
+			[]*check{},
+			true,
+			"",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			a := assert.New(t)
+			r := (&checks{cc: tt.checks}).Run(context.Background())
+			a.Equal(tt.healthy, r.Healthy(), "healthy result failed")
+			a.Equal(tt.string, r.String(), "stringer output match failed")
+		})
+	}
+}

extra/server-discovery/pkg/healthcheck/http_handler.go

@@ -0,0 +1,16 @@
+package healthcheck
+
+import "net/http"
+
+func HttpHandler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		results := Defaults().Run(r.Context())
+		if results.Healthy() {
+			w.WriteHeader(http.StatusOK)
+		} else {
+			w.WriteHeader(http.StatusInternalServerError)
+		}
+
+		results.WriteTo(w)
+	}
+}

extra/server-discovery/pkg/options/HttpServer.go

@@ -0,0 +1,43 @@
+package options
+
+import "os"
+
+type (
+	HttpServerOpt struct {
+		Addr                   string `env:"HTTP_ADDR"`
+		EnableHealthcheckRoute bool   `env:"HTTP_ENABLE_HEALTHCHECK_ROUTE"`
+		EnableVersionRoute     bool   `env:"HTTP_ENABLE_VERSION_ROUTE"`
+		BaseUrl                string `env:"HTTP_BASE_URL"`
+		ApiBaseUrl             string `env:"HTTP_API_BASE_URL"`
+	}
+)
+
+// HttpServer initializes and returns a HTTPServerOpt with default values
+func HttpServer() (o *HttpServerOpt) {
+	o = &HttpServerOpt{
+		Addr:                   ":80",
+		EnableHealthcheckRoute: true,
+		EnableVersionRoute:     true,
+		BaseUrl:                "/",
+		ApiBaseUrl:             "/",
+	}
+
+	fill(o)
+
+	// Function that allows access to custom logic inside the parent function.
+	// The custom logic in the other file should be like:
+	// func (o *HTTPServer) Defaults() {...}
+	func(o interface{}) {
+		if def, ok := o.(interface{ Defaults() }); ok {
+			def.Defaults()
+		}
+	}(o)
+
+	return
+}
+
+func (o *HttpServerOpt) Defaults() {
+	if val, has := os.LookupEnv("PORT"); has {
+		o.Addr = ":" + val
+	}
+}

extra/server-discovery/pkg/options/corteza.go

@@ -0,0 +1,41 @@
+package options
+
+import (
+	"fmt"
+	"github.com/cortezaproject/corteza/server/pkg/options"
+)
+
+type (
+	CortezaOpt struct {
+		BaseUrl      string
+		AuthUrl      string
+		DiscoveryUrl string
+	}
+)
+
+const (
+	envKeyBaseUrl      = "CORTEZA_SERVER_BASE_URL"
+	envKeyAuthUrl      = "CORTEZA_SERVER_AUTH_URL"
+	envKeyDiscoveryUrl = "CORTEZA_SERVER_DISCOVERY_URL"
+)
+
+func Corteza() (o *CortezaOpt, err error) {
+	o = &CortezaOpt{}
+
+	return o, func() error {
+		baseUrl := options.EnvString(envKeyBaseUrl, "http://server:80")
+		o.BaseUrl = baseUrl
+
+		o.AuthUrl = options.EnvString(envKeyAuthUrl, baseUrl+"/auth")
+		if o.AuthUrl == "" {
+			return fmt.Errorf("corteza Auth endpoint value empty, set it directly with %s or indirectly with %s", envKeyAuthUrl, envKeyBaseUrl)
+		}
+
+		o.DiscoveryUrl = options.EnvString(envKeyDiscoveryUrl, baseUrl+"/api/discovery")
+		if o.DiscoveryUrl == "" {
+			return fmt.Errorf("corteza Discovery API endpoint value empty, set it directly with %s or indirectly with %s", envKeyDiscoveryUrl, envKeyBaseUrl)
+		}
+
+		return nil
+	}()
+}

extra/server-discovery/pkg/options/environment.go

@@ -0,0 +1,42 @@
+package options
+
+import "strings"
+
+type (
+	EnvironmentOpt struct {
+		Environment string `env:"ENVIRONMENT"`
+	}
+)
+
+// Environment initializes and returns a EnvironmentOpt with default values
+func Environment() (o *EnvironmentOpt) {
+	o = &EnvironmentOpt{
+		Environment: "production",
+	}
+
+	fill(o)
+
+	// Function that allows access to custom logic inside the parent function.
+	// The custom logic in the other file should be like:
+	// func (o *Environment) Defaults() {...}
+	func(o interface{}) {
+		if def, ok := o.(interface{ Defaults() }); ok {
+			def.Defaults()
+		}
+	}(o)
+
+	return
+}
+
+func (e EnvironmentOpt) IsDevelopment() bool {
+	return strings.HasPrefix(e.Environment, "dev")
+}
+
+func (e EnvironmentOpt) IsTest() bool {
+	return strings.HasPrefix(e.Environment, "test")
+}
+
+func (e EnvironmentOpt) IsProduction() bool {
+	return !e.IsDevelopment() &&
+		!e.IsTest()
+}

extra/server-discovery/pkg/options/es.go

@@ -0,0 +1,43 @@
+package options
+
+import (
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/options"
+)
+
+type (
+	EsOpt struct {
+		Addresses            []string `env:"ES_ADDRESS"`
+		Username             string   `env:"ES_USERNAME"`
+		Password             string   `env:"ES_PASSWORD"`
+		CertFile             string   `env:"ES_CERT_FILE"`
+		Secure               bool     `env:"ES_SECURE"`
+		EnableRetryOnTimeout bool     `env:"ES_ENABLE_RETRY_ON_TIMEOUT"`
+		MaxRetries           int      `env:"ES_MAX_RETRIES"`
+		IndexInterval        int      `env:"ES_INDEX_INTERVAL"`
+		IndexBackFillMonths  int      `env:"ES_INDEX_BACKFILL_MONTHS"`
+	}
+)
+
+func ES() (o *EsOpt, err error) {
+	o = &EsOpt{}
+	return o, func() error {
+		o.Username = options.EnvString("ES_USERNAME", "")
+		o.Password = options.EnvString("ES_PASSWORD", "")
+		o.Secure = options.EnvBool("ES_SECURE", true)
+		o.CertFile = options.EnvString("ES_CERT_FILE", "")
+
+		o.EnableRetryOnTimeout = options.EnvBool("ES_ENABLE_RETRY_ON_TIMEOUT", true)
+		o.MaxRetries = options.EnvInt("ES_MAX_RETRIES", 5)
+		o.IndexInterval = options.EnvInt("ES_INDEX_INTERVAL", 30)
+		o.IndexBackFillMonths = options.EnvInt("ES_INDEX_BACKFILL_MONTHS", 0)
+
+		for _, a := range strings.Split(options.EnvString("ES_ADDRESS", "http://es:9200"), " ") {
+			if a = strings.TrimSpace(a); a != "" {
+				o.Addresses = append(o.Addresses, a)
+			}
+		}
+		return nil
+	}()
+}

extra/server-discovery/pkg/options/helpers.go

@@ -0,0 +1,169 @@
+package options
+
+import (
+	"os"
+	"path"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/spf13/cast"
+)
+
+func fill(opt interface{}) {
+	v := reflect.ValueOf(opt)
+	if v.Kind() != reflect.Ptr {
+		panic("expecting a pointer, not a value")
+	}
+
+	if v.IsNil() {
+		panic("nil pointer passed")
+	}
+
+	v = v.Elem()
+
+	length := v.NumField()
+	for i := 0; i < length; i++ {
+		f := v.Field(i)
+		t := v.Type().Field(i)
+		if tag := t.Tag.Get("env"); tag != "" {
+			if !f.CanSet() {
+				panic("unexpected pointer for field " + t.Name)
+			}
+
+			if f.Type() == reflect.TypeOf(time.Duration(1)) {
+				v.FieldByName(t.Name).SetInt(int64(EnvDuration(tag, time.Duration(f.Int()))))
+				continue
+			}
+
+			if f.Kind() == reflect.String {
+				v.FieldByName(t.Name).SetString(EnvString(tag, f.String()))
+				continue
+			}
+
+			if f.Kind() == reflect.Bool {
+				v.FieldByName(t.Name).SetBool(EnvBool(tag, f.Bool()))
+				continue
+			}
+
+			if f.Kind() == reflect.Int {
+				v.FieldByName(t.Name).SetInt(int64(EnvInt(tag, int(f.Int()))))
+				continue
+			}
+
+			if f.Kind() == reflect.Float32 {
+				v.FieldByName(t.Name).SetFloat(float64(EnvFloat32(tag, float32(f.Float()))))
+				continue
+			}
+
+			panic("unsupported type/kind for field " + t.Name)
+		}
+	}
+}
+
+func guessHostname() string {
+	// All env keys we'll check, first that has any value set, will be used as hostname
+	candidates := []string{
+		os.Getenv("DOMAIN"),
+		os.Getenv("LETSENCRYPT_HOST"),
+		os.Getenv("VIRTUAL_HOST"),
+		os.Getenv("HOSTNAME"),
+		os.Getenv("HOST"),
+	}
+
+	for _, host := range candidates {
+		if len(host) > 0 {
+			return host
+		}
+	}
+
+	return "local.cortezaproject.org"
+}
+
+// returns path prefix
+func pathPrefix(pp ...string) string {
+	return CleanBase(append([]string{EnvString("HTTP_BASE_URL", "")}, pp...)...)
+}
+
+// will return base URL with domain and prefix path
+func fullURL(pp ...string) string {
+	var (
+		full string
+		host = guessHostname()
+	)
+
+	if strings.Contains(host, "local.") || strings.Contains(host, "localhost") || !isSecure() {
+		full = "http://" + host
+	} else {
+		full = "https://" + host
+	}
+
+	if pfix := pathPrefix(pp...); pfix != "" {
+		return full + pfix
+	}
+
+	return full
+}
+
+// Checks value of HTTP_SSL_TERMINATED and fallbacks to checking existence of LETSENCRYPT_HOST
+// to determinate we are behind a SSL termination infrastructure
+func isSecure() bool {
+	var _, is = os.LookupEnv("LETSENCRYPT_HOST")
+
+	// We're accessing to HTTP_SSL_TERMINATED directly here and not via HttpServerOpt.SslTerminated
+	// because we need to get to this value early and directly
+	return EnvBool("HTTP_SSL_TERMINATED", is)
+}
+
+// Path joins all parts with / and prefixes result (if not empty) with /
+func CleanBase(pp ...string) string {
+	//if p := strings.Trim(path.Join(pp...), "/"); p != "" {
+	//	// prefix base with slash
+	//	return "/" + p
+	//}
+
+	return "/" + strings.Trim(path.Join(pp...), "/")
+}
+
+func EnvString(key string, def string) string {
+	if val, has := os.LookupEnv(key); has {
+		return val
+	}
+	return def
+}
+
+func EnvBool(key string, def bool) bool {
+	if val, has := os.LookupEnv(key); has {
+		if b, err := cast.ToBoolE(val); err == nil {
+			return b
+		}
+	}
+	return def
+}
+
+func EnvInt(key string, def int) int {
+	if val, has := os.LookupEnv(key); has {
+		if i, err := cast.ToIntE(val); err == nil {
+			return i
+		}
+	}
+	return def
+}
+
+func EnvFloat32(key string, def float32) float32 {
+	if val, has := os.LookupEnv(key); has {
+		if i, err := cast.ToFloat32E(val); err == nil {
+			return i
+		}
+	}
+	return def
+}
+
+func EnvDuration(key string, def time.Duration) time.Duration {
+	if val, has := os.LookupEnv(key); has {
+		if d, err := cast.ToDurationE(val); err == nil {
+			return d
+		}
+	}
+	return def
+}

extra/server-discovery/pkg/options/indexer.go

@@ -0,0 +1,91 @@
+package options
+
+import (
+	"fmt"
+	"github.com/cortezaproject/corteza/server/pkg/options"
+	"os"
+	"strings"
+)
+
+type (
+	IndexerOpt struct {
+		Enabled bool
+		//HttpAddr             string
+		CortezaServerBaseUrl string
+		CortezaServerAuthUrl string
+		CortezaDiscoveryAPI  string
+
+		Schemas []*schema
+	}
+
+	schema struct {
+		IndexPrefix  string
+		ClientKey    string
+		ClientSecret string
+	}
+)
+
+const (
+	discoveryIndexer     = "DISCOVERY_INDEXER_"
+	indexerEnvKeyEnabled = discoveryIndexer + "ENABLED"
+	//indexerEnvKeyHttpAddr = discoveryIndexer + "HTTP_ADDR"
+)
+
+func Indexer() (o *IndexerOpt, err error) {
+	o = &IndexerOpt{}
+
+	return o, func() error {
+		o.Enabled = options.EnvBool(indexerEnvKeyEnabled, true)
+
+		//baseUrl := options.EnvString(envKeyBaseUrl, "http://server:80")
+
+		//o.HttpAddr = options.EnvString(indexerEnvKeyHttpAddr, "0.0.0.0:80")
+
+		//o.CortezaServerAuthUrl = options.EnvString(envKeyAuthUrl, baseUrl+"/auth")
+		//if o.CortezaServerAuthUrl == "" {
+		//	return fmt.Errorf("corteza Auth endpoint value empty, set it directly with %s or indirectly with %s", envKeyAuthUrl, envKeyBaseUrl)
+		//}
+		//
+		//o.CortezaDiscoveryAPI = options.EnvString(envKeyDiscoveryUrl, baseUrl+"/api/discovery")
+		//if o.CortezaDiscoveryAPI == "" {
+		//	return fmt.Errorf("corteza Discovery API endpoint value empty, set it directly with %s or indirectly with %s", envKeyDiscoveryUrl, envKeyBaseUrl)
+		//}
+
+		//o.IndexInterval = options.EnvInt(indexerEnvKeyIndexInterval, 30)
+
+		for _, ar := range []string{"public", "protected", "private"} {
+			var (
+				has  bool
+				ucAr = strings.ToUpper(ar)
+				s    = &schema{IndexPrefix: ar}
+
+				keyEnv = discoveryIndexer + ucAr + "_INDEX_CLIENT_KEY"
+				secEnv = discoveryIndexer + ucAr + "_INDEX_CLIENT_SECRET"
+			)
+
+			if s.ClientKey, has = os.LookupEnv(keyEnv); !has {
+				continue
+			} else if s.ClientKey == "" {
+				return fmt.Errorf("client key (%s) for '%s' is empty or missing", keyEnv, s.IndexPrefix)
+			}
+
+			if s.ClientSecret = os.Getenv(secEnv); s.ClientSecret == "" {
+				return fmt.Errorf("client secret (%s) for '%s' is empty or missing", secEnv, s.IndexPrefix)
+			}
+
+			o.Schemas = append(o.Schemas, s)
+		}
+
+		if len(o.Schemas) == 0 {
+			return fmt.Errorf("set at least one client secret pair using %s<PREFIX>_INDEX_CLIENT_KEY and <PREFIX>_INDEX_CLIENT_SECRET where prefix is one of 'public', 'protected' or 'private'", discoveryIndexer)
+		}
+
+		//for _, a := range strings.Split(options.EnvString(indexerEnvKeyEsAddr, "http://es:9200"), " ") {
+		//	if a = strings.TrimSpace(a); a != "" {
+		//		o.Es.Addresses = append(o.Es.Addresses, a)
+		//	}
+		//}
+
+		return nil
+	}()
+}

extra/server-discovery/pkg/options/options.go

@@ -0,0 +1,47 @@
+package options
+
+type (
+	Options struct {
+		Corteza     CortezaOpt
+		ES          EsOpt
+		Indexer     IndexerOpt
+		Searcher    SearcherOpt
+		Environment EnvironmentOpt
+		HTTPServer  HttpServerOpt
+		WaitFor     WaitForOpt
+	}
+)
+
+// @todo: add codegen for options
+
+func Init() (opt *Options, err error) {
+	indexer, err := Indexer()
+	if err != nil {
+		return
+	}
+
+	searcher, err := Searcher()
+	if err != nil {
+		return
+	}
+
+	es, err := ES()
+	if err != nil {
+		return
+	}
+
+	corteza, err := Corteza()
+	if err != nil {
+		return
+	}
+
+	return &Options{
+		Corteza:     *corteza,
+		ES:          *es,
+		Indexer:     *indexer,
+		Searcher:    *searcher,
+		Environment: *Environment(),
+		HTTPServer:  *HttpServer(),
+		WaitFor:     *WaitFor(),
+	}, nil
+}

extra/server-discovery/pkg/options/searcher.go

@@ -0,0 +1,82 @@
+package options
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/options"
+)
+
+type (
+	SearcherOpt struct {
+		Enabled bool
+		//HttpAddr     string
+		JwtSecret    []byte
+		ClientKey    string
+		ClientSecret string
+
+		// temp fix: remove it once allowed role is fixed on server
+		AllowedRole map[interface{}]bool
+	}
+)
+
+const (
+	discoverySearcher     = "DISCOVERY_SEARCHER_"
+	searcherEnvKeyEnabled = discoverySearcher + "ENABLED"
+	//searcherEnvKeyHttpAddr = discoverySearcher + "HTTP_ADDR"
+	envKeyJwtSecret    = discoverySearcher + "JWT_SECRET"
+	envKeyClientKey    = discoverySearcher + "CLIENT_KEY"
+	envKeyClientSecret = discoverySearcher + "CLIENT_SECRET"
+	envKeyAllowedRole  = discoverySearcher + "ALLOWED_ROLE"
+)
+
+func Searcher() (*SearcherOpt, error) {
+	o := &SearcherOpt{}
+
+	return o, func() error {
+		o.Enabled = options.EnvBool(searcherEnvKeyEnabled, true)
+
+		//o.CortezaServerBaseUrl = options.EnvString(envKeyBaseUrl, "http://server:80")
+		//if o.CortezaServerBaseUrl == "" {
+		//	return fmt.Errorf("endpoint URL for corteza (%s) is empty or missing", envKeyAuthUrl)
+		//}
+
+		//o.HttpAddr = options.EnvString(searcherEnvKeyHttpAddr, "0.0.0.0:80")
+
+		//o.CortezaServerAuthUrl = options.EnvString(envKeyAuthUrl, o.CortezaServerBaseUrl+"/auth")
+		//if o.CortezaServerAuthUrl == "" {
+		//	return fmt.Errorf("endpoint URL for corteza auth (%s) is empty or missing", envKeyAuthUrl)
+		//}
+
+		if tmp := os.Getenv(envKeyJwtSecret); tmp != "" {
+			o.JwtSecret = []byte(tmp)
+		}
+
+		if o.ClientKey = os.Getenv(envKeyClientKey); o.ClientKey == "" {
+			return fmt.Errorf("client key (%s) is empty or missing", envKeyClientKey)
+		}
+
+		if o.ClientSecret = os.Getenv(envKeyClientSecret); o.ClientSecret == "" {
+			return fmt.Errorf("client secret (%s) is empty or missing", envKeyClientSecret)
+		}
+
+		for _, a := range strings.Split(options.EnvString(envKeyAllowedRole, ""), " ") {
+			o.AllowedRole = make(map[interface{}]bool)
+			if a = strings.TrimSpace(a); a != "" {
+				o.AllowedRole[a] = false
+			}
+		}
+
+		//for _, a := range strings.Split(options.EnvString(envKeyEsAddr, "http://localhost:9200"), " ") {
+		//	if a = strings.TrimSpace(a); a != "" {
+		//		o.Es.Addresses = append(o.Es.Addresses, a)
+		//	}
+		//}
+
+		//o.EnableRetryOnTimeout = options.EnvBool(searcherEnvKeyEsEnableRetryOnTimeout, true)
+		//o.MaxRetries = options.EnvInt(searcherEnvKeyEsMaxRetries, 5)
+
+		return nil
+	}()
+}

extra/server-discovery/pkg/options/waitFor.go

@@ -0,0 +1,33 @@
+package options
+
+import (
+	"time"
+)
+
+type (
+	WaitForOpt struct {
+		Delay      time.Duration `env:"WAIT_FOR"`
+		StatusPage bool          `env:"WAIT_FOR_STATUS_PAGE"`
+	}
+)
+
+// WaitFor initializes and returns a WaitForOpt with default values
+func WaitFor() (o *WaitForOpt) {
+	o = &WaitForOpt{
+		Delay:      0,
+		StatusPage: true,
+	}
+
+	fill(o)
+
+	// Function that allows access to custom logic inside the parent function.
+	// The custom logic in the other file should be like:
+	// func (o *WaitFor) Defaults() {...}
+	func(o interface{}) {
+		if def, ok := o.(interface{ Defaults() }); ok {
+			def.Defaults()
+		}
+	}(o)
+
+	return
+}