snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/pkg/messagebus/consumer/eventbus.go

@@ -0,0 +1,38 @@
+package consumer
+
+import (
+	"context"
+
+	"github.com/cortezaproject/corteza/server/pkg/eventbus"
+	"github.com/cortezaproject/corteza/server/pkg/messagebus/types"
+)
+
+type (
+	Dispatcher interface {
+		Dispatch(ctx context.Context, ev eventbus.Event)
+		WaitFor(ctx context.Context, ev eventbus.Event) (err error)
+	}
+
+	EventbusConsumer struct {
+		queue      string
+		handle     types.ConsumerType
+		dispatcher Dispatcher
+		servicer   types.QueueEventBuilder
+	}
+)
+
+func NewEventbusConsumer(q string, servicer types.QueueEventBuilder) *EventbusConsumer {
+	h := &EventbusConsumer{
+		queue:      q,
+		handle:     types.ConsumerEventbus,
+		dispatcher: eventbus.Service(),
+		servicer:   servicer,
+	}
+
+	return h
+}
+
+func (cq *EventbusConsumer) Write(ctx context.Context, p []byte) error {
+	cq.dispatcher.Dispatch(ctx, cq.servicer.CreateQueueEvent(cq.queue, p))
+	return nil
+}

server/pkg/messagebus/consumer/store.go

@@ -0,0 +1,40 @@
+package consumer
+
+import (
+	"context"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/pkg/messagebus/store"
+	"github.com/cortezaproject/corteza/server/pkg/messagebus/types"
+)
+
+type (
+	StoreConsumer struct {
+		queue  string
+		handle types.ConsumerType
+		client store.StoreClient
+		poll   *time.Ticker
+	}
+)
+
+func NewStoreConsumer(q string, s types.QueueServicer) *StoreConsumer {
+	h := &StoreConsumer{
+		queue:  q,
+		handle: types.ConsumerStore,
+		client: store.NewClient(s),
+	}
+
+	return h
+}
+
+func (cq *StoreConsumer) Write(ctx context.Context, p []byte) error {
+	return cq.client.Add(ctx, cq.queue, p)
+}
+
+func (cq *StoreConsumer) SetStore(s types.QueueStorer) {
+	cq.client.SetStore(s)
+}
+
+func (cq *StoreConsumer) GetStore() types.QueueStorer {
+	return cq.client.GetStore()
+}

server/pkg/messagebus/consumer/store_test.go

@@ -0,0 +1,94 @@
+package consumer
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/pkg/messagebus/store"
+	"github.com/cortezaproject/corteza/server/pkg/messagebus/types"
+	"github.com/stretchr/testify/require"
+)
+
+type (
+	mockClient struct {
+		add     func(context.Context, string, []byte) (err error)
+		process func(context.Context, uint64, types.QueueMessage) (err error)
+
+		setStore func(types.QueueStorer)
+		getStore func() types.QueueStorer
+	}
+)
+
+var (
+	successfulClient = mockClient{
+		add: func(c context.Context, q string, p []byte) error {
+			return nil
+		},
+		process: func(c context.Context, u uint64, qm types.QueueMessage) (err error) { return },
+	}
+
+	unsuccessfulClient = mockClient{
+		add: func(c context.Context, q string, p []byte) error {
+			return fmt.Errorf("could not write messages")
+		},
+		process: func(c context.Context, u uint64, qm types.QueueMessage) (err error) {
+			return fmt.Errorf("could not process messages")
+		},
+	}
+)
+
+func Test_handlerSqlWrite(t *testing.T) {
+	var (
+		ctx = context.Background()
+		tcc = []struct {
+			name    string
+			err     error
+			payload []byte
+			client  store.StoreClient
+		}{
+			{
+				name:   "write success",
+				err:    nil,
+				client: &successfulClient,
+			},
+			{
+				name:   "write error",
+				err:    fmt.Errorf("could not write messages"),
+				client: &unsuccessfulClient,
+			},
+		}
+	)
+
+	for _, tc := range tcc {
+		t.Run(tc.name, func(t *testing.T) {
+			req := require.New(t)
+			h := StoreConsumer{queue: "foobar", handle: types.ConsumerStore, client: tc.client}
+
+			err := h.Write(ctx, []byte("foo bar"))
+
+			req.Equal(tc.err, err)
+		})
+	}
+}
+
+func (mc *mockClient) Add(ctx context.Context, q string, p []byte) error {
+	return mc.add(ctx, q, p)
+}
+
+func (mc *mockClient) Process(ctx context.Context, ID uint64, m types.QueueMessage) error {
+	return mc.process(ctx, ID, m)
+}
+
+func (mc *mockClient) GetStore() types.QueueStorer {
+	return mc.getStore()
+}
+
+func (mc *mockClient) SetStore(s types.QueueStorer) {
+	mc.setStore(s)
+}
+
+func makeDelay(d time.Duration) *time.Duration {
+	return &d
+}

server/pkg/messagebus/service.go

@@ -0,0 +1,223 @@
+package messagebus
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/cortezaproject/corteza/server/pkg/messagebus/consumer"
+	"github.com/cortezaproject/corteza/server/pkg/messagebus/store"
+	"github.com/cortezaproject/corteza/server/pkg/messagebus/types"
+	"github.com/cortezaproject/corteza/server/pkg/options"
+	"go.uber.org/zap"
+)
+
+var (
+	// global service
+	gMbus *messageBus
+)
+
+type (
+	messageBus struct {
+		opts      *options.MessagebusOpt
+		qservicer types.QueueServicer
+		logger    *zap.Logger
+		mutex     sync.Mutex
+		queues    types.QueueSet
+
+		in     chan types.Message
+		quit   chan bool
+		reload chan bool
+	}
+)
+
+func Service() *messageBus {
+	return gMbus
+}
+
+func Set(m *messageBus) {
+	gMbus = m
+}
+
+// Setup handles the singleton service
+func Setup(opts *options.MessagebusOpt, log *zap.Logger) {
+	if gMbus != nil {
+		return
+	}
+
+	gMbus = New(opts, log)
+}
+
+func New(opts *options.MessagebusOpt, logger *zap.Logger) *messageBus {
+	return &messageBus{
+		opts:   opts,
+		logger: logger,
+		queues: types.QueueSet{},
+
+		in:     make(chan types.Message),
+		quit:   make(chan bool),
+		reload: make(chan bool),
+	}
+}
+
+// Init takes care of preloading the queue and creating
+// a connection to the store of their choice
+func (mb *messageBus) Init(ctx context.Context, storer types.QueueServicer) {
+	// set store now, on New() we do not have store yet
+	mb.qservicer = storer
+
+	mb.initQueues(ctx)
+}
+
+// Listen sets read and write channel listeners,
+// used on boot
+func (mb *messageBus) Listen(ctx context.Context) {
+	// check in channel and act accordingly
+	go func() {
+		for {
+			select {
+			case t := <-mb.in:
+				q := mb.queue(t.Q)
+				log := mb.logger.With(zap.String("queue", t.Q))
+
+				if q == nil {
+					log.Warn("could not get queue settings")
+					break
+				}
+
+				// get the consumer
+				err := q.Consumer.Write(ctx, t.P)
+
+				if err != nil {
+					log.Warn("could not add message to queue", zap.Error(err))
+					break
+				}
+
+				log.Debug("wrote payload to queue")
+
+			case <-mb.quit:
+				mb.logger.Debug("quitting from messagebus listener")
+				return
+
+			case <-ctx.Done():
+				mb.logger.Debug("quitting from messagebus listener")
+				return
+			}
+		}
+	}()
+}
+
+// Watch checks the channel for restart and loads the queues
+// and adds the listeners again (the same process as on boot)
+func (mb *messageBus) Watch(ctx context.Context, storer types.QueueServicer) {
+	go func() {
+		for {
+			select {
+			case <-mb.reload:
+				mb.logger.Debug("reloading queues")
+
+				// refresh queue list
+				mb.Init(ctx, storer)
+
+				// reload listeners, we removed them in ReloadQueues()
+				mb.Listen(ctx)
+
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+}
+
+// ReloadQueues sends an error signal to all of the read and
+// write watchers, dispatch and processed signal watchers
+//
+// Once the watchers are finished, the Init() and Listen()
+// reruns with the new list of queues
+func (mb *messageBus) ReloadQueues() {
+	go func() {
+		// quit all the listeners
+		mb.quit <- true
+
+		// re-add all the watchers
+		mb.reload <- true
+	}()
+}
+
+func (mb *messageBus) Push(q string, p []byte) {
+	if !mb.opts.Enabled {
+		mb.logger.Debug("message will not be sent, messagebus disabled", zap.String("queue", q))
+		return
+	}
+
+	mb.logger.Debug("pushing message", zap.String("queue", q))
+
+	mb.in <- types.Message{P: p, Q: q}
+}
+
+func (mb *messageBus) Register(ctx context.Context, qs *types.Queue) {
+	mb.queues[qs.Name] = qs
+}
+
+func (mb *messageBus) queue(q string) *types.Queue {
+	if _, ok := mb.queues[q]; ok {
+		return mb.queues[q]
+	}
+
+	return nil
+}
+
+func (mb *messageBus) initQueues(ctx context.Context) error {
+	list, _, err := mb.qservicer.SearchQueues(ctx, types.QueueFilter{})
+
+	if err != nil {
+		return err
+	}
+
+	mb.mutex.Lock()
+	defer mb.mutex.Unlock()
+
+	// empty first
+	mb.queues = make(types.QueueSet)
+
+	// add to list of consumers
+	for _, q := range list {
+		mb.logger.Debug("initializing queue", zap.String("queue", q.Queue))
+
+		c, err := mb.initConsumer(ctx, q.Queue, q.Consumer)
+
+		if err != nil {
+			mb.logger.Warn("could not init consumer for queue", zap.String("queue", q.Queue), zap.Error(err))
+			continue
+		}
+
+		if _, is := c.(store.Storer); is {
+			c.(store.Storer).SetStore(mb.qservicer)
+		}
+
+		mb.Register(ctx, &types.Queue{
+			Consumer: c,
+			Name:     q.Queue,
+			Meta:     types.QueueMeta(q.Meta),
+		})
+	}
+
+	return nil
+}
+
+// initHandler returns a new instance for a specific handler
+func (mb *messageBus) initConsumer(ctx context.Context, q string, c string) (cns types.Consumer, err error) {
+	switch c {
+	case string(types.ConsumerEventbus):
+		cns = consumer.NewEventbusConsumer(q, mb.qservicer)
+		return
+
+	case string(types.ConsumerStore):
+		cns = consumer.NewStoreConsumer(q, mb.qservicer)
+		return
+
+	default:
+		err = fmt.Errorf("message queue consumer %s not implemented", c)
+		return
+	}
+}

server/pkg/messagebus/service_test.go

@@ -0,0 +1,124 @@
+package messagebus
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/pkg/eventbus"
+	"github.com/cortezaproject/corteza/server/pkg/messagebus/types"
+	"github.com/cortezaproject/corteza/server/pkg/options"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+)
+
+type (
+	mockStorer struct{}
+
+	// only use this one to get .messages
+	// from mockQueueHandler
+	messageTesterHandler interface {
+		GetAllMessages() [][]byte
+	}
+
+	mockDispatcher struct {
+		register func(h eventbus.HandlerFn, ops ...eventbus.HandlerRegOp) (p uintptr)
+		dispatch func(ctx context.Context, ev eventbus.Event)
+		waitFor  func(ctx context.Context, ev eventbus.Event) (err error)
+	}
+
+	mockQueueHandler struct {
+		messages     [][]byte
+		notification func(ctx context.Context) <-chan interface{}
+		ticker       func(ctx context.Context) <-chan time.Time
+		write        func(ctx context.Context, p []byte) error
+		setStore     func(qs types.QueueStorer)
+	}
+)
+
+var (
+	logger              = zap.NewNop()
+	mOptions            = &options.MessagebusOpt{Enabled: true, LogEnabled: true}
+	foobarQueueSettings = types.Queue{Consumer: nil, Name: "foobar"}
+)
+
+func Test_messageBusRegister(t *testing.T) {
+	req := require.New(t)
+	ctx := context.Background()
+
+	mb := New(mOptions, logger)
+	mb.Register(ctx, &types.Queue{Name: "foobar", Consumer: &mockQueueHandler{}})
+
+	req.NotEmpty(mb.queues)
+	req.NotEmpty(mb.queues["foobar"])
+	req.Empty(mb.queues["non_existing_queue"])
+}
+
+func Test_consume(t *testing.T) {
+	req := require.New(t)
+	ctx := context.Background()
+	w := sync.WaitGroup{}
+
+	mb := New(mOptions, logger)
+
+	mockDb := [][]byte{}
+	expectedDb := [][]byte{
+		[]byte("mock payload"),
+		[]byte("second mock payload"),
+	}
+
+	mb.queues[foobarQueueSettings.Name] = &types.Queue{
+		Consumer: &mockQueueHandler{
+			write: func(ctx context.Context, p []byte) error {
+				mockDb = append(mockDb, p)
+				w.Done()
+				return nil
+			},
+		},
+	}
+
+	mb.Listen(ctx)
+
+	w.Add(2)
+	mb.Push("foobar", expectedDb[0])
+	mb.Push("foobar", expectedDb[1])
+	w.Wait()
+
+	req.Equal(expectedDb, mockDb)
+}
+
+func (mh *mockQueueHandler) Notification(ctx context.Context) <-chan interface{} {
+	return mh.notification(ctx)
+}
+func (mh *mockQueueHandler) Ticker(ctx context.Context) <-chan time.Time {
+	return mh.ticker(ctx)
+}
+func (mh *mockQueueHandler) Write(ctx context.Context, p []byte) error {
+	return mh.write(ctx, p)
+}
+func (mh *mockQueueHandler) SetStore(qs types.QueueStorer) {
+	mh.setStore(qs)
+}
+
+func (md mockDispatcher) Register(h eventbus.HandlerFn, ops ...eventbus.HandlerRegOp) (p uintptr) {
+	return md.register(h, ops...)
+}
+func (md mockDispatcher) Dispatch(ctx context.Context, ev eventbus.Event) {
+	md.dispatch(ctx, ev)
+}
+func (md mockDispatcher) WaitFor(ctx context.Context, ev eventbus.Event) (err error) {
+	return md.waitFor(ctx, ev)
+}
+
+func tickOnce(tt time.Ticker) {
+	go func() {
+		for ; true; <-tt.C {
+		}
+	}()
+}
+
+func now() *time.Time {
+	t := time.Now()
+	return &t
+}

server/pkg/messagebus/store/client.go

@@ -0,0 +1,50 @@
+package store
+
+import (
+	"context"
+
+	"github.com/cortezaproject/corteza/server/pkg/messagebus/types"
+)
+
+type (
+	Client interface {
+		Add(context.Context, string, []byte) error
+		Process(context.Context, uint64, types.QueueMessage) error
+	}
+
+	StoreClient interface {
+		Client
+		Storer
+	}
+
+	sClient struct {
+		store types.QueueStorer
+	}
+)
+
+func NewClient(s types.QueueStorer) *sClient {
+	return &sClient{store: s}
+}
+
+func (c *sClient) Process(ctx context.Context, ID uint64, m types.QueueMessage) (err error) {
+	err = c.store.ProcessQueueMessage(ctx, ID, m)
+
+	return
+}
+
+func (c *sClient) Add(ctx context.Context, q string, payload []byte) (err error) {
+	err = c.store.CreateQueueMessage(ctx, types.QueueMessage{
+		Queue:   q,
+		Payload: payload,
+	})
+
+	return
+}
+
+func (c *sClient) GetStore() types.QueueStorer {
+	return c.store
+}
+
+func (c *sClient) SetStore(s types.QueueStorer) {
+	c.store = s
+}

server/pkg/messagebus/store/storer.go

@@ -0,0 +1,12 @@
+package store
+
+import (
+	"github.com/cortezaproject/corteza/server/pkg/messagebus/types"
+)
+
+type (
+	Storer interface {
+		SetStore(types.QueueStorer)
+		GetStore() types.QueueStorer
+	}
+)

server/pkg/messagebus/types/consumer.go

@@ -0,0 +1,30 @@
+package types
+
+import (
+	"context"
+)
+
+const (
+	ConsumerCorteza  ConsumerType = "corteza"
+	ConsumerNoop     ConsumerType = "noop"
+	ConsumerRedis    ConsumerType = "redis"
+	ConsumerStore    ConsumerType = "store"
+	ConsumerEventbus ConsumerType = "eventbus"
+)
+
+type (
+	ConsumerType string
+
+	Consumer interface {
+		Write(ctx context.Context, p []byte) error
+	}
+)
+
+func ConsumerTypes() []ConsumerType {
+	return []ConsumerType{
+		ConsumerCorteza,
+		ConsumerEventbus,
+		ConsumerRedis,
+		ConsumerStore,
+	}
+}

server/pkg/messagebus/types/producer.go

@@ -0,0 +1,26 @@
+package types
+
+import (
+	"context"
+	"time"
+)
+
+type (
+	Producer interface {
+		Reader
+		Subscriber
+		Poller
+	}
+
+	Subscriber interface {
+		Subscribe(ctx context.Context) <-chan interface{}
+	}
+
+	Poller interface {
+		Poll(ctx context.Context) <-chan time.Time
+	}
+
+	Reader interface {
+		Read(ctx context.Context) ([]QueueMessage, error)
+	}
+)

server/pkg/messagebus/types/queue.go

@@ -0,0 +1,99 @@
+package types
+
+import (
+	"database/sql/driver"
+	"encoding/json"
+	"github.com/cortezaproject/corteza/server/pkg/sql"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/pkg/filter"
+	"github.com/spf13/cast"
+)
+
+type (
+	Message struct {
+		Q string
+		P []byte
+	}
+
+	Queue struct {
+		Name     string
+		Consumer Consumer
+		Meta     QueueMeta
+	}
+
+	QueueDb struct {
+		Queue    string
+		Consumer string
+		Meta     QueueMeta
+	}
+
+	QueueFilter struct {
+		Query   string
+		Deleted filter.State
+
+		filter.Sorting
+		filter.Paging
+	}
+
+	QueueSet map[string]*Queue
+
+	QueueMeta struct {
+		PollDelay      *time.Duration `json:"poll_delay"`
+		DispatchEvents bool           `json:"dispatch_events"`
+	}
+
+	QueueMessage struct {
+		Queue   string
+		Payload []byte
+	}
+
+	QueueMessageSet []QueueMessage
+)
+
+func (h *QueueMeta) UnmarshalJSON(s []byte) error {
+	type Alias QueueMeta
+
+	aux := &struct {
+		PollDelay string `json:"poll_delay"`
+		*Alias
+	}{
+		Alias: (*Alias)(h),
+	}
+
+	// set default
+	h.DispatchEvents = false
+
+	if err := json.Unmarshal(s, aux); err != nil {
+		return err
+	}
+
+	if d, err := cast.ToDurationE(aux.PollDelay); err == nil {
+		h.PollDelay = &d
+	}
+
+	return nil
+}
+
+func (m *QueueMeta) Scan(src any) error          { return sql.ParseJSON(src, m) }
+func (m QueueMeta) Value() (driver.Value, error) { return json.Marshal(m) }
+
+func (m QueueMeta) MarshalJSON() ([]byte, error) {
+
+	pollDelay := ""
+	if m.PollDelay != nil {
+		pollDelay = m.PollDelay.String()
+	}
+
+	return json.Marshal(struct {
+		PollDelay      string `json:"poll_delay"`
+		DispatchEvents bool   `json:"dispatch_events"`
+	}{
+		PollDelay:      pollDelay,
+		DispatchEvents: m.DispatchEvents,
+	})
+}
+
+func (s *Queue) CanDispatch() bool {
+	return s.Meta.DispatchEvents
+}

server/pkg/messagebus/types/storer.go

@@ -0,0 +1,24 @@
+package types
+
+import (
+	"context"
+
+	"github.com/cortezaproject/corteza/server/pkg/eventbus"
+)
+
+type (
+	QueueStorer interface {
+		SearchQueues(context.Context, QueueFilter) ([]QueueDb, QueueFilter, error)
+		CreateQueueMessage(context.Context, QueueMessage) error
+		ProcessQueueMessage(context.Context, uint64, QueueMessage) error
+	}
+
+	QueueEventBuilder interface {
+		CreateQueueEvent(string, []byte) eventbus.Event
+	}
+
+	QueueServicer interface {
+		QueueStorer
+		QueueEventBuilder
+	}
+)