snapshot of https://github.com/ThreeDotsLabs/watermill.git (master, 2026-09-15; blobs > 131072B omitted)

components/forwarder/envelope.go

@@ -0,0 +1,80 @@
+package forwarder
+
+import (
+	"github.com/ThreeDotsLabs/watermill"
+	"github.com/ThreeDotsLabs/watermill/message"
+	"github.com/pkg/errors"
+)
+
+// messageEnvelope wraps Watermill message and contains destination topic.
+type messageEnvelope struct {
+	DestinationTopic string `json:"destination_topic"`
+
+	UUID     string            `json:"uuid"`
+	Payload  []byte            `json:"payload"`
+	Metadata map[string]string `json:"metadata"`
+}
+
+func newMessageEnvelope(destTopic string, msg *message.Message) (*messageEnvelope, error) {
+	e := &messageEnvelope{
+		DestinationTopic: destTopic,
+		UUID:             msg.UUID,
+		Payload:          msg.Payload,
+		Metadata:         msg.Metadata,
+	}
+
+	if err := e.validate(); err != nil {
+		return nil, errors.Wrap(err, "cannot create a message envelope")
+	}
+
+	return e, nil
+}
+
+func (e *messageEnvelope) validate() error {
+	if e.DestinationTopic == "" {
+		return errors.New("unknown destination topic")
+	}
+
+	return nil
+}
+
+func wrapMessageInEnvelope(
+	destinationTopic string,
+	msg *message.Message,
+	marshaler Marshaler,
+) (*message.Message, error) {
+	envelope, err := newMessageEnvelope(destinationTopic, msg)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot envelope a message")
+	}
+
+	envelopedMessage, err := marshaler.Marshal(envelope)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot marshal a message")
+	}
+
+	wrappedMsg := message.NewMessage(watermill.NewUUID(), envelopedMessage)
+	wrappedMsg.SetContext(msg.Context())
+
+	return wrappedMsg, nil
+}
+
+func unwrapMessageFromEnvelope(
+	msg *message.Message,
+	marshaler Marshaler,
+) (destinationTopic string, unwrappedMsg *message.Message, err error) {
+	envelopedMsg := messageEnvelope{}
+	if err := marshaler.Unmarshal(msg.Payload, &envelopedMsg); err != nil {
+		return "", nil, errors.Wrap(err, "cannot unmarshal message wrapped in an envelope")
+	}
+
+	if err := envelopedMsg.validate(); err != nil {
+		return "", nil, errors.Wrap(err, "an unmarshalled message envelope is invalid")
+	}
+
+	watermillMessage := message.NewMessage(envelopedMsg.UUID, envelopedMsg.Payload)
+	watermillMessage.Metadata = envelopedMsg.Metadata
+	watermillMessage.SetContext(msg.Context())
+
+	return envelopedMsg.DestinationTopic, watermillMessage, nil
+}

components/forwarder/envelope_test.go

@@ -0,0 +1,46 @@
+package forwarder
+
+import (
+	"context"
+	"testing"
+
+	"github.com/ThreeDotsLabs/watermill"
+	"github.com/ThreeDotsLabs/watermill/message"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type contextKey string
+
+func TestEnvelope(t *testing.T) {
+	expectedUUID := watermill.NewUUID()
+	marshaler := DefaultMarshaler{}
+	expectedPayload := message.Payload("msg content")
+	expectedMetadata := message.Metadata{"key": "value"}
+	expectedDestinationTopic := "dest_topic"
+
+	ctx := context.WithValue(context.Background(), contextKey("key"), "value")
+
+	msg := message.NewMessage(expectedUUID, expectedPayload)
+	msg.Metadata = expectedMetadata
+	msg.SetContext(ctx)
+
+	wrappedMsg, err := wrapMessageInEnvelope(expectedDestinationTopic, msg, marshaler)
+	require.NoError(t, err)
+	require.NotNil(t, wrappedMsg)
+	v, ok := wrappedMsg.Context().Value(contextKey("key")).(string)
+	require.True(t, ok)
+	require.Equal(t, "value", v)
+
+	destinationTopic, unwrappedMsg, err := unwrapMessageFromEnvelope(wrappedMsg, marshaler)
+	require.NoError(t, err)
+	require.NotNil(t, unwrappedMsg)
+	assert.Equal(t, expectedUUID, unwrappedMsg.UUID)
+	assert.Equal(t, expectedPayload, unwrappedMsg.Payload)
+	assert.Equal(t, expectedMetadata, unwrappedMsg.Metadata)
+	assert.Equal(t, expectedDestinationTopic, destinationTopic)
+
+	v, ok = unwrappedMsg.Context().Value(contextKey("key")).(string)
+	require.True(t, ok)
+	require.Equal(t, "value", v)
+}

components/forwarder/forwarder.go

@@ -0,0 +1,169 @@
+package forwarder
+
+import (
+	"context"
+	"time"
+
+	"github.com/ThreeDotsLabs/watermill"
+	"github.com/ThreeDotsLabs/watermill/message"
+	"github.com/pkg/errors"
+)
+
+const (
+	defaultForwarderTopic = "forwarder_topic"
+	defaultHandlerName    = "events_forwarder"
+)
+
+type Config struct {
+	// ForwarderTopic is a topic on which the forwarder will be listening to enveloped messages to forward.
+	// Defaults to `forwarder_topic`.
+	ForwarderTopic string
+
+	// HandlerName is the name of the forwarder's handler registered in the router.
+	// Defaults to `events_forwarder`.
+	//
+	// Handler names must be unique within a router, so it needs to be set explicitly
+	// if you want to run multiple forwarders on the same router.
+	HandlerName string
+
+	// Middlewares are used to decorate forwarder's handler function.
+	Middlewares []message.HandlerMiddleware
+
+	// CloseTimeout determines how long router should work for handlers when closing.
+	CloseTimeout time.Duration
+
+	// AckWhenCannotUnwrap enables acking of messages which cannot be unwrapped from an envelope.
+	AckWhenCannotUnwrap bool
+
+	// Router is a router used by the forwarder.
+	// If not provided, a new router will be created.
+	//
+	// If router is provided, it's not necessary to call `Forwarder.Run()` if the router is started with `router.Run()`.
+	Router *message.Router
+
+	// Marshaler is used to deserialize envelopes received on ForwarderTopic.
+	// It must match the Marshaler used by the decorated Publisher.
+	// Defaults to DefaultMarshaler, which uses encoding/json.
+	Marshaler Marshaler
+}
+
+func (c *Config) setDefaults() {
+	if c.CloseTimeout == 0 {
+		c.CloseTimeout = time.Second * 30
+	}
+	if c.ForwarderTopic == "" {
+		c.ForwarderTopic = defaultForwarderTopic
+	}
+	if c.HandlerName == "" {
+		c.HandlerName = defaultHandlerName
+	}
+	if c.Marshaler == nil {
+		c.Marshaler = DefaultMarshaler{}
+	}
+}
+
+func (c *Config) Validate() error {
+	if c.ForwarderTopic == "" {
+		return errors.New("empty forwarder topic")
+	}
+	if c.HandlerName == "" {
+		return errors.New("empty handler name")
+	}
+
+	return nil
+}
+
+// Forwarder subscribes to the topic provided in the config and publishes them to the destination topic embedded in the enveloped message.
+type Forwarder struct {
+	router    *message.Router
+	publisher message.Publisher
+	logger    watermill.LoggerAdapter
+	config    Config
+}
+
+// NewForwarder creates a forwarder which will subscribe to the topic provided in the config using the provided subscriber.
+// It will publish messages received on this subscription to the destination topic embedded in the enveloped message using the provided publisher.
+//
+// Provided subscriber and publisher can be from different Watermill Pub/Sub implementations, i.e. MySQL subscriber and Google Pub/Sub publisher.
+//
+// Note: Keep in mind that by default the forwarder will nack all messages which weren't sent using a decorated publisher.
+// You can change this behavior by passing a middleware which will ack them instead.
+func NewForwarder(
+	subscriberIn message.Subscriber,
+	publisherOut message.Publisher,
+	logger watermill.LoggerAdapter,
+	config Config,
+) (*Forwarder, error) {
+	config.setDefaults()
+
+	routerConfig := message.RouterConfig{CloseTimeout: config.CloseTimeout}
+	if err := routerConfig.Validate(); err != nil {
+		return nil, errors.Wrap(err, "invalid router config")
+	}
+
+	var router *message.Router
+	if config.Router != nil {
+		router = config.Router
+	} else {
+		var err error
+		router, err = message.NewRouter(routerConfig, logger)
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot create a router")
+		}
+	}
+
+	f := &Forwarder{router, publisherOut, logger, config}
+
+	handler := router.AddConsumerHandler(
+		config.HandlerName,
+		config.ForwarderTopic,
+		subscriberIn,
+		f.forwardMessage,
+	)
+
+	handler.AddMiddleware(config.Middlewares...)
+
+	return f, nil
+}
+
+// Run runs forwarder's handler responsible for forwarding messages.
+// This call is blocking while the forwarder is running.
+// ctx will be propagated to the forwarder's subscription.
+//
+// To stop Run() you should call Close() on the forwarder.
+func (f *Forwarder) Run(ctx context.Context) error {
+	return f.router.Run(ctx)
+}
+
+// Close stops forwarder's handler.
+func (f *Forwarder) Close() error {
+	return f.router.Close()
+}
+
+// Running returns channel which is closed when the forwarder is running.
+func (f *Forwarder) Running() chan struct{} {
+	return f.router.Running()
+}
+
+func (f *Forwarder) forwardMessage(msg *message.Message) error {
+	destTopic, unwrappedMsg, err := unwrapMessageFromEnvelope(msg, f.config.Marshaler)
+	if err != nil {
+		f.logger.Error("Could not unwrap a message from an envelope", err, watermill.LogFields{
+			"uuid":     msg.UUID,
+			"payload":  msg.Payload,
+			"metadata": msg.Metadata,
+			"acked":    f.config.AckWhenCannotUnwrap,
+		})
+
+		if f.config.AckWhenCannotUnwrap {
+			return nil
+		}
+		return errors.Wrap(err, "cannot unwrap message from an envelope")
+	}
+
+	if err := f.publisher.Publish(destTopic, unwrappedMsg); err != nil {
+		return errors.Wrap(err, "cannot publish a message")
+	}
+
+	return nil
+}

components/forwarder/forwarder_test.go

@@ -0,0 +1,263 @@
+package forwarder_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/ThreeDotsLabs/watermill"
+	"github.com/ThreeDotsLabs/watermill/components/forwarder"
+	"github.com/ThreeDotsLabs/watermill/message"
+	"github.com/ThreeDotsLabs/watermill/pubsub/gochannel"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+)
+
+var (
+	logger = watermill.NewStdLogger(true, true)
+
+	forwarderTopic = "forwarder_topic"
+	outTopic       = "out_topic"
+)
+
+func TestForwarder(t *testing.T) {
+	suite.Run(t, new(ForwarderSuite))
+}
+
+// ForwarderSuite tests forwarding messages from PubSubIn to PubSubOut (which are GoChannel implementation underneath).
+type ForwarderSuite struct {
+	suite.Suite
+	ctx       context.Context
+	cancelCtx func()
+
+	publisherIn  PubSubInPublisher
+	subscriberIn PubSubInSubscriber
+
+	publisherOut  PubSubOutPublisher
+	subscriberOut PubSubOutSubscriber
+
+	decoratedPublisherIn *forwarder.Publisher
+
+	outMessagesCh <-chan *message.Message
+}
+
+func (s *ForwarderSuite) SetupTest() {
+	// Create test context with a 5 seconds timeout so it will close any subscriptions/handlers running in the background
+	// in case of too long test execution.
+	s.ctx, s.cancelCtx = context.WithTimeout(context.Background(), time.Second*5)
+
+	// Create a set of publisher and subscribers for both In and Out Pub/Subs.
+	s.publisherIn, s.subscriberIn = newPubSubIn()
+	s.publisherOut, s.subscriberOut = newPubSubOut()
+
+	s.decoratedPublisherIn = forwarder.NewPublisher(s.publisherIn, forwarder.PublisherConfig{ForwarderTopic: forwarderTopic})
+	s.listenOnOutTopic()
+}
+
+func (s *ForwarderSuite) TearDownTest() {
+	s.NoError(s.publisherIn.Close())
+	s.NoError(s.subscriberIn.Close())
+	s.NoError(s.publisherOut.Close())
+	s.NoError(s.subscriberOut.Close())
+	s.cancelCtx()
+}
+
+func (s *ForwarderSuite) TestForwarder_publish_using_decorated_publisher() {
+	fwd := s.setupForwarder(forwarder.Config{ForwarderTopic: forwarderTopic})
+	defer func() {
+		s.NoError(fwd.Close())
+	}()
+
+	sentMsg := s.sampleMessage()
+	err := s.decoratedPublisherIn.Publish(outTopic, sentMsg)
+	s.Require().NoError(err)
+
+	s.requireFirstMessage(sentMsg)
+}
+
+func (s *ForwarderSuite) TestForwarder_publish_using_non_decorated_publisher() {
+	msgAckedDetectorMiddleware, msgAckedCh := s.setupMessageAckedDetectorMiddleware()
+	fwd := s.setupForwarder(forwarder.Config{
+		ForwarderTopic: forwarderTopic,
+		Middlewares:    []message.HandlerMiddleware{msgAckedDetectorMiddleware},
+	})
+	defer func() {
+		s.NoError(fwd.Close())
+	}()
+
+	sentMsg := s.sampleMessage()
+	err := s.publisherIn.Publish(forwarderTopic, sentMsg)
+	s.Require().NoError(err)
+
+	s.requireFirstAckingResult(msgAckedCh, false)
+}
+
+func (s *ForwarderSuite) TestForwarder_publish_using_non_decorated_publisher_acking_enabled() {
+	msgAckedDetectorMiddleware, msgAckedCh := s.setupMessageAckedDetectorMiddleware()
+	fwd := s.setupForwarder(forwarder.Config{
+		ForwarderTopic:      forwarderTopic,
+		Middlewares:         []message.HandlerMiddleware{msgAckedDetectorMiddleware},
+		AckWhenCannotUnwrap: true,
+	})
+	defer func() {
+		s.NoError(fwd.Close())
+	}()
+
+	sentMsg := s.sampleMessage()
+	err := s.publisherIn.Publish(forwarderTopic, sentMsg)
+	s.Require().NoError(err)
+
+	s.requireFirstAckingResult(msgAckedCh, true)
+}
+
+// TestForwarder_multiple_forwarders_on_one_router ensures that forwarders sharing a router
+// can coexist as long as they have unique handler names.
+func TestForwarder_multiple_forwarders_on_one_router(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	defer cancel()
+
+	pubSubIn := gochannel.NewGoChannel(gochannel.Config{}, logger)
+	pubSubOut := gochannel.NewGoChannel(gochannel.Config{}, logger)
+
+	router, err := message.NewRouter(message.RouterConfig{}, logger)
+	require.NoError(t, err)
+
+	handlerNames := []string{"first_forwarder", "second_forwarder"}
+
+	for _, handlerName := range handlerNames {
+		_, err := forwarder.NewForwarder(pubSubIn, pubSubOut, logger, forwarder.Config{
+			ForwarderTopic: handlerName + "_topic",
+			HandlerName:    handlerName,
+			Router:         router,
+		})
+		require.NoError(t, err)
+	}
+
+	go func() {
+		assert.NoError(t, router.Run(ctx))
+	}()
+	defer func() {
+		assert.NoError(t, router.Close())
+	}()
+
+	select {
+	case <-router.Running():
+	case <-ctx.Done():
+		t.Fatal("router not running")
+	}
+
+	outMessagesCh, err := pubSubOut.Subscribe(ctx, outTopic)
+	require.NoError(t, err)
+
+	for _, handlerName := range handlerNames {
+		publisher := forwarder.NewPublisher(pubSubIn, forwarder.PublisherConfig{
+			ForwarderTopic: handlerName + "_topic",
+		})
+
+		msg := message.NewMessage(watermill.NewUUID(), message.Payload(handlerName))
+		require.NoError(t, publisher.Publish(outTopic, msg))
+	}
+
+	// Both forwarders publish to the same out topic, so messages can arrive in any order.
+	receivedPayloads := map[string]bool{}
+	for range handlerNames {
+		select {
+		case receivedMsg := <-outMessagesCh:
+			receivedPayloads[string(receivedMsg.Payload)] = true
+			receivedMsg.Ack()
+		case <-time.After(time.Second * 3):
+			t.Fatal("didn't receive all forwarded messages")
+		}
+	}
+
+	assert.Equal(t, map[string]bool{"first_forwarder": true, "second_forwarder": true}, receivedPayloads)
+}
+
+type PubSubInPublisher struct {
+	message.Publisher
+}
+type PubSubInSubscriber struct {
+	message.Subscriber
+}
+
+type PubSubOutPublisher struct {
+	message.Publisher
+}
+type PubSubOutSubscriber struct {
+	message.Subscriber
+}
+
+func newPubSubIn() (PubSubInPublisher, PubSubInSubscriber) {
+	channelPubSub := gochannel.NewGoChannel(gochannel.Config{}, logger)
+	return PubSubInPublisher{channelPubSub}, PubSubInSubscriber{channelPubSub}
+}
+
+func newPubSubOut() (PubSubOutPublisher, PubSubOutSubscriber) {
+	channelPubSub := gochannel.NewGoChannel(gochannel.Config{}, logger)
+	return PubSubOutPublisher{channelPubSub}, PubSubOutSubscriber{channelPubSub}
+}
+
+func (s *ForwarderSuite) setupForwarder(config forwarder.Config) *forwarder.Forwarder {
+	f, err := forwarder.NewForwarder(s.subscriberIn, s.publisherOut, logger, config)
+	s.Require().NoError(err)
+
+	go func() {
+		s.Require().NoError(f.Run(s.ctx))
+	}()
+
+	select {
+	case <-f.Running():
+	case <-s.ctx.Done():
+		s.T().Fatal("forwarder not running")
+	}
+
+	return f
+}
+
+func (s *ForwarderSuite) listenOnOutTopic() {
+	var err error
+	s.outMessagesCh, err = s.subscriberOut.Subscribe(s.ctx, outTopic)
+	s.Require().NoError(err)
+}
+
+func (s *ForwarderSuite) requireFirstMessage(expectedMessage *message.Message) {
+	select {
+	case receivedMessage := <-s.outMessagesCh:
+		s.Require().NotNil(receivedMessage)
+		s.Require().Truef(receivedMessage.Equals(expectedMessage), "received message: '%s', expected: '%s'", receivedMessage, expectedMessage)
+		receivedMessage.Ack()
+	case <-time.After(time.Second):
+		s.T().Fatal("didn't receive any message after 1 sec")
+	}
+}
+
+func (s *ForwarderSuite) setupMessageAckedDetectorMiddleware() (message.HandlerMiddleware, <-chan bool) {
+	messageAckedCh := make(chan bool, 1)
+	messageAckedDetector := func(handlerFunc message.HandlerFunc) message.HandlerFunc {
+		return func(msg *message.Message) ([]*message.Message, error) {
+			msgs, err := handlerFunc(msg)
+			messageAckedCh <- err == nil
+
+			// Always return nil as we don't want to nack the message in tests.
+			return msgs, nil
+		}
+	}
+
+	return messageAckedDetector, messageAckedCh
+}
+
+func (s *ForwarderSuite) requireFirstAckingResult(msgAckedCh <-chan bool, expected bool) {
+	select {
+	case msgAcked := <-msgAckedCh:
+		s.Require().Equal(expected, msgAcked)
+	case <-time.After(time.Second):
+		s.T().Fatal("acking result not received after 1 sec")
+	}
+}
+
+func (s *ForwarderSuite) sampleMessage() *message.Message {
+	msg := message.NewMessage(watermill.NewUUID(), message.Payload("message payload"))
+	msg.Metadata = message.Metadata{"key": "value"}
+	return msg
+}

components/forwarder/marshaler.go

@@ -0,0 +1,27 @@
+package forwarder
+
+import "encoding/json"
+
+// Marshaler is used by the Forwarder Publisher to serialize the envelope sent
+// to the forwarder topic, and by the Forwarder consumer to deserialize it.
+//
+// The interface matches the signatures of encoding/json's Marshal and Unmarshal,
+// so any drop-in JSON library (e.g. github.com/bytedance/sonic, github.com/goccy/go-json)
+// can be adapted with a small wrapper.
+//
+// The default is DefaultMarshaler, which uses encoding/json.
+type Marshaler interface {
+	Marshal(v any) ([]byte, error)
+	Unmarshal(data []byte, v any) error
+}
+
+// DefaultMarshaler uses the standard library's encoding/json package.
+type DefaultMarshaler struct{}
+
+func (DefaultMarshaler) Marshal(v any) ([]byte, error) {
+	return json.Marshal(v)
+}
+
+func (DefaultMarshaler) Unmarshal(data []byte, v any) error {
+	return json.Unmarshal(data, v)
+}

components/forwarder/publisher.go

@@ -0,0 +1,73 @@
+package forwarder
+
+import (
+	"github.com/ThreeDotsLabs/watermill/message"
+	"github.com/pkg/errors"
+)
+
+// PublisherConfig configures the decorating Publisher returned by NewPublisher.
+type PublisherConfig struct {
+	// ForwarderTopic is a topic which the forwarder is listening to. Publisher will send enveloped messages to this topic.
+	// Defaults to `forwarder_topic`.
+	ForwarderTopic string
+
+	// Marshaler is used to serialize envelopes sent to ForwarderTopic.
+	// It must match the Marshaler used by the Forwarder consuming the topic.
+	// Defaults to DefaultMarshaler, which uses encoding/json.
+	Marshaler Marshaler
+}
+
+func (c *PublisherConfig) setDefaults() {
+	if c.ForwarderTopic == "" {
+		c.ForwarderTopic = defaultForwarderTopic
+	}
+	if c.Marshaler == nil {
+		c.Marshaler = DefaultMarshaler{}
+	}
+}
+
+func (c *PublisherConfig) Validate() error {
+	if c.ForwarderTopic == "" {
+		return errors.New("empty forwarder topic")
+	}
+
+	return nil
+}
+
+// Publisher changes `Publish` method behavior so it wraps a sent message in an envelope
+// and sends it to the forwarder topic provided in the config.
+type Publisher struct {
+	wrappedPublisher message.Publisher
+	config           PublisherConfig
+}
+
+func NewPublisher(publisher message.Publisher, config PublisherConfig) *Publisher {
+	config.setDefaults()
+
+	return &Publisher{
+		wrappedPublisher: publisher,
+		config:           config,
+	}
+}
+
+func (p *Publisher) Publish(topic string, messages ...*message.Message) error {
+	envelopedMessages := make([]*message.Message, 0, len(messages))
+	for _, msg := range messages {
+		envelopedMsg, err := wrapMessageInEnvelope(topic, msg, p.config.Marshaler)
+		if err != nil {
+			return errors.Wrapf(err, "cannot wrap message, target topic: '%s', uuid: '%s'", topic, msg.UUID)
+		}
+
+		envelopedMessages = append(envelopedMessages, envelopedMsg)
+	}
+
+	if err := p.wrappedPublisher.Publish(p.config.ForwarderTopic, envelopedMessages...); err != nil {
+		return errors.Wrapf(err, "cannot publish messages to forwarder topic: '%s'", p.config.ForwarderTopic)
+	}
+
+	return nil
+}
+
+func (p *Publisher) Close() error {
+	return p.wrappedPublisher.Close()
+}