snapshot of https://github.com/ThreeDotsLabs/watermill.git (master, 2026-09-15; blobs > 131072B omitted)

tools/mill/cmd/amqp.go

@@ -0,0 +1,174 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/ThreeDotsLabs/watermill-amqp/v3/pkg/amqp"
+)
+
+var amqpCmd = &cobra.Command{
+	Use:   "amqp",
+	Short: "Commands for the AMQP Pub/Sub provider",
+	Long: `Consume or produce messages from the AMQP Pub/Sub provider.
+
+For the configuration of consuming/producing of the messages, check the help of the relevant command.`,
+	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		err := rootCmd.PersistentPreRunE(cmd, args)
+		if err != nil {
+			return err
+		}
+
+		logger.Debug("Using AMQP Pub/Sub", nil)
+
+		if cmd.Use == "consume" {
+			consumer, err = amqp.NewSubscriber(amqpConsumerConfig(), logger)
+			if err != nil {
+				return err
+			}
+		}
+
+		if cmd.Use == "produce" {
+			producer, err = amqp.NewPublisher(amqpProducerConfig(), logger)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	},
+}
+
+func amqpConsumerConfig() amqp.Config {
+	uri := viper.GetString("amqp.uri")
+	queue := viper.GetString("amqp.consume.queue")
+	exchangeName := viper.GetString("amqp.consume.exchange")
+	exchangeType := viper.GetString("amqp.produce.exchangeType")
+	durable := viper.GetBool("amqp.durable")
+
+	return amqp.Config{
+		Connection: amqp.ConnectionConfig{
+			AmqpURI: uri,
+		},
+		Marshaler: amqp.DefaultMarshaler{},
+		Queue: amqp.QueueConfig{
+			GenerateName: func(topic string) string {
+				return queue
+			},
+			Durable: durable,
+		},
+		Consume: amqp.ConsumeConfig{
+			Qos: amqp.QosConfig{
+				PrefetchCount: 1,
+			},
+		},
+
+		Exchange: amqp.ExchangeConfig{
+			GenerateName: func(topic string) string {
+				return exchangeName
+			},
+			Type:    exchangeType,
+			Durable: durable,
+		},
+	}
+}
+
+func amqpProducerConfig() amqp.Config {
+	uri := viper.GetString("amqp.uri")
+	exchangeName := viper.GetString("amqp.produce.exchange")
+	exchangeType := viper.GetString("amqp.produce.exchangeType")
+	routingKey := viper.GetString("amqp.produce.routingKey")
+	durable := viper.GetBool("amqp.durable")
+
+	return amqp.Config{
+		Connection: amqp.ConnectionConfig{
+			AmqpURI: uri,
+		},
+		Marshaler: amqp.DefaultMarshaler{},
+		Exchange: amqp.ExchangeConfig{
+			GenerateName: func(topic string) string {
+				return exchangeName
+			},
+			Type:    exchangeType,
+			Durable: durable,
+		},
+		Publish: amqp.PublishConfig{
+			GenerateRoutingKey: func(topic string) string {
+				return routingKey
+			},
+		},
+	}
+}
+
+func init() {
+	rootCmd.AddCommand(amqpCmd)
+	configureAmqpCmd()
+	consumeCmd := addConsumeCmd(amqpCmd, "amqp.consume.queue")
+	configureConsumeCmd(consumeCmd)
+	produceCmd := addProduceCmd(amqpCmd, "amqp.produce.exchange")
+	configureProduceCmd(produceCmd)
+}
+
+func configureAmqpCmd() {
+	amqpCmd.PersistentFlags().StringP(
+		"uri",
+		"u",
+		"",
+		"The URI to the AMQP instance (required)",
+	)
+	ensure(amqpCmd.MarkPersistentFlagRequired("uri"))
+	ensure(viper.BindPFlag("amqp.uri", amqpCmd.PersistentFlags().Lookup("uri")))
+
+	amqpCmd.PersistentFlags().Bool(
+		"durable",
+		true,
+		"If true, the queues and exchanges created automatically (if any) will be durable",
+	)
+	ensure(viper.BindPFlag("amqp.durable", amqpCmd.PersistentFlags().Lookup("durable")))
+
+	amqpCmd.PersistentFlags().String(
+		"exchange-type",
+		"fanout",
+		"If exchange needs to be created, it will be created with this type. The common types are 'direct', 'fanout', 'topic' and 'headers'.",
+	)
+	ensure(viper.BindPFlag("amqp.produce.exchangeType", amqpCmd.PersistentFlags().Lookup("exchange-type")))
+}
+
+func configureConsumeCmd(consumeCmd *cobra.Command) {
+	consumeCmd.PersistentFlags().StringP(
+		"queue",
+		"q",
+		"",
+		"The name of the AMQP queue to consume messages from (required)",
+	)
+	ensure(consumeCmd.MarkPersistentFlagRequired("queue"))
+	ensure(viper.BindPFlag("amqp.consume.queue", consumeCmd.PersistentFlags().Lookup("queue")))
+
+	consumeCmd.PersistentFlags().StringP(
+		"exchange",
+		"x",
+		"",
+		"If non-empty, an exchange with this name is created if it didn't exist. Then, the queue is bound to this exchange.",
+	)
+	ensure(viper.BindPFlag("amqp.consume.exchange", consumeCmd.PersistentFlags().Lookup("exchange")))
+}
+
+func configureProduceCmd(produceCmd *cobra.Command) {
+	produceCmd.PersistentFlags().StringP(
+		"exchange",
+		"x",
+		"",
+		"The name of the AMQP exchange to produce messages to (required)",
+	)
+	ensure(produceCmd.MarkPersistentFlagRequired("exchange"))
+	ensure(viper.BindPFlag("amqp.produce.exchange", produceCmd.PersistentFlags().Lookup("exchange")))
+
+	produceCmd.PersistentFlags().StringP(
+		"routing-key",
+		"r",
+		"",
+		"The routing key to use when publishing the message.",
+	)
+	ensure(produceCmd.MarkPersistentFlagRequired("routing-key"))
+	ensure(viper.BindPFlag("amqp.produce.routingKey", produceCmd.PersistentFlags().Lookup("routing-key")))
+}

tools/mill/cmd/consume.go

@@ -0,0 +1,70 @@
+package cmd
+
+import (
+	"context"
+	"os"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/ThreeDotsLabs/watermill-io/pkg/io"
+	"github.com/ThreeDotsLabs/watermill/message"
+	"github.com/ThreeDotsLabs/watermill/message/router/plugin"
+)
+
+// consumer is initialized by the parent command to the pub/sub provider of choice.
+var consumer message.Subscriber
+
+func addConsumeCmd(parent *cobra.Command, topicKey string) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "consume",
+		Short: "Consume messages from a pub/sub and print them to stdout",
+		Long: `Consume messages from the pub/sub of your choice and print them on the standard output.
+
+For the configuration of particular pub/sub providers, see the help for the provider commands.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			topic := viper.GetString(topicKey)
+			router, err := message.NewRouter(
+				message.RouterConfig{
+					CloseTimeout: 5 * time.Second,
+				},
+				logger,
+			)
+			if err != nil {
+				return errors.Wrap(err, "could not create router")
+			}
+
+			router.AddPlugin(plugin.SignalsHandler)
+
+			out, err := io.NewPublisher(
+				os.Stdout,
+				io.PublisherConfig{
+					MarshalFunc: io.PayloadMarshalFunc,
+				},
+				logger,
+			)
+			if err != nil {
+				return errors.Wrap(err, "could not create console producer")
+			}
+
+			router.AddHandler(
+				"dump_to_stdout",
+				topic,
+				consumer,
+				"",
+				out,
+				func(msg *message.Message) ([]*message.Message, error) {
+					// just forward the message to stdout
+					return message.Messages{msg}, nil
+				},
+			)
+
+			return router.Run(context.Background())
+		},
+	}
+
+	parent.AddCommand(cmd)
+	return cmd
+}

tools/mill/cmd/googlecloud.go

@@ -0,0 +1,487 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"cloud.google.com/go/pubsub"
+	"github.com/ThreeDotsLabs/watermill"
+	"github.com/ThreeDotsLabs/watermill-googlecloud/pkg/googlecloud"
+	"github.com/ThreeDotsLabs/watermill/tools/mill/cmd/internal"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"google.golang.org/api/iterator"
+	"gopkg.in/yaml.v2"
+)
+
+var googleCloudTempSubscriptionID string
+
+var googleCloudCmd = &cobra.Command{
+	Use:   "googlecloud",
+	Short: "Commands for the Google Cloud Pub/Sub provider",
+	Long: `Consume or produce messages from the Google Cloud Pub/Sub provider. Manage subscriptions.
+
+For the configuration of consuming/producing of the messages, check the help of the relevant command.`,
+	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		err := rootCmd.PersistentPreRunE(cmd, args)
+		if err != nil {
+			return err
+		}
+
+		logger.Debug("Using Google Cloud Pub/Sub", nil)
+		if cmd.Use == "consume" {
+			subName := viper.GetString("googlecloud.consume.subscription")
+			if subName == "" {
+				subName, err = generateTempSubscription()
+				if err != nil {
+					return err
+				}
+			}
+
+			consumer, err = googlecloud.NewSubscriber(
+				googlecloud.SubscriberConfig{
+					GenerateSubscriptionName: func(topic string) string {
+						return subName
+					},
+					ProjectID: projectID(),
+				},
+				logger,
+			)
+			if err != nil {
+				return err
+			}
+		}
+
+		if cmd.Use == "produce" {
+			producer, err = googlecloud.NewPublisher(
+				googlecloud.PublisherConfig{
+					ProjectID: projectID(),
+				},
+				logger,
+			)
+
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	},
+	PersistentPostRunE: func(cmd *cobra.Command, args []string) error {
+		logger.Debug("Google Cloud Pub/Sub cleanup", nil)
+		if googleCloudTempSubscriptionID != "" {
+			if err := removeTempSubscription(); err != nil {
+				return err
+			}
+		}
+		return nil
+	},
+}
+
+var googleCloudSubscriptionCmd = &cobra.Command{
+	Use:   "subscription",
+	Short: "Manage Google Cloud Pub/Sub subscriptions",
+	Long:  `Add or remove subscriptions for the Google Cloud Pub/Sub provider.`,
+}
+
+var googleCloudSubscriptionAddCmd = &cobra.Command{
+	Use:       "add <subscription_id>",
+	Short:     "Add a new subscription in Google Cloud Pub/Sub",
+	Args:      cobra.ExactArgs(1),
+	ValidArgs: []string{"subscriptionID"},
+	RunE: func(cmd *cobra.Command, args []string) (err error) {
+		subID := args[0]
+
+		topic := viper.GetString("googlecloud.subscription.add.topic")
+		ackDeadline := viper.GetDuration("googlecloud.subscription.add.ackDeadline")
+		retainAcked := viper.GetBool("googlecloud.subscription.add.retainAcked")
+		retentionDuration := viper.GetDuration("googlecloud.subscription.add.retentionDuration")
+
+		// StringToString doesn't work with viper, so let's parse this manually
+		labels := strings.Split(viper.GetString("googlecloud.subscription.add.labels"), ",")
+		labelsMap := make(map[string]string, len(labels))
+		for _, l := range labels {
+			fields := strings.Split(l, "=")
+			if len(fields) < 2 {
+				continue
+			}
+			labelsMap[fields[0]] = fields[1]
+		}
+
+		logger := logger.With(watermill.LogFields{
+			"subscription_id":   subID,
+			"topic":             topic,
+			"ackDeadline":       ackDeadline,
+			"retainAcked":       retainAcked,
+			"retentionDuration": retentionDuration,
+			"labels":            labelsMap,
+		})
+		logger.Info("Creating new subscription", nil)
+
+		defer func() {
+			if err == nil {
+				logger.Info("Subscription created", nil)
+			}
+		}()
+
+		return addSubscription(
+			subID,
+			topic,
+			ackDeadline,
+			retainAcked,
+			retentionDuration,
+			labelsMap,
+		)
+	},
+}
+
+var googleCloudSubscriptionRmCmd = &cobra.Command{
+	Use:       "rm <subscription_id>",
+	Short:     "Remove a subscription in Google Cloud Pub/Sub",
+	Args:      cobra.ExactArgs(1),
+	ValidArgs: []string{"subscriptionID"},
+	RunE: func(cmd *cobra.Command, args []string) (err error) {
+		subID := args[0]
+
+		logger := logger.With(watermill.LogFields{
+			"subscription_id": subID,
+		})
+		logger.Info("Removing a subscription", nil)
+
+		defer func() {
+			if err == nil {
+				logger.Info("Subscription removed", nil)
+			}
+		}()
+
+		return removeSubscription(subID)
+	},
+}
+
+var googleCloudSubscriptionLsCmd = &cobra.Command{
+	Use:   "ls",
+	Short: "List subscriptions in Google Cloud Pub/Sub. Topic may be provided optionally to filter subscriptions by topic.",
+	Args:  cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, args []string) (err error) {
+		topic := viper.GetString("googlecloud.subscription.ls.topic")
+		verbose := viper.GetBool("googlecloud.subscription.ls.verbose")
+
+		logger := logger
+		if topic != "" {
+			logger = logger.With(watermill.LogFields{
+				"topic": topic,
+			})
+		}
+		logger.Info("Listing all subscriptions", nil)
+
+		return listSubscriptions(topic, logger, verbose)
+	},
+}
+
+func generateTempSubscription() (id string, err error) {
+	defer func() {
+		if err == nil {
+			logger.Debug("Temp subscription created", watermill.LogFields{
+				"subscription_name": id,
+			})
+			googleCloudTempSubscriptionID = id
+		}
+	}()
+
+	randomID := "watermill_console_consumer_" + watermill.NewShortUUID()
+	return randomID, addSubscription(
+		randomID,
+		viper.GetString("googlecloud.topic"),
+		10*time.Second,
+		false,
+		10*time.Minute,
+		nil,
+	)
+}
+
+func addSubscription(
+	id string,
+	topic string,
+	ackDeadline time.Duration,
+	retainAckedMessages bool,
+	retentionDuration time.Duration,
+	labels map[string]string,
+) error {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	defer cancel()
+
+	client, err := pubsub.NewClient(ctx, projectID())
+	if err != nil {
+		return errors.Wrap(err, "could not create pubsub client")
+	}
+
+	t := client.Topic(topic)
+	exists, err := t.Exists(ctx)
+	if err != nil {
+		return errors.Wrap(err, "could not check if topic exists")
+	}
+	if !exists {
+		t, err = client.CreateTopic(ctx, t.ID())
+		if err != nil {
+			return errors.Wrap(err, "could not create topic")
+		}
+	}
+
+	_, err = client.CreateSubscription(ctx, id, pubsub.SubscriptionConfig{
+		Topic:               t,
+		AckDeadline:         ackDeadline,
+		RetainAckedMessages: retainAckedMessages,
+		RetentionDuration:   retentionDuration,
+		Labels:              labels,
+	})
+	if err != nil {
+		return errors.Wrap(err, "could not create subscription")
+	}
+
+	return nil
+}
+
+func removeTempSubscription() (err error) {
+	defer func() {
+		if err == nil {
+			logger.Debug("Temporary subscription removed", watermill.LogFields{
+				"subscription_name": googleCloudTempSubscriptionID,
+			})
+		}
+	}()
+	return removeSubscription(googleCloudTempSubscriptionID)
+}
+
+func removeSubscription(id string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	defer cancel()
+
+	client, err := pubsub.NewClient(ctx, projectID())
+	if err != nil {
+		return errors.Wrap(err, "could not create pubsub client")
+	}
+
+	sub := client.Subscription(id)
+	exists, err := sub.Exists(ctx)
+	if err != nil {
+		return errors.Wrap(err, "could not check if sub exists")
+	}
+
+	if !exists {
+		return nil
+	}
+
+	return sub.Delete(ctx)
+}
+
+func listSubscriptions(topic string, adapter watermill.LoggerAdapter, verbose bool) error {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	defer cancel()
+
+	client, err := pubsub.NewClient(ctx, projectID())
+	if err != nil {
+		return errors.Wrap(err, "could not create pubsub client")
+	}
+
+	if topic != "" {
+		topic := client.Topic(topic)
+		return listSubscriptionsForTopic(ctx, client, topic, logger, verbose)
+	}
+
+	it := client.Topics(ctx)
+	noTopics := true
+	for {
+		topic, err := it.Next()
+		if err == iterator.Done {
+			if noTopics {
+				logger.Info("No topics in project", nil)
+			}
+			return nil
+		}
+		if err != nil {
+			return errors.Wrap(err, "could not retrieve next subscription")
+		}
+
+		noTopics = false
+		err = listSubscriptionsForTopic(ctx, client, topic, logger, verbose)
+		if err != nil {
+			return errors.Wrap(err, "error listing subscriptions for topic")
+		}
+	}
+
+	return nil
+}
+
+func listSubscriptionsForTopic(ctx context.Context, client *pubsub.Client, topic *pubsub.Topic, logger watermill.LoggerAdapter, verbose bool) error {
+	noSubs := true
+	exists, err := topic.Exists(ctx)
+	if err != nil {
+		return errors.Wrap(err, "could not check if topic exists")
+	}
+	if !exists {
+		logger.Info("Topic does not exist", watermill.LogFields{"topic": topic.String()})
+		return nil
+	}
+
+	it := topic.Subscriptions(ctx)
+	for {
+		sub, err := it.Next()
+		if err == iterator.Done {
+			if noSubs {
+				logger.Info("No subscriptions for the topic", watermill.LogFields{"topic": topic.String()})
+			}
+			return nil
+		}
+		if err != nil {
+			return errors.Wrap(err, "could not retrieve next subscription")
+		}
+
+		if noSubs {
+			noSubs = false
+			fmt.Printf("Topic %s:\n", topic.String())
+		}
+		name := sub.String()
+		config, err := sub.Config(ctx)
+		if err != nil {
+			return errors.Wrapf(err, "could not retrieve subscription config for subscription '%s'", name)
+		}
+
+		err = printSubscriptionInfo(name, config)
+		if err != nil {
+			return errors.Wrapf(err, "error printing subscription '%s'", name)
+		}
+	}
+}
+
+func printSubscriptionInfo(name string, config pubsub.SubscriptionConfig) error {
+	b, err := yaml.Marshal(subscriptionConfig{
+		Name: name,
+		PushConfig: subscriptionConfigPushConfig{
+			Endpoint:   config.PushConfig.Endpoint,
+			Attributes: config.PushConfig.Attributes,
+		},
+		AckDeadline:         config.AckDeadline,
+		RetainAckedMessages: config.RetainAckedMessages,
+		RetentionDuration:   config.RetentionDuration,
+		Labels:              config.Labels,
+	})
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf(internal.Indent(string(b), "  "))
+	return nil
+}
+
+// subscriptionConfig provides a marshallable form to pubsub.SubscriptionConfig
+type subscriptionConfig struct {
+	Name                string
+	PushConfig          subscriptionConfigPushConfig `yaml:"push_config"`
+	AckDeadline         time.Duration                `yaml:"ack_deadline"`
+	RetainAckedMessages bool                         `yaml:"retain_acked_messages"`
+	RetentionDuration   time.Duration                `yaml:"retention_duration"`
+	Labels              map[string]string            `yaml:"labels"`
+}
+
+type subscriptionConfigPushConfig struct {
+	Endpoint   string
+	Attributes map[string]string
+}
+
+func projectID() string {
+	projectID := viper.GetString("googlecloud.projectID")
+	if projectID == "" {
+		projectID = os.Getenv("GOOGLE_CLOUD_PROJECT")
+	}
+
+	return projectID
+}
+
+func init() {
+	rootCmd.AddCommand(googleCloudCmd)
+
+	googleCloudCmd.PersistentFlags().StringP(
+		"topic",
+		"t",
+		"",
+		"The topic to produce messages to (produce), consume message from (consume), list from (ls) or the topic for the newly created subscription (subscription.add)",
+	)
+	ensure(viper.BindPFlag("googlecloud.topic", googleCloudCmd.PersistentFlags().Lookup("topic")))
+	ensure(googleCloudCmd.MarkPersistentFlagRequired("topic"))
+
+	consumeCmd := addConsumeCmd(googleCloudCmd, "googlecloud.topic")
+	addProduceCmd(googleCloudCmd, "googlecloud.topic")
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	googleCloudCmd.PersistentFlags().String("project", "", "The projectID for Google Cloud Pub/Sub. Defaults to the GOOGLE_CLOUD_PROJECT environment variable.")
+	ensure(viper.BindPFlag("googlecloud.projectID", googleCloudCmd.PersistentFlags().Lookup("project")))
+
+	consumeCmd.PersistentFlags().StringP(
+		"subscription",
+		"s",
+		"",
+		"The subscription for Google Cloud Pub/Sub. If left empty, a temporary subscription is created and removed when the consumer is closed",
+	)
+	ensure(viper.BindPFlag("googlecloud.consume.subscription", consumeCmd.PersistentFlags().Lookup("subscription")))
+
+	googleCloudCmd.AddCommand(googleCloudSubscriptionCmd)
+	googleCloudSubscriptionCmd.AddCommand(googleCloudSubscriptionAddCmd)
+
+	googleCloudSubscriptionAddCmd.Flags().StringP("topic", "t", "", "The topic for the new subscription (required)")
+	ensure(googleCloudSubscriptionAddCmd.MarkFlagRequired("topic"))
+	ensure(viper.BindPFlag("googlecloud.subscription.add.topic", googleCloudSubscriptionAddCmd.Flags().Lookup("topic")))
+
+	googleCloudSubscriptionAddCmd.Flags().DurationP(
+		"ack-deadline",
+		"a",
+		10*time.Second,
+		"How long Pub/Sub waits for the subscriber to acknowledge receipt before resending the message. Deadline time is from 10 seconds to 600 seconds",
+	)
+	ensure(viper.BindPFlag("googlecloud.subscription.add.ackDeadline", googleCloudSubscriptionAddCmd.Flags().Lookup("ack-deadline")))
+
+	googleCloudSubscriptionAddCmd.Flags().Bool(
+		"retain-acked",
+		false,
+		"Acknowledged messages will be kept 7 days from publication unless set otherwise in \"message retention duration\".",
+	)
+	ensure(viper.BindPFlag("googlecloud.subscription.add.retainAcked", googleCloudSubscriptionAddCmd.Flags().Lookup("retain-acked")))
+
+	googleCloudSubscriptionAddCmd.Flags().Duration(
+		"retention-duration",
+		7*24*time.Hour,
+		"How long the retained messages will be kept. The allowed duration is from 10 minutes to 7 days, which is the default.",
+	)
+	ensure(viper.BindPFlag("googlecloud.subscription.add.retentionDuration", googleCloudSubscriptionAddCmd.Flags().Lookup("retention-duration")))
+
+	// StringToString doesn't work correctly with viper
+	googleCloudSubscriptionAddCmd.Flags().String(
+		"labels",
+		"",
+		"The set of labels for the subscription. Format: '--labels key1=value1,key2=value2,...'",
+	)
+	ensure(viper.BindPFlag("googlecloud.subscription.add.labels", googleCloudSubscriptionAddCmd.Flags().Lookup("labels")))
+
+	googleCloudSubscriptionCmd.AddCommand(googleCloudSubscriptionRmCmd)
+
+	googleCloudSubscriptionCmd.AddCommand(googleCloudSubscriptionLsCmd)
+	googleCloudSubscriptionLsCmd.Flags().StringP(
+		"topic",
+		"t",
+		"",
+		"The topic for the new subscription (optional, will list subscriptions for all topics if omitted)",
+	)
+	ensure(viper.BindPFlag("googlecloud.subscription.ls.topic", googleCloudSubscriptionLsCmd.Flags().Lookup("topic")))
+
+	googleCloudSubscriptionLsCmd.Flags().BoolP(
+		"verbose",
+		"v",
+		false,
+		"will print more information, including the subscription config",
+	)
+	ensure(viper.BindPFlag("googlecloud.subscription.ls.verbose", googleCloudSubscriptionLsCmd.Flags().Lookup("verbose")))
+}

tools/mill/cmd/internal/indent.go

@@ -0,0 +1,19 @@
+package internal
+
+import "strings"
+
+// Indent indents all lines in the given string with a given prefix.
+func Indent(s, prefix string) string {
+	endsWithNewline := strings.HasSuffix(s, "\n")
+	split := strings.Split(s, "\n")
+
+	for i, ss := range split {
+		split[i] = prefix + ss
+	}
+	joined := strings.Join(split, "\n")
+	if endsWithNewline {
+		joined += "\n"
+	}
+
+	return joined
+}

tools/mill/cmd/kafka.go

@@ -0,0 +1,92 @@
+package cmd
+
+import (
+	"github.com/IBM/sarama"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/ThreeDotsLabs/watermill-kafka/v3/pkg/kafka"
+)
+
+var kafkaCmd = &cobra.Command{
+	Use:   "kafka",
+	Short: "Commands for the kafka Pub/Sub provider",
+	Long: `Consume or produce messages from the kafka Pub/Sub provider.
+
+For the configuration of consuming/producing of the messages, check the help of the relevant command.`,
+	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		err := rootCmd.PersistentPreRunE(cmd, args)
+		if err != nil {
+			return err
+		}
+		logger.Debug("Using kafka pub/sub", nil)
+
+		brokers := viper.GetStringSlice("kafka.brokers")
+
+		if cmd.Use == "consume" {
+			saramaSubscriberConfig := kafka.DefaultSaramaSubscriberConfig()
+
+			if viper.GetBool("kafka.consume.fromBeginning") {
+				logger.Trace("Configured sarama to consume messages from beginning", nil)
+				// equivalent of auto.offset.reset: earliest
+				saramaSubscriberConfig.Consumer.Offsets.Initial = sarama.OffsetOldest
+			}
+
+			consumer, err = kafka.NewSubscriber(
+				kafka.SubscriberConfig{
+					Brokers:               brokers,
+					Unmarshaler:           kafka.DefaultMarshaler{},
+					OverwriteSaramaConfig: saramaSubscriberConfig,
+					ConsumerGroup:         viper.GetString("kafka.consume.consumerGroup"),
+				},
+				logger,
+			)
+			if err != nil {
+				return err
+			}
+		}
+
+		if cmd.Use == "produce" {
+			producer, err = kafka.NewPublisher(
+				kafka.PublisherConfig{
+					Brokers:   brokers,
+					Marshaler: kafka.DefaultMarshaler{},
+				},
+				logger,
+			)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(kafkaCmd)
+
+	kafkaCmd.PersistentFlags().StringP(
+		"topic",
+		"t",
+		"",
+		"The topic to produce messages to (produce) or consume message from (consume)",
+	)
+	ensure(kafkaCmd.MarkPersistentFlagRequired("topic"))
+	ensure(viper.BindPFlag("kafka.topic", kafkaCmd.PersistentFlags().Lookup("topic")))
+
+	consumeCmd := addConsumeCmd(kafkaCmd, "kafka.topic")
+	_ = addProduceCmd(kafkaCmd, "kafka.topic")
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	kafkaCmd.PersistentFlags().StringSliceP("brokers", "b", nil, "A list of kafka brokers")
+	ensure(kafkaCmd.MarkPersistentFlagRequired("brokers"))
+	ensure(viper.BindPFlag("kafka.brokers", kafkaCmd.PersistentFlags().Lookup("brokers")))
+
+	consumeCmd.PersistentFlags().Bool("from-beginning", false, "Equivalent to auto.offset.reset: earliest")
+	ensure(viper.BindPFlag("kafka.consume.fromBeginning", consumeCmd.PersistentFlags().Lookup("from-beginning")))
+
+	consumeCmd.PersistentFlags().StringP("consumer-group", "c", "", "The kafka consumer group. Defaults to empty.")
+	ensure(viper.BindPFlag("kafka.consume.consumerGroup", consumeCmd.PersistentFlags().Lookup("consumer-group")))
+}

tools/mill/cmd/produce.go

@@ -0,0 +1,78 @@
+package cmd
+
+import (
+	"context"
+	"os"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/ThreeDotsLabs/watermill-io/pkg/io"
+	"github.com/ThreeDotsLabs/watermill/message"
+	"github.com/ThreeDotsLabs/watermill/message/router/plugin"
+)
+
+// producer is initialized by parent command to the pub/sub provider of choice.
+var producer message.Publisher
+
+func addProduceCmd(parent *cobra.Command, topicKey string) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "produce",
+		Short: "Produce messages to a pub/sub from the stdin",
+		Long: `Produce messages to the pub/sub of your choice from the standard input.
+
+For the configuration of particular pub/sub providers, see the help for the provider commands.`,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			topic := viper.GetString(topicKey)
+			router, err := message.NewRouter(
+				message.RouterConfig{
+					CloseTimeout: 10 * time.Second,
+				},
+				logger,
+			)
+			if err != nil {
+				return errors.Wrap(err, "could not create router")
+			}
+
+			router.AddPlugin(plugin.SignalsHandler)
+
+			in, err := io.NewSubscriber(
+				os.Stdin,
+				io.SubscriberConfig{
+					PollInterval:  time.Second,
+					UnmarshalFunc: io.PayloadUnmarshalFunc,
+				},
+				logger,
+			)
+			if err != nil {
+				return errors.Wrap(err, "could not create console subscriber")
+			}
+
+			router.AddHandler(
+				"produce_from_stdin",
+				"stdin",
+				in,
+				topic,
+				producer,
+				func(msg *message.Message) ([]*message.Message, error) {
+					if string(msg.Payload) == "\n" {
+						logger.Trace("Message is empty, don't publish", nil)
+						return nil, nil
+					}
+					// just pass the message along
+					return message.Messages{msg}, nil
+				},
+			)
+
+			return router.Run(context.Background())
+		},
+	}
+
+	parent.AddCommand(cmd)
+	return cmd
+}

tools/mill/cmd/root.go

@@ -0,0 +1,149 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+	yaml "gopkg.in/yaml.v2"
+
+	"github.com/ThreeDotsLabs/watermill"
+)
+
+var cfgFile string
+var logger watermill.LoggerAdapter
+
+var rootCmd = &cobra.Command{
+	Use:   "mill",
+	Short: "A CLI for Watermill.",
+	Long: `A CLI for Watermill.
+
+Use console-based producer or consumer for various pub/sub providers.`,
+	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		log := viper.GetBool("log")
+		debug := viper.GetBool("debug")
+		trace := viper.GetBool("trace")
+		if log || debug || trace {
+			logger = watermill.NewStdLogger(debug, trace)
+		} else {
+			logger = watermill.NopLogger{}
+		}
+
+		if err := checkRequiredFlags(cmd.Flags()); err != nil {
+			return err
+		}
+
+		writeConfig := viper.GetString("writeConfig")
+		if writeConfig != "" {
+			settings := viper.AllSettings()
+			delete(settings, "writeconfig")
+			b, err := yaml.Marshal(settings)
+			if err != nil {
+				return errors.Wrap(err, "could not marshal config to yaml")
+			}
+
+			f, err := os.Create(writeConfig)
+			if err != nil {
+				return errors.Wrap(err, "could not create file for write")
+			}
+			_, err = fmt.Fprintf(f, "%s", b)
+			if err != nil {
+				return errors.Wrap(err, "could not write to file")
+			}
+		}
+
+		return nil
+	},
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func init() {
+	cobra.OnInitialize(initConfig)
+	rootCmd.PersistentFlags().SortFlags = false
+
+	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.mill.yaml)")
+
+	outputFlags := pflag.NewFlagSet("output", pflag.ExitOnError)
+	outputFlags.BoolP("log", "l", false, "If true, the logger output is sent to stderr. No logger output otherwise.")
+	ensure(viper.BindPFlag("log", outputFlags.Lookup("log")))
+
+	outputFlags.BoolP("debug", "d", false, "If true, debug output is enabled from the logger")
+	ensure(viper.BindPFlag("debug", outputFlags.Lookup("debug")))
+
+	outputFlags.Bool("trace", false, "If true, trace output is enabled from the logger")
+	ensure(viper.BindPFlag("trace", outputFlags.Lookup("trace")))
+
+	outputFlags.String("write-config", "", "Write the config of the current command as yaml to the specified path")
+	ensure(viper.BindPFlag("writeConfig", outputFlags.Lookup("write-config")))
+
+	rootCmd.PersistentFlags().AddFlagSet(outputFlags)
+}
+
+// initConfig reads in config file and ENV variables if set.
+func initConfig() {
+	if cfgFile != "" {
+		// Use config file from the flag.
+		viper.SetConfigFile(cfgFile)
+	} else {
+		// Find home directory.
+		home, err := homedir.Dir()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		viper.AddConfigPath(home)
+		viper.SetConfigName(".mill")
+	}
+
+	// read in environment variables that match
+	viper.AutomaticEnv()
+
+	// If a config file is found, read it in.
+	if err := viper.ReadInConfig(); err == nil {
+		fmt.Println("Using config file:", viper.ConfigFileUsed())
+	}
+}
+
+func ensure(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func checkRequiredFlags(flags *pflag.FlagSet) error {
+	requiredError := false
+	flagName := ""
+
+	flags.VisitAll(func(flag *pflag.Flag) {
+		requiredAnnotation := flag.Annotations[cobra.BashCompOneRequiredFlag]
+		if len(requiredAnnotation) == 0 {
+			return
+		}
+
+		flagRequired := requiredAnnotation[0] == "true"
+
+		if flagRequired && !flag.Changed {
+			requiredError = true
+			flagName = flag.Name
+		}
+	})
+
+	if requiredError {
+		return errors.New("Required flag `" + flagName + "` has not been set")
+	}
+
+	return nil
+}