snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/notification/kafka/kafka_queue.go

@@ -0,0 +1,98 @@
+package kafka
+
+import (
+	"fmt"
+
+	"github.com/Shopify/sarama"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/notification"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/proto"
+)
+
+func init() {
+	notification.MessageQueues = append(notification.MessageQueues, &KafkaQueue{})
+}
+
+type KafkaQueue struct {
+	topic    string
+	producer sarama.AsyncProducer
+}
+
+func (k *KafkaQueue) GetName() string {
+	return "kafka"
+}
+
+func (k *KafkaQueue) Initialize(configuration util.Configuration, prefix string) (err error) {
+	glog.V(0).Infof("filer.notification.kafka.hosts: %v\n", configuration.GetStringSlice(prefix+"hosts"))
+	glog.V(0).Infof("filer.notification.kafka.topic: %v\n", configuration.GetString(prefix+"topic"))
+	return k.initialize(
+		configuration.GetStringSlice(prefix+"hosts"),
+		configuration.GetString(prefix+"topic"),
+		SASLTLSConfig{
+			SASLEnabled:           configuration.GetBool(prefix + "sasl_enabled"),
+			SASLMechanism:         configuration.GetString(prefix + "sasl_mechanism"),
+			SASLUsername:          configuration.GetString(prefix + "sasl_username"),
+			SASLPassword:          configuration.GetString(prefix + "sasl_password"),
+			TLSEnabled:            configuration.GetBool(prefix + "tls_enabled"),
+			TLSCACert:             configuration.GetString(prefix + "tls_ca_cert"),
+			TLSClientCert:         configuration.GetString(prefix + "tls_client_cert"),
+			TLSClientKey:          configuration.GetString(prefix + "tls_client_key"),
+			TLSInsecureSkipVerify: configuration.GetBool(prefix + "tls_insecure_skip_verify"),
+		},
+	)
+}
+
+func (k *KafkaQueue) initialize(hosts []string, topic string, saslTLS SASLTLSConfig) (err error) {
+	config := sarama.NewConfig()
+	config.Producer.RequiredAcks = sarama.WaitForLocal
+	config.Producer.Partitioner = sarama.NewHashPartitioner
+	config.Producer.Return.Successes = true
+	config.Producer.Return.Errors = true
+	if err = ConfigureSASLTLS(config, saslTLS); err != nil {
+		return fmt.Errorf("kafka producer security configuration: %w", err)
+	}
+	k.producer, err = sarama.NewAsyncProducer(hosts, config)
+	if err != nil {
+		return err
+	}
+	k.topic = topic
+	go k.handleSuccess()
+	go k.handleError()
+	return nil
+}
+
+func (k *KafkaQueue) SendMessage(key string, message proto.Message) (err error) {
+	bytes, err := proto.Marshal(message)
+	if err != nil {
+		return
+	}
+
+	msg := &sarama.ProducerMessage{
+		Topic: k.topic,
+		Key:   sarama.StringEncoder(key),
+		Value: sarama.ByteEncoder(bytes),
+	}
+
+	k.producer.Input() <- msg
+
+	return nil
+}
+
+func (k *KafkaQueue) handleSuccess() {
+	for {
+		pm := <-k.producer.Successes()
+		if pm != nil {
+			glog.V(3).Infof("producer message success, partition:%d offset:%d key:%v", pm.Partition, pm.Offset, pm.Key)
+		}
+	}
+}
+
+func (k *KafkaQueue) handleError() {
+	for {
+		err := <-k.producer.Errors()
+		if err != nil {
+			glog.Errorf("producer message error, partition:%d offset:%d key:%v value:%s error(%v) topic:%s", err.Msg.Partition, err.Msg.Offset, err.Msg.Key, err.Msg.Value, err.Err, k.topic)
+		}
+	}
+}

weed/notification/kafka/kafka_sasl_tls.go

@@ -0,0 +1,112 @@
+package kafka
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/Shopify/sarama"
+	"github.com/xdg-go/scram"
+)
+
+// SASLTLSConfig holds SASL and TLS configuration for Kafka connections.
+type SASLTLSConfig struct {
+	SASLEnabled   bool
+	SASLMechanism string
+	SASLUsername  string
+	SASLPassword  string
+
+	TLSEnabled            bool
+	TLSCACert             string
+	TLSClientCert         string
+	TLSClientKey          string
+	TLSInsecureSkipVerify bool
+}
+
+// ConfigureSASLTLS applies SASL and TLS settings to a sarama config.
+func ConfigureSASLTLS(config *sarama.Config, st SASLTLSConfig) error {
+	if st.SASLEnabled {
+		config.Net.SASL.Enable = true
+		config.Net.SASL.User = st.SASLUsername
+		config.Net.SASL.Password = st.SASLPassword
+
+		mechanism := strings.ToUpper(st.SASLMechanism)
+		switch mechanism {
+		case "PLAIN", "":
+			config.Net.SASL.Mechanism = sarama.SASLTypePlaintext
+		case "SCRAM-SHA-256":
+			config.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA256
+			config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
+				return &scramClient{HashGeneratorFcn: scram.SHA256}
+			}
+		case "SCRAM-SHA-512":
+			config.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA512
+			config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
+				return &scramClient{HashGeneratorFcn: scram.SHA512}
+			}
+		default:
+			return fmt.Errorf("unsupported SASL mechanism: %s", mechanism)
+		}
+	}
+
+	if st.TLSEnabled {
+		if (st.TLSClientCert == "") != (st.TLSClientKey == "") {
+			return fmt.Errorf("both tls_client_cert and tls_client_key must be provided for mTLS, or neither")
+		}
+
+		tlsConfig := &tls.Config{
+			MinVersion:         tls.VersionTLS12,
+			InsecureSkipVerify: st.TLSInsecureSkipVerify,
+		}
+
+		if st.TLSCACert != "" {
+			caCert, err := os.ReadFile(st.TLSCACert)
+			if err != nil {
+				return fmt.Errorf("failed to read CA certificate: %w", err)
+			}
+			caCertPool := x509.NewCertPool()
+			if !caCertPool.AppendCertsFromPEM(caCert) {
+				return fmt.Errorf("failed to parse CA certificate")
+			}
+			tlsConfig.RootCAs = caCertPool
+		}
+
+		if st.TLSClientCert != "" && st.TLSClientKey != "" {
+			cert, err := tls.LoadX509KeyPair(st.TLSClientCert, st.TLSClientKey)
+			if err != nil {
+				return fmt.Errorf("failed to load client certificate/key: %w", err)
+			}
+			tlsConfig.Certificates = []tls.Certificate{cert}
+		}
+
+		config.Net.TLS.Enable = true
+		config.Net.TLS.Config = tlsConfig
+	}
+
+	return nil
+}
+
+// scramClient implements the sarama.SCRAMClient interface.
+type scramClient struct {
+	*scram.ClientConversation
+	scram.HashGeneratorFcn
+}
+
+func (c *scramClient) Begin(userName, password, authzID string) (err error) {
+	client, err := c.HashGeneratorFcn.NewClient(userName, password, authzID)
+	if err != nil {
+		return err
+	}
+	c.ClientConversation = client.NewConversation()
+	return nil
+}
+
+func (c *scramClient) Step(challenge string) (string, error) {
+	return c.ClientConversation.Step(challenge)
+}
+
+func (c *scramClient) Done() bool {
+	return c.ClientConversation.Done()
+}

weed/notification/kafka/kafka_sasl_tls_test.go

@@ -0,0 +1,65 @@
+package kafka
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/Shopify/sarama"
+)
+
+func TestConfigureSASLTLSRejectsPartialMTLSConfig(t *testing.T) {
+	tests := []struct {
+		name string
+		cfg  SASLTLSConfig
+	}{
+		{
+			name: "missing key",
+			cfg: SASLTLSConfig{
+				TLSEnabled:    true,
+				TLSClientCert: "/tmp/client.crt",
+			},
+		},
+		{
+			name: "missing cert",
+			cfg: SASLTLSConfig{
+				TLSEnabled:   true,
+				TLSClientKey: "/tmp/client.key",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := ConfigureSASLTLS(sarama.NewConfig(), tt.cfg)
+			if err == nil {
+				t.Fatal("expected error")
+			}
+			if !strings.Contains(err.Error(), "both tls_client_cert and tls_client_key must be provided") {
+				t.Fatalf("unexpected error: %v", err)
+			}
+		})
+	}
+}
+
+func TestConfigureSASLTLSConfiguresSCRAMSHA256(t *testing.T) {
+	config := sarama.NewConfig()
+	err := ConfigureSASLTLS(config, SASLTLSConfig{
+		SASLEnabled:   true,
+		SASLMechanism: "SCRAM-SHA-256",
+		SASLUsername:  "alice",
+		SASLPassword:  "secret",
+	})
+	if err != nil {
+		t.Fatalf("ConfigureSASLTLS returned error: %v", err)
+	}
+
+	if !config.Net.SASL.Enable {
+		t.Fatal("expected SASL to be enabled")
+	}
+	if config.Net.SASL.Mechanism != sarama.SASLTypeSCRAMSHA256 {
+		t.Fatalf("unexpected mechanism: %v", config.Net.SASL.Mechanism)
+	}
+	if config.Net.SASL.SCRAMClientGeneratorFunc == nil {
+		t.Fatal("expected SCRAM client generator")
+	}
+}