snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

View File

@@ -0,0 +1,95 @@
package aws_sqs
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/sqs"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/notification"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/proto"
)
func init() {
notification.MessageQueues = append(notification.MessageQueues, &AwsSqsPub{})
}
type AwsSqsPub struct {
svc *sqs.SQS
queueUrl string
}
func (k *AwsSqsPub) GetName() string {
return "aws_sqs"
}
func (k *AwsSqsPub) Initialize(configuration util.Configuration, prefix string) (err error) {
glog.V(0).Infof("filer.notification.aws_sqs.region: %v", configuration.GetString(prefix+"region"))
glog.V(0).Infof("filer.notification.aws_sqs.sqs_queue_name: %v", configuration.GetString(prefix+"sqs_queue_name"))
return k.initialize(
configuration.GetString(prefix+"aws_access_key_id"),
configuration.GetString(prefix+"aws_secret_access_key"),
configuration.GetString(prefix+"region"),
configuration.GetString(prefix+"sqs_queue_name"),
)
}
func (k *AwsSqsPub) initialize(awsAccessKeyId, awsSecretAccessKey, region, queueName string) (err error) {
config := &aws.Config{
Region: aws.String(region),
}
if awsAccessKeyId != "" && awsSecretAccessKey != "" {
config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, awsSecretAccessKey, "")
}
sess, err := session.NewSession(config)
if err != nil {
return fmt.Errorf("create aws session: %w", err)
}
k.svc = sqs.New(sess)
result, err := k.svc.GetQueueUrl(&sqs.GetQueueUrlInput{
QueueName: aws.String(queueName),
})
if err != nil {
if aerr, ok := err.(awserr.Error); ok && aerr.Code() == sqs.ErrCodeQueueDoesNotExist {
return fmt.Errorf("unable to find queue %s", queueName)
}
return fmt.Errorf("get queue %s url: %v", queueName, err)
}
k.queueUrl = *result.QueueUrl
return nil
}
func (k *AwsSqsPub) SendMessage(key string, message proto.Message) (err error) {
text, err := proto.Marshal(message)
if err != nil {
return fmt.Errorf("send message marshal %+v: %v", message, err)
}
_, err = k.svc.SendMessage(&sqs.SendMessageInput{
DelaySeconds: aws.Int64(10),
MessageAttributes: map[string]*sqs.MessageAttributeValue{
"key": &sqs.MessageAttributeValue{
DataType: aws.String("String"),
StringValue: aws.String(key),
},
},
MessageBody: aws.String(string(text)),
QueueUrl: &k.queueUrl,
})
if err != nil {
return fmt.Errorf("send message to sqs %s: %v", k.queueUrl, err)
}
return nil
}

View File

@@ -0,0 +1,56 @@
package notification
import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/proto"
)
type MessageQueue interface {
// GetName gets the name to locate the configuration in filer.toml file
GetName() string
// Initialize initializes the file store
Initialize(configuration util.Configuration, prefix string) error
SendMessage(key string, message proto.Message) error
}
var (
MessageQueues []MessageQueue
Queue MessageQueue
)
func LoadConfiguration(config *util.ViperProxy, prefix string) {
if config == nil {
return
}
validateOneEnabledQueue(config)
for _, queue := range MessageQueues {
if config.GetBool(prefix + queue.GetName() + ".enabled") {
if err := queue.Initialize(config, prefix+queue.GetName()+"."); err != nil {
glog.Fatalf("Failed to initialize notification for %s: %+v",
queue.GetName(), err)
}
Queue = queue
glog.V(0).Infof("Configure notification message queue for %s", queue.GetName())
return
}
}
}
func validateOneEnabledQueue(config *util.ViperProxy) {
enabledQueue := ""
for _, queue := range MessageQueues {
if config.GetBool(queue.GetName() + ".enabled") {
if enabledQueue == "" {
enabledQueue = queue.GetName()
} else {
glog.Fatalf("Notification message queue is enabled for both %s and %s", enabledQueue, queue.GetName())
}
}
}
}

View File

@@ -0,0 +1,7 @@
/*
Package gocdk_pub_sub is for Azure Service Bus and RabbitMQ.
The referenced "gocloud.dev/pubsub" library is too big when compiled.
So this is only compiled in "make full_install".
*/
package gocdk_pub_sub

View File

@@ -0,0 +1,120 @@
//go:build gocdk
// Package gocdk_pub_sub supports the Go CDK (Cloud Development Kit) PubSub API,
// which in turn supports many providers, including Amazon SNS/SQS, Azure Service Bus,
// Google Cloud PubSub, and RabbitMQ.
//
// In the config, select a provider and topic using a URL. See
// https://godoc.org/gocloud.dev/pubsub and its sub-packages for details.
//
// The Go CDK PubSub API does not support administrative operations like topic
// creation. Create the topic using a UI, CLI or provider-specific API before running
// weed.
//
// The Go CDK obtains credentials via environment variables and other
// provider-specific default mechanisms. See the provider's documentation for
// details.
package gocdk_pub_sub
import (
"context"
"fmt"
"net/url"
"os"
"path"
"sync"
"time"
amqp "github.com/rabbitmq/amqp091-go"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/notification"
"github.com/seaweedfs/seaweedfs/weed/util"
"gocloud.dev/pubsub"
_ "gocloud.dev/pubsub/awssnssqs"
"gocloud.dev/pubsub/rabbitpubsub"
"google.golang.org/protobuf/proto"
// _ "gocloud.dev/pubsub/azuresb"
_ "gocloud.dev/pubsub/gcppubsub"
_ "gocloud.dev/pubsub/natspubsub"
_ "gocloud.dev/pubsub/rabbitpubsub"
)
func init() {
notification.MessageQueues = append(notification.MessageQueues, &GoCDKPubSub{})
}
func getPath(rawUrl string) string {
parsedUrl, _ := url.Parse(rawUrl)
return path.Join(parsedUrl.Host, parsedUrl.Path)
}
type GoCDKPubSub struct {
topicURL string
topic *pubsub.Topic
topicLock sync.RWMutex
}
func (k *GoCDKPubSub) GetName() string {
return "gocdk_pub_sub"
}
func (k *GoCDKPubSub) setTopic(topic *pubsub.Topic) {
k.topicLock.Lock()
k.topic = topic
k.topicLock.Unlock()
k.doReconnect()
}
func (k *GoCDKPubSub) doReconnect() {
var conn *amqp.Connection
k.topicLock.RLock()
defer k.topicLock.RUnlock()
if k.topic.As(&conn) {
go func(c *amqp.Connection) {
<-c.NotifyClose(make(chan *amqp.Error))
c.Close()
k.topicLock.RLock()
k.topic.Shutdown(context.Background())
k.topicLock.RUnlock()
for {
glog.Info("Try reconnect")
conn, err := amqp.Dial(os.Getenv("RABBIT_SERVER_URL"))
if err == nil {
k.setTopic(rabbitpubsub.OpenTopic(conn, getPath(k.topicURL), nil))
break
}
glog.Error(err)
time.Sleep(time.Second)
}
}(conn)
}
}
func (k *GoCDKPubSub) Initialize(configuration util.Configuration, prefix string) error {
k.topicURL = configuration.GetString(prefix + "topic_url")
glog.V(0).Infof("notification.gocdk_pub_sub.topic_url: %v", k.topicURL)
topic, err := pubsub.OpenTopic(context.Background(), k.topicURL)
if err != nil {
glog.Fatalf("Failed to open topic: %v", err)
}
k.setTopic(topic)
return nil
}
func (k *GoCDKPubSub) SendMessage(key string, message proto.Message) error {
bytes, err := proto.Marshal(message)
if err != nil {
return err
}
k.topicLock.RLock()
defer k.topicLock.RUnlock()
err = k.topic.Send(context.Background(), &pubsub.Message{
Body: bytes,
Metadata: map[string]string{"key": key},
})
if err != nil {
return fmt.Errorf("send message via Go CDK pubsub %s: %v", k.topicURL, err)
}
return nil
}

View File

@@ -0,0 +1,89 @@
package google_pub_sub
import (
"context"
"fmt"
"os"
"cloud.google.com/go/pubsub"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/notification"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/api/option"
"google.golang.org/protobuf/proto"
)
func init() {
notification.MessageQueues = append(notification.MessageQueues, &GooglePubSub{})
}
type GooglePubSub struct {
topic *pubsub.Topic
}
func (k *GooglePubSub) GetName() string {
return "google_pub_sub"
}
func (k *GooglePubSub) Initialize(configuration util.Configuration, prefix string) (err error) {
glog.V(0).Infof("notification.google_pub_sub.project_id: %v", configuration.GetString(prefix+"project_id"))
glog.V(0).Infof("notification.google_pub_sub.topic: %v", configuration.GetString(prefix+"topic"))
return k.initialize(
configuration.GetString(prefix+"google_application_credentials"),
configuration.GetString(prefix+"project_id"),
configuration.GetString(prefix+"topic"),
)
}
func (k *GooglePubSub) initialize(google_application_credentials, projectId, topicName string) (err error) {
ctx := context.Background()
// Creates a client.
if google_application_credentials == "" {
var found bool
google_application_credentials, found = os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS")
if !found {
glog.Fatalf("need to specific GOOGLE_APPLICATION_CREDENTIALS env variable or google_application_credentials in filer.toml")
}
}
client, err := pubsub.NewClient(ctx, projectId, option.WithCredentialsFile(google_application_credentials))
if err != nil {
glog.Fatalf("Failed to create client: %v", err)
}
k.topic = client.Topic(topicName)
if exists, err := k.topic.Exists(ctx); err == nil {
if !exists {
k.topic, err = client.CreateTopic(ctx, topicName)
if err != nil {
glog.Fatalf("Failed to create topic %s: %v", topicName, err)
}
}
} else {
glog.Fatalf("Failed to check topic %s: %v", topicName, err)
}
return nil
}
func (k *GooglePubSub) SendMessage(key string, message proto.Message) (err error) {
bytes, err := proto.Marshal(message)
if err != nil {
return
}
ctx := context.Background()
result := k.topic.Publish(ctx, &pubsub.Message{
Data: bytes,
Attributes: map[string]string{"key": key},
})
_, err = result.Get(ctx)
if err != nil {
return fmt.Errorf("send message to google pub sub %s: %v", k.topic.String(), err)
}
return nil
}

View File

@@ -0,0 +1,98 @@
package kafka
import (
"fmt"
"github.com/Shopify/sarama"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/notification"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/proto"
)
func init() {
notification.MessageQueues = append(notification.MessageQueues, &KafkaQueue{})
}
type KafkaQueue struct {
topic string
producer sarama.AsyncProducer
}
func (k *KafkaQueue) GetName() string {
return "kafka"
}
func (k *KafkaQueue) Initialize(configuration util.Configuration, prefix string) (err error) {
glog.V(0).Infof("filer.notification.kafka.hosts: %v\n", configuration.GetStringSlice(prefix+"hosts"))
glog.V(0).Infof("filer.notification.kafka.topic: %v\n", configuration.GetString(prefix+"topic"))
return k.initialize(
configuration.GetStringSlice(prefix+"hosts"),
configuration.GetString(prefix+"topic"),
SASLTLSConfig{
SASLEnabled: configuration.GetBool(prefix + "sasl_enabled"),
SASLMechanism: configuration.GetString(prefix + "sasl_mechanism"),
SASLUsername: configuration.GetString(prefix + "sasl_username"),
SASLPassword: configuration.GetString(prefix + "sasl_password"),
TLSEnabled: configuration.GetBool(prefix + "tls_enabled"),
TLSCACert: configuration.GetString(prefix + "tls_ca_cert"),
TLSClientCert: configuration.GetString(prefix + "tls_client_cert"),
TLSClientKey: configuration.GetString(prefix + "tls_client_key"),
TLSInsecureSkipVerify: configuration.GetBool(prefix + "tls_insecure_skip_verify"),
},
)
}
func (k *KafkaQueue) initialize(hosts []string, topic string, saslTLS SASLTLSConfig) (err error) {
config := sarama.NewConfig()
config.Producer.RequiredAcks = sarama.WaitForLocal
config.Producer.Partitioner = sarama.NewHashPartitioner
config.Producer.Return.Successes = true
config.Producer.Return.Errors = true
if err = ConfigureSASLTLS(config, saslTLS); err != nil {
return fmt.Errorf("kafka producer security configuration: %w", err)
}
k.producer, err = sarama.NewAsyncProducer(hosts, config)
if err != nil {
return err
}
k.topic = topic
go k.handleSuccess()
go k.handleError()
return nil
}
func (k *KafkaQueue) SendMessage(key string, message proto.Message) (err error) {
bytes, err := proto.Marshal(message)
if err != nil {
return
}
msg := &sarama.ProducerMessage{
Topic: k.topic,
Key: sarama.StringEncoder(key),
Value: sarama.ByteEncoder(bytes),
}
k.producer.Input() <- msg
return nil
}
func (k *KafkaQueue) handleSuccess() {
for {
pm := <-k.producer.Successes()
if pm != nil {
glog.V(3).Infof("producer message success, partition:%d offset:%d key:%v", pm.Partition, pm.Offset, pm.Key)
}
}
}
func (k *KafkaQueue) handleError() {
for {
err := <-k.producer.Errors()
if err != nil {
glog.Errorf("producer message error, partition:%d offset:%d key:%v value:%s error(%v) topic:%s", err.Msg.Partition, err.Msg.Offset, err.Msg.Key, err.Msg.Value, err.Err, k.topic)
}
}
}

View File

@@ -0,0 +1,112 @@
package kafka
import (
"crypto/tls"
"crypto/x509"
"fmt"
"os"
"strings"
"github.com/Shopify/sarama"
"github.com/xdg-go/scram"
)
// SASLTLSConfig holds SASL and TLS configuration for Kafka connections.
type SASLTLSConfig struct {
SASLEnabled bool
SASLMechanism string
SASLUsername string
SASLPassword string
TLSEnabled bool
TLSCACert string
TLSClientCert string
TLSClientKey string
TLSInsecureSkipVerify bool
}
// ConfigureSASLTLS applies SASL and TLS settings to a sarama config.
func ConfigureSASLTLS(config *sarama.Config, st SASLTLSConfig) error {
if st.SASLEnabled {
config.Net.SASL.Enable = true
config.Net.SASL.User = st.SASLUsername
config.Net.SASL.Password = st.SASLPassword
mechanism := strings.ToUpper(st.SASLMechanism)
switch mechanism {
case "PLAIN", "":
config.Net.SASL.Mechanism = sarama.SASLTypePlaintext
case "SCRAM-SHA-256":
config.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA256
config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
return &scramClient{HashGeneratorFcn: scram.SHA256}
}
case "SCRAM-SHA-512":
config.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA512
config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
return &scramClient{HashGeneratorFcn: scram.SHA512}
}
default:
return fmt.Errorf("unsupported SASL mechanism: %s", mechanism)
}
}
if st.TLSEnabled {
if (st.TLSClientCert == "") != (st.TLSClientKey == "") {
return fmt.Errorf("both tls_client_cert and tls_client_key must be provided for mTLS, or neither")
}
tlsConfig := &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: st.TLSInsecureSkipVerify,
}
if st.TLSCACert != "" {
caCert, err := os.ReadFile(st.TLSCACert)
if err != nil {
return fmt.Errorf("failed to read CA certificate: %w", err)
}
caCertPool := x509.NewCertPool()
if !caCertPool.AppendCertsFromPEM(caCert) {
return fmt.Errorf("failed to parse CA certificate")
}
tlsConfig.RootCAs = caCertPool
}
if st.TLSClientCert != "" && st.TLSClientKey != "" {
cert, err := tls.LoadX509KeyPair(st.TLSClientCert, st.TLSClientKey)
if err != nil {
return fmt.Errorf("failed to load client certificate/key: %w", err)
}
tlsConfig.Certificates = []tls.Certificate{cert}
}
config.Net.TLS.Enable = true
config.Net.TLS.Config = tlsConfig
}
return nil
}
// scramClient implements the sarama.SCRAMClient interface.
type scramClient struct {
*scram.ClientConversation
scram.HashGeneratorFcn
}
func (c *scramClient) Begin(userName, password, authzID string) (err error) {
client, err := c.HashGeneratorFcn.NewClient(userName, password, authzID)
if err != nil {
return err
}
c.ClientConversation = client.NewConversation()
return nil
}
func (c *scramClient) Step(challenge string) (string, error) {
return c.ClientConversation.Step(challenge)
}
func (c *scramClient) Done() bool {
return c.ClientConversation.Done()
}

View File

@@ -0,0 +1,65 @@
package kafka
import (
"strings"
"testing"
"github.com/Shopify/sarama"
)
func TestConfigureSASLTLSRejectsPartialMTLSConfig(t *testing.T) {
tests := []struct {
name string
cfg SASLTLSConfig
}{
{
name: "missing key",
cfg: SASLTLSConfig{
TLSEnabled: true,
TLSClientCert: "/tmp/client.crt",
},
},
{
name: "missing cert",
cfg: SASLTLSConfig{
TLSEnabled: true,
TLSClientKey: "/tmp/client.key",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := ConfigureSASLTLS(sarama.NewConfig(), tt.cfg)
if err == nil {
t.Fatal("expected error")
}
if !strings.Contains(err.Error(), "both tls_client_cert and tls_client_key must be provided") {
t.Fatalf("unexpected error: %v", err)
}
})
}
}
func TestConfigureSASLTLSConfiguresSCRAMSHA256(t *testing.T) {
config := sarama.NewConfig()
err := ConfigureSASLTLS(config, SASLTLSConfig{
SASLEnabled: true,
SASLMechanism: "SCRAM-SHA-256",
SASLUsername: "alice",
SASLPassword: "secret",
})
if err != nil {
t.Fatalf("ConfigureSASLTLS returned error: %v", err)
}
if !config.Net.SASL.Enable {
t.Fatal("expected SASL to be enabled")
}
if config.Net.SASL.Mechanism != sarama.SASLTypeSCRAMSHA256 {
t.Fatalf("unexpected mechanism: %v", config.Net.SASL.Mechanism)
}
if config.Net.SASL.SCRAMClientGeneratorFunc == nil {
t.Fatal("expected SCRAM client generator")
}
}

View File

@@ -0,0 +1,29 @@
package kafka
import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/notification"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/proto"
)
func init() {
notification.MessageQueues = append(notification.MessageQueues, &LogQueue{})
}
type LogQueue struct {
}
func (k *LogQueue) GetName() string {
return "log"
}
func (k *LogQueue) Initialize(configuration util.Configuration, prefix string) (err error) {
return nil
}
func (k *LogQueue) SendMessage(key string, message proto.Message) (err error) {
glog.V(0).Infof("%v: %+v", key, message)
return nil
}

View File

@@ -0,0 +1,77 @@
package webhook
import (
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
type filter struct {
eventTypes map[eventType]bool
pathPrefixes []string
}
func newFilter(cfg *config) *filter {
f := &filter{
eventTypes: make(map[eventType]bool),
pathPrefixes: cfg.pathPrefixes,
}
if len(cfg.eventTypes) == 0 {
f.eventTypes[eventTypeCreate] = true
f.eventTypes[eventTypeDelete] = true
f.eventTypes[eventTypeUpdate] = true
f.eventTypes[eventTypeRename] = true
} else {
for _, et := range cfg.eventTypes {
t := eventType(et)
if !t.valid() {
glog.Warningf("invalid event type: %v", t)
continue
}
f.eventTypes[t] = true
}
}
return f
}
func (f *filter) shouldPublish(key string, notification *filer_pb.EventNotification) bool {
if !f.matchesPath(key, notification) {
return false
}
eventType := detectEventType(key, notification)
return f.eventTypes[eventType]
}
func (f *filter) matchesPath(key string, notification *filer_pb.EventNotification) bool {
if len(f.pathPrefixes) == 0 {
return true
}
if f.matchesAnyPathPrefix(key) {
return true
}
if notification != nil && notification.NewEntry != nil && notification.NewParentPath != "" {
newKey := string(util.FullPath(notification.NewParentPath).Child(notification.NewEntry.Name))
return f.matchesAnyPathPrefix(newKey)
}
return false
}
func (f *filter) matchesAnyPathPrefix(key string) bool {
for _, prefix := range f.pathPrefixes {
if strings.HasPrefix(key, prefix) {
return true
}
}
return false
}

View File

@@ -0,0 +1,268 @@
package webhook
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func TestFilterEventTypes(t *testing.T) {
tests := []struct {
name string
key string
eventTypes []string
notification *filer_pb.EventNotification
expectedType eventType
shouldPublish bool
}{
{
name: "create event - allowed",
key: "/test/test.txt",
eventTypes: []string{"create", "delete"},
notification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "test.txt"},
},
expectedType: eventTypeCreate,
shouldPublish: true,
},
{
name: "create event - not allowed",
key: "/test/test.txt",
eventTypes: []string{"delete", "update"},
notification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "test.txt"},
},
expectedType: eventTypeCreate,
shouldPublish: false,
},
{
name: "delete event - allowed",
key: "/test/test.txt",
eventTypes: []string{"create", "delete"},
notification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "test.txt"},
},
expectedType: eventTypeDelete,
shouldPublish: true,
},
{
name: "update event - allowed",
key: "/test/test.txt",
eventTypes: []string{"update"},
notification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "test.txt"},
NewEntry: &filer_pb.Entry{Name: "test.txt"},
NewParentPath: "/test",
},
expectedType: eventTypeUpdate,
shouldPublish: true,
},
{
name: "rename event - allowed",
key: "/old/path/old.txt",
eventTypes: []string{"rename"},
notification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "old.txt"},
NewEntry: &filer_pb.Entry{Name: "new.txt"},
NewParentPath: "/new/path",
},
expectedType: eventTypeRename,
shouldPublish: true,
},
{
name: "rename event same name different parent - allowed",
key: "/old/path/file.txt",
eventTypes: []string{"rename"},
notification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "file.txt"},
NewEntry: &filer_pb.Entry{Name: "file.txt"},
NewParentPath: "/new/path",
},
expectedType: eventTypeRename,
shouldPublish: true,
},
{
name: "rename event - not allowed",
key: "/old/path/old.txt",
eventTypes: []string{"create", "delete", "update"},
notification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "old.txt"},
NewEntry: &filer_pb.Entry{Name: "new.txt"},
NewParentPath: "/new/path",
},
expectedType: eventTypeRename,
shouldPublish: false,
},
{
name: "all events allowed when empty",
key: "/test/test.txt",
eventTypes: []string{},
notification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "test.txt"},
},
expectedType: eventTypeCreate,
shouldPublish: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &config{eventTypes: tt.eventTypes}
f := newFilter(cfg)
eventType := detectEventType(tt.key, tt.notification)
if eventType != tt.expectedType {
t.Errorf("detectEventType() = %v, want %v", eventType, tt.expectedType)
}
shouldPublish := f.shouldPublish(tt.key, tt.notification)
if shouldPublish != tt.shouldPublish {
t.Errorf("shouldPublish() = %v, want %v", shouldPublish, tt.shouldPublish)
}
})
}
}
func TestFilterPathPrefixes(t *testing.T) {
tests := []struct {
name string
pathPrefixes []string
eventTypes []string
key string
notification *filer_pb.EventNotification
shouldPublish bool
}{
{
name: "matches single prefix",
pathPrefixes: []string{"/data/"},
eventTypes: []string{"create"},
key: "/data/file.txt",
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "file.txt"}},
shouldPublish: true,
},
{
name: "matches one of multiple prefixes",
pathPrefixes: []string{"/data/", "/logs/", "/tmp/"},
eventTypes: []string{"create"},
key: "/logs/app.log",
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "app.log"}},
shouldPublish: true,
},
{
name: "no match",
pathPrefixes: []string{"/data/", "/logs/"},
eventTypes: []string{"create"},
key: "/other/file.txt",
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "file.txt"}},
shouldPublish: false,
},
{
name: "empty prefixes allows all",
pathPrefixes: []string{},
eventTypes: []string{"create"},
key: "/any/path/file.txt",
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "file.txt"}},
shouldPublish: true,
},
{
name: "exact prefix match",
pathPrefixes: []string{"/data"},
eventTypes: []string{"create"},
key: "/data",
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "data"}},
shouldPublish: true,
},
{
name: "partial match not allowed",
pathPrefixes: []string{"/data/"},
eventTypes: []string{"create"},
key: "/database/file.txt",
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "file.txt"}},
shouldPublish: false,
},
{
name: "rename matches destination prefix",
pathPrefixes: []string{"/watched/"},
eventTypes: []string{"rename"},
key: "/outside/old.txt",
notification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "old.txt"},
NewEntry: &filer_pb.Entry{Name: "new.txt"},
NewParentPath: "/watched",
},
shouldPublish: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &config{
pathPrefixes: tt.pathPrefixes,
eventTypes: tt.eventTypes,
}
f := newFilter(cfg)
shouldPublish := f.shouldPublish(tt.key, tt.notification)
if shouldPublish != tt.shouldPublish {
t.Errorf("shouldPublish() = %v, want %v", shouldPublish, tt.shouldPublish)
}
})
}
}
func TestFilterCombined(t *testing.T) {
cfg := &config{
eventTypes: []string{"create", "update"},
pathPrefixes: []string{"/data/", "/logs/"},
}
f := newFilter(cfg)
tests := []struct {
name string
key string
notification *filer_pb.EventNotification
shouldPublish bool
}{
{
name: "allowed event and path",
key: "/data/file.txt",
notification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "file.txt"},
},
shouldPublish: true,
},
{
name: "allowed event but wrong path",
key: "/other/file.txt",
notification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "file.txt"},
},
shouldPublish: false,
},
{
name: "wrong event but allowed path",
key: "/data/file.txt",
notification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "file.txt"},
},
shouldPublish: false,
},
{
name: "wrong event and wrong path",
key: "/other/file.txt",
notification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "file.txt"},
},
shouldPublish: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
shouldPublish := f.shouldPublish(tt.key, tt.notification)
if shouldPublish != tt.shouldPublish {
t.Errorf("shouldPublish() = %v, want %v", shouldPublish, tt.shouldPublish)
}
})
}
}

View File

@@ -0,0 +1,184 @@
package webhook
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/security"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
const (
maxWebhookRetryDepth = 2
)
type httpClient struct {
endpoint string
token string
timeout time.Duration
client *http.Client // Reused HTTP client with redirect prevention
endpointMu sync.RWMutex
finalURL string // Cached final URL after following redirects
}
func newHTTPClient(cfg *config) (*httpClient, error) {
return &httpClient{
endpoint: cfg.endpoint,
token: cfg.authBearerToken,
timeout: time.Duration(cfg.timeoutSeconds) * time.Second,
client: &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
},
}, nil
}
func (h *httpClient) sendMessage(message *webhookMessage) error {
return h.sendMessageWithRetry(message, 0)
}
func (h *httpClient) sendMessageWithRetry(message *webhookMessage, depth int) error {
// Prevent infinite recursion
if depth > maxWebhookRetryDepth {
return fmt.Errorf("webhook max retry depth exceeded")
}
// Serialize the protobuf message to JSON for HTTP payload
notificationData, err := json.Marshal(message.Notification)
if err != nil {
return fmt.Errorf("failed to marshal notification: %w", err)
}
payload := map[string]interface{}{
"key": message.Key,
"event_type": message.EventType,
"message": json.RawMessage(notificationData),
}
jsonData, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("failed to marshal message: %w", err)
}
// Use cached final URL if available, otherwise use original endpoint
h.endpointMu.RLock()
targetURL := h.finalURL
usingCachedURL := targetURL != ""
if targetURL == "" {
targetURL = h.endpoint
}
h.endpointMu.RUnlock()
req, err := http.NewRequest(http.MethodPost, targetURL, bytes.NewBuffer(jsonData))
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
if h.token != "" {
req.Header.Set("Authorization", security.BearerPrefix+h.token)
}
// Apply timeout via context (not on client) to avoid redundancy
if h.timeout > 0 {
ctx, cancel := context.WithTimeout(context.Background(), h.timeout)
defer cancel()
req = req.WithContext(ctx)
}
resp, err := h.client.Do(req)
if err != nil {
if drainErr := drainResponse(resp); drainErr != nil {
glog.Errorf("failed to drain response: %v", drainErr)
}
// If using cached URL and request failed, clear cache and retry with original endpoint
if usingCachedURL && depth == 0 {
glog.V(1).Infof("Webhook request to cached URL %s failed, clearing cache and retrying with original endpoint", targetURL)
h.setFinalURL("")
return h.sendMessageWithRetry(message, depth+1)
}
return fmt.Errorf("failed to send request: %w", err)
}
// Handle redirects by caching the final destination and recreating POST request
if resp.StatusCode >= 300 && resp.StatusCode < 400 {
// Drain and close response body to enable connection reuse
util_http.CloseResponse(resp)
location := resp.Header.Get("Location")
if location == "" {
return fmt.Errorf("webhook returned redirect status %d without Location header", resp.StatusCode)
}
// Resolve relative URLs against the request URL
reqURL := req.URL
finalURL, err := reqURL.Parse(location)
if err != nil {
return fmt.Errorf("failed to parse redirect location: %w", err)
}
// Update finalURL to follow the redirect for this attempt
finalURLStr := finalURL.String()
h.setFinalURL(finalURLStr)
if depth == 0 {
glog.V(1).Infof("Webhook endpoint redirected from %s to %s, caching final destination", targetURL, finalURLStr)
} else {
glog.V(1).Infof("Webhook endpoint redirected from %s to %s (following redirect on retry)", targetURL, finalURLStr)
}
// Recreate the POST request to the final destination (increment depth to prevent infinite loops)
return h.sendMessageWithRetry(message, depth+1)
}
// If using cached URL and got an error response, clear cache and retry with original endpoint
if resp.StatusCode >= 400 && usingCachedURL && depth == 0 {
// Drain and close response body to enable connection reuse
util_http.CloseResponse(resp)
glog.V(1).Infof("Webhook request to cached URL %s returned error %d, clearing cache and retrying with original endpoint", targetURL, resp.StatusCode)
h.setFinalURL("")
return h.sendMessageWithRetry(message, depth+1)
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
// Drain and close response body before returning error to enable connection reuse
util_http.CloseResponse(resp)
return fmt.Errorf("webhook returned status code: %d", resp.StatusCode)
}
// Drain and close response body on success to enable connection reuse
util_http.CloseResponse(resp)
return nil
}
func (h *httpClient) setFinalURL(url string) {
h.endpointMu.Lock()
h.finalURL = url
h.endpointMu.Unlock()
}
func drainResponse(resp *http.Response) error {
if resp == nil || resp.Body == nil {
return nil
}
_, err := io.ReadAll(resp.Body)
return errors.Join(
err,
resp.Body.Close(),
)
}

View File

@@ -0,0 +1,522 @@
package webhook
import (
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
func init() {
util_http.InitGlobalHttpClient()
}
func TestHttpClientSendMessage(t *testing.T) {
var receivedPayload map[string]interface{}
var receivedHeaders http.Header
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
receivedHeaders = r.Header
body, _ := io.ReadAll(r.Body)
if err := json.Unmarshal(body, &receivedPayload); err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
cfg := &config{
endpoint: server.URL,
authBearerToken: "test-token",
}
client, err := newHTTPClient(cfg)
if err != nil {
t.Fatalf("Failed to create HTTP client: %v", err)
}
message := &filer_pb.EventNotification{
OldEntry: nil,
NewEntry: &filer_pb.Entry{
Name: "test.txt",
IsDirectory: false,
},
}
err = client.sendMessage(newWebhookMessage("/test/path", message))
if err != nil {
t.Fatalf("Failed to send message: %v", err)
}
if receivedPayload["key"] != "/test/path" {
t.Errorf("Expected key '/test/path', got %v", receivedPayload["key"])
}
if receivedPayload["event_type"] != "create" {
t.Errorf("Expected event_type 'create', got %v", receivedPayload["event_type"])
}
if receivedPayload["message"] == nil {
t.Error("Expected message to be present")
}
if receivedHeaders.Get("Content-Type") != "application/json" {
t.Errorf("Expected Content-Type 'application/json', got %s", receivedHeaders.Get("Content-Type"))
}
expectedAuth := "Bearer test-token"
if receivedHeaders.Get("Authorization") != expectedAuth {
t.Errorf("Expected Authorization '%s', got %s", expectedAuth, receivedHeaders.Get("Authorization"))
}
}
func TestHttpClientSendMessageWithoutToken(t *testing.T) {
var receivedHeaders http.Header
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
receivedHeaders = r.Header
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
cfg := &config{
endpoint: server.URL,
authBearerToken: "",
}
client, err := newHTTPClient(cfg)
if err != nil {
t.Fatalf("Failed to create HTTP client: %v", err)
}
message := &filer_pb.EventNotification{}
err = client.sendMessage(newWebhookMessage("/test/path", message))
if err != nil {
t.Fatalf("Failed to send message: %v", err)
}
if receivedHeaders.Get("Authorization") != "" {
t.Errorf("Expected no Authorization header, got %s", receivedHeaders.Get("Authorization"))
}
}
func TestHttpClientSendMessageServerError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
defer server.Close()
cfg := &config{
endpoint: server.URL,
authBearerToken: "test-token",
}
client, err := newHTTPClient(cfg)
if err != nil {
t.Fatalf("Failed to create HTTP client: %v", err)
}
message := &filer_pb.EventNotification{}
err = client.sendMessage(newWebhookMessage("/test/path", message))
if err == nil {
t.Error("Expected error for server error response")
}
}
func TestHttpClientSendMessageNetworkError(t *testing.T) {
cfg := &config{
endpoint: "http://localhost:99999",
authBearerToken: "",
}
client, err := newHTTPClient(cfg)
if err != nil {
t.Fatalf("Failed to create HTTP client: %v", err)
}
message := &filer_pb.EventNotification{}
err = client.sendMessage(newWebhookMessage("/test/path", message))
if err == nil {
t.Error("Expected error for network failure")
}
}
// TestHttpClientFollowsRedirectAsPost verifies that redirects are followed with POST method preserved
func TestHttpClientFollowsRedirectAsPost(t *testing.T) {
redirectCalled := false
finalCalled := false
var finalMethod string
var finalBody map[string]interface{}
// Create final destination server
finalServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
finalCalled = true
finalMethod = r.Method
body, _ := io.ReadAll(r.Body)
json.Unmarshal(body, &finalBody)
w.WriteHeader(http.StatusOK)
}))
defer finalServer.Close()
// Create redirect server
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
redirectCalled = true
// Return 301 redirect to final server
http.Redirect(w, r, finalServer.URL, http.StatusMovedPermanently)
}))
defer redirectServer.Close()
cfg := &config{
endpoint: redirectServer.URL,
authBearerToken: "test-token",
timeoutSeconds: 5,
}
client, err := newHTTPClient(cfg)
if err != nil {
t.Fatalf("Failed to create HTTP client: %v", err)
}
message := &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{
Name: "test.txt",
},
}
// Send message - should follow redirect and recreate POST request
err = client.sendMessage(newWebhookMessage("/test/path", message))
if err != nil {
t.Fatalf("Failed to send message: %v", err)
}
if !redirectCalled {
t.Error("Expected redirect server to be called")
}
if !finalCalled {
t.Error("Expected final server to be called after redirect")
}
if finalMethod != "POST" {
t.Errorf("Expected POST method at final destination, got %s", finalMethod)
}
if finalBody["key"] != "/test/path" {
t.Errorf("Expected key '/test/path' at final destination, got %v", finalBody["key"])
}
// Verify the final URL is cached
client.endpointMu.RLock()
cachedURL := client.finalURL
client.endpointMu.RUnlock()
if cachedURL != finalServer.URL {
t.Errorf("Expected cached URL %s, got %s", finalServer.URL, cachedURL)
}
}
// TestHttpClientUsesCachedRedirect verifies that subsequent requests use the cached redirect destination
func TestHttpClientUsesCachedRedirect(t *testing.T) {
redirectCount := 0
finalCount := 0
// Create final destination server
finalServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
finalCount++
w.WriteHeader(http.StatusOK)
}))
defer finalServer.Close()
// Create redirect server
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
redirectCount++
http.Redirect(w, r, finalServer.URL, http.StatusMovedPermanently)
}))
defer redirectServer.Close()
cfg := &config{
endpoint: redirectServer.URL,
authBearerToken: "test-token",
timeoutSeconds: 5,
}
client, err := newHTTPClient(cfg)
if err != nil {
t.Fatalf("Failed to create HTTP client: %v", err)
}
message := &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{
Name: "test.txt",
},
}
// First request - should hit redirect server
err = client.sendMessage(newWebhookMessage("/test/path1", message))
if err != nil {
t.Fatalf("Failed to send first message: %v", err)
}
if redirectCount != 1 {
t.Errorf("Expected 1 redirect call, got %d", redirectCount)
}
if finalCount != 1 {
t.Errorf("Expected 1 final call, got %d", finalCount)
}
// Second request - should use cached URL and skip redirect server
err = client.sendMessage(newWebhookMessage("/test/path2", message))
if err != nil {
t.Fatalf("Failed to send second message: %v", err)
}
if redirectCount != 1 {
t.Errorf("Expected redirect server to be called only once (cached), got %d calls", redirectCount)
}
if finalCount != 2 {
t.Errorf("Expected 2 final calls, got %d", finalCount)
}
}
// TestHttpClientPreservesPostMethod verifies POST method is preserved and not converted to GET
func TestHttpClientPreservesPostMethod(t *testing.T) {
var receivedMethod string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
receivedMethod = r.Method
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
cfg := &config{
endpoint: server.URL,
authBearerToken: "test-token",
timeoutSeconds: 5,
}
client, err := newHTTPClient(cfg)
if err != nil {
t.Fatalf("Failed to create HTTP client: %v", err)
}
message := &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{
Name: "test.txt",
},
}
err = client.sendMessage(newWebhookMessage("/test/path", message))
if err != nil {
t.Fatalf("Failed to send message: %v", err)
}
if receivedMethod != "POST" {
t.Errorf("Expected POST method, got %s", receivedMethod)
}
}
// TestHttpClientInvalidatesCacheOnError verifies that cache is invalidated when cached URL fails
func TestHttpClientInvalidatesCacheOnError(t *testing.T) {
finalServerDown := false // Start with server UP
originalCallCount := 0
finalCallCount := 0
// Create final destination server that can be toggled
finalServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
finalCallCount++
if finalServerDown {
w.WriteHeader(http.StatusServiceUnavailable)
} else {
w.WriteHeader(http.StatusOK)
}
}))
defer finalServer.Close()
// Create redirect server
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
originalCallCount++
http.Redirect(w, r, finalServer.URL, http.StatusMovedPermanently)
}))
defer redirectServer.Close()
cfg := &config{
endpoint: redirectServer.URL,
authBearerToken: "test-token",
timeoutSeconds: 5,
}
client, err := newHTTPClient(cfg)
if err != nil {
t.Fatalf("Failed to create HTTP client: %v", err)
}
message := &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{
Name: "test.txt",
},
}
// First request - should follow redirect and cache the final URL
err = client.sendMessage(newWebhookMessage("/test/path1", message))
if err != nil {
t.Fatalf("Failed to send first message: %v", err)
}
if originalCallCount != 1 {
t.Errorf("Expected 1 original call, got %d", originalCallCount)
}
if finalCallCount != 1 {
t.Errorf("Expected 1 final call, got %d", finalCallCount)
}
// Verify cache was set
client.endpointMu.RLock()
cachedURL := client.finalURL
client.endpointMu.RUnlock()
if cachedURL != finalServer.URL {
t.Errorf("Expected cached URL %s, got %s", finalServer.URL, cachedURL)
}
// Second request with cached URL working - should use cache
err = client.sendMessage(newWebhookMessage("/test/path2", message))
if err != nil {
t.Fatalf("Failed to send second message: %v", err)
}
if originalCallCount != 1 {
t.Errorf("Expected still 1 original call (using cache), got %d", originalCallCount)
}
if finalCallCount != 2 {
t.Errorf("Expected 2 final calls, got %d", finalCallCount)
}
// Third request - bring final server DOWN, should invalidate cache and retry with original
// Flow: cached URL (fail, depth=0) -> clear cache -> retry original (depth=1) -> redirect -> final (fail, depth=2)
finalServerDown = true
err = client.sendMessage(newWebhookMessage("/test/path3", message))
if err == nil {
t.Error("Expected error when cached URL fails and retry also fails")
}
// originalCallCount: 1 (initial) + 1 (retry after cache invalidation) = 2
if originalCallCount != 2 {
t.Errorf("Expected 2 original calls, got %d", originalCallCount)
}
// finalCallCount: 2 (previous) + 1 (cached fail) + 1 (retry after redirect) = 4
if finalCallCount != 4 {
t.Errorf("Expected 4 final calls, got %d", finalCallCount)
}
// Verify final URL is still set (to the failed destination from the redirect)
client.endpointMu.RLock()
finalURLAfterError := client.finalURL
client.endpointMu.RUnlock()
if finalURLAfterError != finalServer.URL {
t.Errorf("Expected finalURL to be %s after error, got %s", finalServer.URL, finalURLAfterError)
}
// Fourth request - bring final server back UP
// Since cache still has the final URL, it should use it directly
finalServerDown = false
err = client.sendMessage(newWebhookMessage("/test/path4", message))
if err != nil {
t.Fatalf("Failed to send fourth message after recovery: %v", err)
}
// Should have used the cached URL directly (no new original call)
// originalCallCount: still 2
if originalCallCount != 2 {
t.Errorf("Expected 2 original calls (using cache), got %d", originalCallCount)
}
// finalCallCount: 4 + 1 = 5
if finalCallCount != 5 {
t.Errorf("Expected 5 final calls, got %d", finalCallCount)
}
// Verify cache was re-established
client.endpointMu.RLock()
reestablishedCache := client.finalURL
client.endpointMu.RUnlock()
if reestablishedCache != finalServer.URL {
t.Errorf("Expected cache to be re-established to %s, got %s", finalServer.URL, reestablishedCache)
}
}
// TestHttpClientInvalidatesCacheOnNetworkError verifies cache invalidation on network errors
func TestHttpClientInvalidatesCacheOnNetworkError(t *testing.T) {
originalCallCount := 0
var finalServer *httptest.Server
// Create redirect server
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
originalCallCount++
if finalServer != nil {
http.Redirect(w, r, finalServer.URL, http.StatusMovedPermanently)
} else {
w.WriteHeader(http.StatusInternalServerError)
}
}))
defer redirectServer.Close()
// Create final destination server
finalServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
cfg := &config{
endpoint: redirectServer.URL,
authBearerToken: "test-token",
timeoutSeconds: 5,
}
client, err := newHTTPClient(cfg)
if err != nil {
t.Fatalf("Failed to create HTTP client: %v", err)
}
message := &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{
Name: "test.txt",
},
}
// First request - establish cache
err = client.sendMessage(newWebhookMessage("/test/path1", message))
if err != nil {
t.Fatalf("Failed to send first message: %v", err)
}
if originalCallCount != 1 {
t.Errorf("Expected 1 original call, got %d", originalCallCount)
}
// Close final server to simulate network error
cachedURL := finalServer.URL
finalServer.Close()
finalServer = nil
// Second request - cached URL is down, should invalidate and retry with original
err = client.sendMessage(newWebhookMessage("/test/path2", message))
if err == nil {
t.Error("Expected error when network fails")
}
if originalCallCount != 2 {
t.Errorf("Expected 2 original calls (retry after cache invalidation), got %d", originalCallCount)
}
// Verify cache was cleared
client.endpointMu.RLock()
clearedCache := client.finalURL
client.endpointMu.RUnlock()
if clearedCache == cachedURL {
t.Errorf("Expected cache to be invalidated, but still has %s", clearedCache)
}
}

View File

@@ -0,0 +1,186 @@
package webhook
import (
"fmt"
"net/url"
"slices"
"strconv"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/proto"
)
const (
queueName = "webhook"
pubSubTopicName = "webhook_topic"
deadLetterTopic = "webhook_dead_letter"
)
type eventType string
const (
eventTypeCreate eventType = "create"
eventTypeDelete eventType = "delete"
eventTypeUpdate eventType = "update"
eventTypeRename eventType = "rename"
)
func (e eventType) valid() bool {
return slices.Contains([]eventType{
eventTypeCreate,
eventTypeDelete,
eventTypeUpdate,
eventTypeRename,
},
e,
)
}
var (
pubSubHandlerNameTemplate = func(n int) string {
return "webhook_handler_" + strconv.Itoa(n)
}
)
type client interface {
sendMessage(message *webhookMessage) error
}
type webhookMessage struct {
Key string `json:"key"`
EventType string `json:"event_type"`
Notification *filer_pb.EventNotification `json:"message_data"`
}
func newWebhookMessage(key string, message proto.Message) *webhookMessage {
notification, ok := message.(*filer_pb.EventNotification)
if !ok {
return nil
}
eventType := string(detectEventType(key, notification))
return &webhookMessage{
Key: key,
EventType: eventType,
Notification: notification,
}
}
type config struct {
endpoint string
authBearerToken string
timeoutSeconds int
maxRetries int
backoffSeconds int
maxBackoffSeconds int
nWorkers int
bufferSize int
eventTypes []string
pathPrefixes []string
}
func newConfigWithDefaults(configuration util.Configuration, prefix string) *config {
c := &config{
endpoint: configuration.GetString(prefix + "endpoint"),
authBearerToken: configuration.GetString(prefix + "bearer_token"),
timeoutSeconds: 10,
maxRetries: 3,
backoffSeconds: 3,
maxBackoffSeconds: 30,
nWorkers: 5,
bufferSize: 10_000,
}
if bufferSize := configuration.GetInt(prefix + "buffer_size"); bufferSize > 0 {
c.bufferSize = bufferSize
}
if workers := configuration.GetInt(prefix + "workers"); workers > 0 {
c.nWorkers = workers
}
if maxRetries := configuration.GetInt(prefix + "max_retries"); maxRetries > 0 {
c.maxRetries = maxRetries
}
if backoffSeconds := configuration.GetInt(prefix + "backoff_seconds"); backoffSeconds > 0 {
c.backoffSeconds = backoffSeconds
}
if maxBackoffSeconds := configuration.GetInt(prefix + "max_backoff_seconds"); maxBackoffSeconds > 0 {
c.maxBackoffSeconds = maxBackoffSeconds
}
if timeout := configuration.GetInt(prefix + "timeout_seconds"); timeout > 0 {
c.timeoutSeconds = timeout
}
c.eventTypes = configuration.GetStringSlice(prefix + "event_types")
c.pathPrefixes = configuration.GetStringSlice(prefix + "path_prefixes")
return c
}
func (c *config) validate() error {
if c.endpoint == "" {
return fmt.Errorf("webhook endpoint is required")
}
_, err := url.Parse(c.endpoint)
if err != nil {
return fmt.Errorf("invalid webhook endpoint: %w", err)
}
if c.timeoutSeconds < 1 || c.timeoutSeconds > 300 {
return fmt.Errorf("timeout must be between 1 and 300 seconds, got %d", c.timeoutSeconds)
}
if c.maxRetries < 0 || c.maxRetries > 10 {
return fmt.Errorf("max retries must be between 0 and 10, got %d", c.maxRetries)
}
if c.backoffSeconds < 1 || c.backoffSeconds > 60 {
return fmt.Errorf("backoff seconds must be between 1 and 60, got %d", c.backoffSeconds)
}
if c.maxBackoffSeconds < c.backoffSeconds || c.maxBackoffSeconds > 300 {
return fmt.Errorf("max backoff seconds must be between %d and 300, got %d", c.backoffSeconds, c.maxBackoffSeconds)
}
if c.nWorkers < 1 || c.nWorkers > 100 {
return fmt.Errorf("workers must be between 1 and 100, got %d", c.nWorkers)
}
if c.bufferSize < 100 || c.bufferSize > 1_000_000 {
return fmt.Errorf("buffer size must be between 100 and 1,000,000, got %d", c.bufferSize)
}
return nil
}
func detectEventType(key string, notification *filer_pb.EventNotification) eventType {
hasOldEntry := notification.OldEntry != nil
hasNewEntry := notification.NewEntry != nil
if !hasOldEntry && hasNewEntry {
return eventTypeCreate
}
if hasOldEntry && !hasNewEntry {
return eventTypeDelete
}
if hasOldEntry && hasNewEntry {
oldDir, _ := util.FullPath(key).DirAndName()
newDir := notification.NewParentPath
if newDir == "" {
newDir = oldDir
}
if oldDir != newDir || notification.OldEntry.Name != notification.NewEntry.Name {
return eventTypeRename
}
return eventTypeUpdate
}
return eventTypeUpdate
}

View File

@@ -0,0 +1,243 @@
package webhook
import (
"context"
"errors"
"fmt"
"time"
"github.com/ThreeDotsLabs/watermill"
"github.com/ThreeDotsLabs/watermill/message"
"github.com/ThreeDotsLabs/watermill/message/router/middleware"
"github.com/ThreeDotsLabs/watermill/message/router/plugin"
"github.com/ThreeDotsLabs/watermill/pubsub/gochannel"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/notification"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/proto"
)
func init() {
notification.MessageQueues = append(notification.MessageQueues, &Queue{})
}
type Queue struct {
router *message.Router
queueChannel *gochannel.GoChannel
config *config
client client
filter *filter
ctx context.Context
cancel context.CancelFunc
// Semaphore for controlling concurrent webhook requests
sem chan struct{}
}
func (w *Queue) GetName() string {
return queueName
}
func (w *Queue) SendMessage(key string, msg proto.Message) error {
eventNotification, ok := msg.(*filer_pb.EventNotification)
if !ok {
return nil
}
if w.filter != nil && !w.filter.shouldPublish(key, eventNotification) {
return nil
}
m := newWebhookMessage(key, msg)
if m == nil {
return nil
}
wMsg, err := m.toWaterMillMessage()
if err != nil {
return err
}
return w.queueChannel.Publish(pubSubTopicName, wMsg)
}
func (w *webhookMessage) toWaterMillMessage() (*message.Message, error) {
payload, err := proto.Marshal(w.Notification)
if err != nil {
return nil, err
}
msg := message.NewMessage(watermill.NewUUID(), payload)
// Set event type and key as metadata
msg.Metadata.Set("event_type", w.EventType)
msg.Metadata.Set("key", w.Key)
return msg, nil
}
func (w *Queue) Initialize(configuration util.Configuration, prefix string) error {
c := newConfigWithDefaults(configuration, prefix)
if err := c.validate(); err != nil {
return err
}
return w.initialize(c)
}
func (w *Queue) initialize(cfg *config) error {
w.ctx, w.cancel = context.WithCancel(context.Background())
w.config = cfg
w.filter = newFilter(cfg)
// Initialize semaphore for controlling concurrent webhook requests
w.sem = make(chan struct{}, cfg.nWorkers)
hClient, err := newHTTPClient(cfg)
if err != nil {
return fmt.Errorf("failed to create webhook http client: %w", err)
}
w.client = hClient
if err = w.setupWatermillQueue(cfg); err != nil {
return fmt.Errorf("failed to setup watermill queue: %w", err)
}
if err = w.logDeadLetterMessages(); err != nil {
return err
}
return nil
}
func (w *Queue) setupWatermillQueue(cfg *config) error {
logger := watermill.NewStdLogger(false, false)
pubSubConfig := gochannel.Config{
OutputChannelBuffer: int64(cfg.bufferSize),
Persistent: false,
}
w.queueChannel = gochannel.NewGoChannel(pubSubConfig, logger)
router, err := message.NewRouter(
message.RouterConfig{
CloseTimeout: 60 * time.Second,
},
logger,
)
if err != nil {
return fmt.Errorf("failed to create router: %w", err)
}
w.router = router
retryMiddleware := middleware.Retry{
MaxRetries: cfg.maxRetries,
InitialInterval: time.Duration(cfg.backoffSeconds) * time.Second,
MaxInterval: time.Duration(cfg.maxBackoffSeconds) * time.Second,
Multiplier: 2.0,
RandomizationFactor: 0.3,
Logger: logger,
}.Middleware
poisonQueue, err := middleware.PoisonQueue(w.queueChannel, deadLetterTopic)
if err != nil {
return fmt.Errorf("failed to create poison queue: %w", err)
}
router.AddPlugin(plugin.SignalsHandler)
router.AddMiddleware(retryMiddleware, poisonQueue)
// Add a single handler to avoid duplicate message delivery.
// With gochannel's default behavior, each handler call creates
// a separate subscription, and all subscriptions receive their own copy of each message.
// Using a single handler ensures each webhook is sent only once.
// Concurrency is controlled via semaphore in handleWebhook based on nWorkers config.
router.AddConsumerHandler(
"webhook_handler",
pubSubTopicName,
w.queueChannel,
w.handleWebhook,
)
go func() {
// cancels the queue context so the dead letter logger exists in case context not canceled by the shutdown signal already
defer w.cancel()
if err := router.Run(w.ctx); err != nil && !errors.Is(err, context.Canceled) {
glog.Errorf("webhook pubsub worker stopped with error: %v", err)
}
glog.Info("webhook pubsub worker stopped")
}()
return nil
}
func (w *Queue) handleWebhook(msg *message.Message) error {
// Acquire semaphore slot (blocks if at capacity)
w.sem <- struct{}{}
defer func() { <-w.sem }()
var n filer_pb.EventNotification
if err := proto.Unmarshal(msg.Payload, &n); err != nil {
glog.Errorf("failed to unmarshal protobuf message: %v", err)
return err
}
// Reconstruct webhook message from metadata and payload
webhookMsg := &webhookMessage{
Key: msg.Metadata.Get("key"),
EventType: msg.Metadata.Get("event_type"),
Notification: &n,
}
if err := w.client.sendMessage(webhookMsg); err != nil {
glog.Errorf("failed to send message to webhook %s: %v", webhookMsg.Key, err)
return err
}
return nil
}
func (w *Queue) logDeadLetterMessages() error {
ch, err := w.queueChannel.Subscribe(w.ctx, deadLetterTopic)
if err != nil {
return err
}
go func() {
for {
select {
case msg, ok := <-ch:
if !ok {
glog.Info("dead letter channel closed")
return
}
if msg == nil {
glog.Errorf("received nil message from dead letter channel")
continue
}
key := "unknown"
if msg.Metadata != nil {
if keyValue, exists := msg.Metadata["key"]; exists {
key = keyValue
}
}
payload := ""
if msg.Payload != nil {
var n filer_pb.EventNotification
if err := proto.Unmarshal(msg.Payload, &n); err != nil {
payload = fmt.Sprintf("failed to unmarshal payload: %v", err)
} else {
payload = n.String()
}
}
glog.Errorf("received dead letter message: %s, key: %s", payload, key)
case <-w.ctx.Done():
return
}
}
}()
return nil
}

View File

@@ -0,0 +1,609 @@
package webhook
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"google.golang.org/protobuf/proto"
)
func TestConfigValidation(t *testing.T) {
tests := []struct {
name string
config *config
wantErr bool
errMsg string
}{
{
name: "valid config",
config: &config{
endpoint: "https://example.com/webhook",
authBearerToken: "test-token",
timeoutSeconds: 30,
maxRetries: 3,
backoffSeconds: 5,
maxBackoffSeconds: 30,
nWorkers: 5,
bufferSize: 10000,
},
wantErr: false,
},
{
name: "empty endpoint",
config: &config{
endpoint: "",
timeoutSeconds: 30,
maxRetries: 3,
backoffSeconds: 5,
maxBackoffSeconds: 30,
nWorkers: 5,
bufferSize: 10000,
},
wantErr: true,
errMsg: "endpoint is required",
},
{
name: "invalid URL",
config: &config{
endpoint: "://invalid-url",
timeoutSeconds: 30,
maxRetries: 3,
backoffSeconds: 5,
maxBackoffSeconds: 30,
nWorkers: 5,
bufferSize: 10000,
},
wantErr: true,
errMsg: "invalid webhook endpoint",
},
{
name: "timeout too large",
config: &config{
endpoint: "https://example.com/webhook",
timeoutSeconds: 301,
maxRetries: 3,
backoffSeconds: 5,
maxBackoffSeconds: 30,
nWorkers: 5,
bufferSize: 10000,
},
wantErr: true,
errMsg: "timeout must be between",
},
{
name: "too many retries",
config: &config{
endpoint: "https://example.com/webhook",
timeoutSeconds: 30,
maxRetries: 11,
backoffSeconds: 5,
maxBackoffSeconds: 30,
nWorkers: 5,
bufferSize: 10000,
},
wantErr: true,
errMsg: "max retries must be between",
},
{
name: "too many workers",
config: &config{
endpoint: "https://example.com/webhook",
timeoutSeconds: 30,
maxRetries: 3,
backoffSeconds: 5,
maxBackoffSeconds: 30,
nWorkers: 101,
bufferSize: 10000,
},
wantErr: true,
errMsg: "workers must be between",
},
{
name: "buffer too large",
config: &config{
endpoint: "https://example.com/webhook",
timeoutSeconds: 30,
maxRetries: 3,
backoffSeconds: 5,
maxBackoffSeconds: 30,
nWorkers: 5,
bufferSize: 1000001,
},
wantErr: true,
errMsg: "buffer size must be between",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.config.validate()
if (err != nil) != tt.wantErr {
t.Errorf("validate() error = %v, wantErr %v", err, tt.wantErr)
}
if err != nil && tt.errMsg != "" {
if err.Error() == "" || !strings.Contains(err.Error(), tt.errMsg) {
t.Errorf("validate() error message = %v, want to contain %v", err.Error(), tt.errMsg)
}
}
})
}
}
func TestWebhookMessageSerialization(t *testing.T) {
msg := &filer_pb.EventNotification{
OldEntry: nil,
NewEntry: &filer_pb.Entry{
Name: "test.txt",
IsDirectory: false,
},
}
webhookMsg := newWebhookMessage("/test/path", msg)
wmMsg, err := webhookMsg.toWaterMillMessage()
if err != nil {
t.Fatalf("Failed to convert to watermill message: %v", err)
}
// Unmarshal the protobuf payload directly
var eventNotification filer_pb.EventNotification
err = proto.Unmarshal(wmMsg.Payload, &eventNotification)
if err != nil {
t.Fatalf("Failed to unmarshal protobuf message: %v", err)
}
// Check metadata
if wmMsg.Metadata.Get("key") != "/test/path" {
t.Errorf("Expected key '/test/path', got %v", wmMsg.Metadata.Get("key"))
}
if wmMsg.Metadata.Get("event_type") != "create" {
t.Errorf("Expected event type 'create', got %v", wmMsg.Metadata.Get("event_type"))
}
if eventNotification.NewEntry.Name != "test.txt" {
t.Errorf("Expected file name 'test.txt', got %v", eventNotification.NewEntry.Name)
}
}
func TestQueueInitialize(t *testing.T) {
cfg := &config{
endpoint: "https://example.com/webhook",
authBearerToken: "test-token",
timeoutSeconds: 10,
maxRetries: 3,
backoffSeconds: 3,
maxBackoffSeconds: 60,
nWorkers: 1,
bufferSize: 100,
}
q := &Queue{}
err := q.initialize(cfg)
if err != nil {
t.Errorf("Initialize() error = %v", err)
}
defer func() {
if q.cancel != nil {
q.cancel()
}
time.Sleep(100 * time.Millisecond)
if q.router != nil {
q.router.Close()
}
}()
if q.router == nil {
t.Error("Expected router to be initialized")
}
if q.queueChannel == nil {
t.Error("Expected queueChannel to be initialized")
}
if q.client == nil {
t.Error("Expected client to be initialized")
}
if q.config == nil {
t.Error("Expected config to be initialized")
}
}
// TestQueueSendMessage test sending messages to the queue
func TestQueueSendMessage(t *testing.T) {
cfg := &config{
endpoint: "https://example.com/webhook",
authBearerToken: "test-token",
timeoutSeconds: 1,
maxRetries: 1,
backoffSeconds: 1,
maxBackoffSeconds: 1,
nWorkers: 1,
bufferSize: 10,
}
q := &Queue{}
err := q.initialize(cfg)
if err != nil {
t.Fatalf("Failed to initialize queue: %v", err)
}
defer func() {
if q.cancel != nil {
q.cancel()
}
time.Sleep(100 * time.Millisecond)
if q.router != nil {
q.router.Close()
}
}()
msg := &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{
Name: "test.txt",
},
}
err = q.SendMessage("/test/path", msg)
if err != nil {
t.Errorf("SendMessage() error = %v", err)
}
}
func TestQueueHandleWebhook(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
cfg := &config{
endpoint: server.URL,
authBearerToken: "test-token",
timeoutSeconds: 1,
maxRetries: 0,
backoffSeconds: 1,
maxBackoffSeconds: 1,
nWorkers: 1,
bufferSize: 10,
}
client, _ := newHTTPClient(cfg)
q := &Queue{
client: client,
sem: make(chan struct{}, cfg.nWorkers),
}
message := newWebhookMessage("/test/path", &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{
Name: "test.txt",
},
})
wmMsg, err := message.toWaterMillMessage()
if err != nil {
t.Fatalf("Failed to create watermill message: %v", err)
}
err = q.handleWebhook(wmMsg)
if err != nil {
t.Errorf("handleWebhook() error = %v", err)
}
}
func TestQueueEndToEnd(t *testing.T) {
// Simplified test - just verify the queue can be created and message can be sent
// without needing full end-to-end processing
cfg := &config{
endpoint: "https://example.com/webhook",
authBearerToken: "test-token",
timeoutSeconds: 1,
maxRetries: 0,
backoffSeconds: 1,
maxBackoffSeconds: 1,
nWorkers: 1,
bufferSize: 10,
}
q := &Queue{}
err := q.initialize(cfg)
if err != nil {
t.Fatalf("Failed to initialize queue: %v", err)
}
defer func() {
if q.cancel != nil {
q.cancel()
}
time.Sleep(100 * time.Millisecond)
if q.router != nil {
q.router.Close()
}
}()
msg := &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{
Name: "test.txt",
},
}
err = q.SendMessage("/test/path", msg)
if err != nil {
t.Errorf("SendMessage() error = %v", err)
}
}
func TestQueueRetryMechanism(t *testing.T) {
cfg := &config{
endpoint: "https://example.com/webhook",
authBearerToken: "test-token",
timeoutSeconds: 1,
maxRetries: 3, // Test that this config is used
backoffSeconds: 2,
maxBackoffSeconds: 10,
nWorkers: 1,
bufferSize: 10,
}
q := &Queue{}
err := q.initialize(cfg)
if err != nil {
t.Fatalf("Failed to initialize queue: %v", err)
}
defer func() {
if q.cancel != nil {
q.cancel()
}
time.Sleep(100 * time.Millisecond)
if q.router != nil {
q.router.Close()
}
}()
// Verify that the queue is properly configured for retries
if q.config.maxRetries != 3 {
t.Errorf("Expected maxRetries=3, got %d", q.config.maxRetries)
}
if q.config.backoffSeconds != 2 {
t.Errorf("Expected backoffSeconds=2, got %d", q.config.backoffSeconds)
}
if q.config.maxBackoffSeconds != 10 {
t.Errorf("Expected maxBackoffSeconds=10, got %d", q.config.maxBackoffSeconds)
}
// Test that we can send a message (retry behavior is handled by Watermill middleware)
msg := &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "test.txt"},
}
err = q.SendMessage("/test/retry", msg)
if err != nil {
t.Errorf("SendMessage() error = %v", err)
}
}
// TestQueueNoDuplicateWebhooks verifies that webhooks are sent only once regardless of worker count
func TestQueueNoDuplicateWebhooks(t *testing.T) {
tests := []struct {
name string
nWorkers int
expected int
}{
{"1 worker", 1, 1},
{"5 workers", 5, 1},
{"10 workers", 10, 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
callCount := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
cfg := &config{
endpoint: server.URL,
authBearerToken: "test-token",
timeoutSeconds: 5,
maxRetries: 0,
backoffSeconds: 1,
maxBackoffSeconds: 1,
nWorkers: tt.nWorkers,
bufferSize: 10,
}
q := &Queue{}
err := q.initialize(cfg)
if err != nil {
t.Fatalf("Failed to initialize queue: %v", err)
}
defer func() {
if q.cancel != nil {
q.cancel()
}
time.Sleep(100 * time.Millisecond)
if q.router != nil {
q.router.Close()
}
}()
// Wait for router and subscriber to be fully ready
time.Sleep(200 * time.Millisecond)
msg := &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{
Name: "test.txt",
},
}
err = q.SendMessage("/test/path", msg)
if err != nil {
t.Errorf("SendMessage() error = %v", err)
}
// Wait for message processing
time.Sleep(500 * time.Millisecond)
if callCount != tt.expected {
t.Errorf("Expected %d webhook call(s), got %d with %d workers", tt.expected, callCount, tt.nWorkers)
}
})
}
}
func TestQueueSendMessageWithFilter(t *testing.T) {
tests := []struct {
name string
cfg *config
key string
notification *filer_pb.EventNotification
shouldPublish bool
}{
{
name: "allowed event type",
cfg: &config{
endpoint: "https://example.com/webhook",
timeoutSeconds: 10,
maxRetries: 1,
backoffSeconds: 1,
maxBackoffSeconds: 1,
nWorkers: 1,
bufferSize: 10,
eventTypes: []string{"create"},
},
key: "/data/file.txt",
notification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "file.txt"},
},
shouldPublish: true,
},
{
name: "filtered event type",
cfg: &config{
endpoint: "https://example.com/webhook",
timeoutSeconds: 10,
maxRetries: 1,
backoffSeconds: 1,
maxBackoffSeconds: 1,
nWorkers: 1,
bufferSize: 10,
eventTypes: []string{"update", "rename"},
},
key: "/data/file.txt",
notification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "file.txt"},
},
shouldPublish: false,
},
{
name: "allowed path prefix",
cfg: &config{
endpoint: "https://example.com/webhook",
timeoutSeconds: 10,
maxRetries: 1,
backoffSeconds: 1,
maxBackoffSeconds: 1,
nWorkers: 1,
bufferSize: 10,
pathPrefixes: []string{"/data/"},
},
key: "/data/file.txt",
notification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "file.txt"},
},
shouldPublish: true,
},
{
name: "filtered path prefix",
cfg: &config{
endpoint: "https://example.com/webhook",
timeoutSeconds: 10,
maxRetries: 1,
backoffSeconds: 1,
maxBackoffSeconds: 1,
nWorkers: 1,
bufferSize: 10,
pathPrefixes: []string{"/logs/"},
},
key: "/data/file.txt",
notification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "file.txt"},
},
shouldPublish: false,
},
{
name: "combined filters - both pass",
cfg: &config{
endpoint: "https://example.com/webhook",
timeoutSeconds: 10,
maxRetries: 1,
backoffSeconds: 1,
maxBackoffSeconds: 1,
nWorkers: 1,
bufferSize: 10,
eventTypes: []string{"create", "delete"},
pathPrefixes: []string{"/data/", "/logs/"},
},
key: "/data/file.txt",
notification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "file.txt"},
},
shouldPublish: true,
},
{
name: "combined filters - event fails",
cfg: &config{
endpoint: "https://example.com/webhook",
timeoutSeconds: 10,
maxRetries: 1,
backoffSeconds: 1,
maxBackoffSeconds: 1,
nWorkers: 1,
bufferSize: 10,
eventTypes: []string{"update", "delete"},
pathPrefixes: []string{"/data/", "/logs/"},
},
key: "/data/file.txt",
notification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "file.txt"},
},
shouldPublish: false,
},
{
name: "combined filters - path fails",
cfg: &config{
endpoint: "https://example.com/webhook",
timeoutSeconds: 10,
maxRetries: 1,
backoffSeconds: 1,
maxBackoffSeconds: 1,
nWorkers: 1,
bufferSize: 10,
eventTypes: []string{"create", "delete"},
pathPrefixes: []string{"/logs/"},
},
key: "/data/file.txt",
notification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "file.txt"},
},
shouldPublish: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
shouldPublish := newFilter(tt.cfg).shouldPublish(tt.key, tt.notification)
if shouldPublish != tt.shouldPublish {
t.Errorf("Expected shouldPublish=%v, got %v", tt.shouldPublish, shouldPublish)
}
})
}
}