snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
77
weed/notification/webhook/filter.go
Normal file
77
weed/notification/webhook/filter.go
Normal file
@@ -0,0 +1,77 @@
|
||||
package webhook
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
type filter struct {
|
||||
eventTypes map[eventType]bool
|
||||
pathPrefixes []string
|
||||
}
|
||||
|
||||
func newFilter(cfg *config) *filter {
|
||||
f := &filter{
|
||||
eventTypes: make(map[eventType]bool),
|
||||
pathPrefixes: cfg.pathPrefixes,
|
||||
}
|
||||
|
||||
if len(cfg.eventTypes) == 0 {
|
||||
f.eventTypes[eventTypeCreate] = true
|
||||
f.eventTypes[eventTypeDelete] = true
|
||||
f.eventTypes[eventTypeUpdate] = true
|
||||
f.eventTypes[eventTypeRename] = true
|
||||
} else {
|
||||
for _, et := range cfg.eventTypes {
|
||||
t := eventType(et)
|
||||
if !t.valid() {
|
||||
glog.Warningf("invalid event type: %v", t)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
f.eventTypes[t] = true
|
||||
}
|
||||
}
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *filter) shouldPublish(key string, notification *filer_pb.EventNotification) bool {
|
||||
if !f.matchesPath(key, notification) {
|
||||
return false
|
||||
}
|
||||
|
||||
eventType := detectEventType(key, notification)
|
||||
|
||||
return f.eventTypes[eventType]
|
||||
}
|
||||
|
||||
func (f *filter) matchesPath(key string, notification *filer_pb.EventNotification) bool {
|
||||
if len(f.pathPrefixes) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
if f.matchesAnyPathPrefix(key) {
|
||||
return true
|
||||
}
|
||||
|
||||
if notification != nil && notification.NewEntry != nil && notification.NewParentPath != "" {
|
||||
newKey := string(util.FullPath(notification.NewParentPath).Child(notification.NewEntry.Name))
|
||||
return f.matchesAnyPathPrefix(newKey)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (f *filter) matchesAnyPathPrefix(key string) bool {
|
||||
for _, prefix := range f.pathPrefixes {
|
||||
if strings.HasPrefix(key, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
268
weed/notification/webhook/filter_test.go
Normal file
268
weed/notification/webhook/filter_test.go
Normal file
@@ -0,0 +1,268 @@
|
||||
package webhook
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
func TestFilterEventTypes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
eventTypes []string
|
||||
notification *filer_pb.EventNotification
|
||||
expectedType eventType
|
||||
shouldPublish bool
|
||||
}{
|
||||
{
|
||||
name: "create event - allowed",
|
||||
key: "/test/test.txt",
|
||||
eventTypes: []string{"create", "delete"},
|
||||
notification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "test.txt"},
|
||||
},
|
||||
expectedType: eventTypeCreate,
|
||||
shouldPublish: true,
|
||||
},
|
||||
{
|
||||
name: "create event - not allowed",
|
||||
key: "/test/test.txt",
|
||||
eventTypes: []string{"delete", "update"},
|
||||
notification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "test.txt"},
|
||||
},
|
||||
expectedType: eventTypeCreate,
|
||||
shouldPublish: false,
|
||||
},
|
||||
{
|
||||
name: "delete event - allowed",
|
||||
key: "/test/test.txt",
|
||||
eventTypes: []string{"create", "delete"},
|
||||
notification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{Name: "test.txt"},
|
||||
},
|
||||
expectedType: eventTypeDelete,
|
||||
shouldPublish: true,
|
||||
},
|
||||
{
|
||||
name: "update event - allowed",
|
||||
key: "/test/test.txt",
|
||||
eventTypes: []string{"update"},
|
||||
notification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{Name: "test.txt"},
|
||||
NewEntry: &filer_pb.Entry{Name: "test.txt"},
|
||||
NewParentPath: "/test",
|
||||
},
|
||||
expectedType: eventTypeUpdate,
|
||||
shouldPublish: true,
|
||||
},
|
||||
{
|
||||
name: "rename event - allowed",
|
||||
key: "/old/path/old.txt",
|
||||
eventTypes: []string{"rename"},
|
||||
notification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{Name: "old.txt"},
|
||||
NewEntry: &filer_pb.Entry{Name: "new.txt"},
|
||||
NewParentPath: "/new/path",
|
||||
},
|
||||
expectedType: eventTypeRename,
|
||||
shouldPublish: true,
|
||||
},
|
||||
{
|
||||
name: "rename event same name different parent - allowed",
|
||||
key: "/old/path/file.txt",
|
||||
eventTypes: []string{"rename"},
|
||||
notification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{Name: "file.txt"},
|
||||
NewEntry: &filer_pb.Entry{Name: "file.txt"},
|
||||
NewParentPath: "/new/path",
|
||||
},
|
||||
expectedType: eventTypeRename,
|
||||
shouldPublish: true,
|
||||
},
|
||||
{
|
||||
name: "rename event - not allowed",
|
||||
key: "/old/path/old.txt",
|
||||
eventTypes: []string{"create", "delete", "update"},
|
||||
notification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{Name: "old.txt"},
|
||||
NewEntry: &filer_pb.Entry{Name: "new.txt"},
|
||||
NewParentPath: "/new/path",
|
||||
},
|
||||
expectedType: eventTypeRename,
|
||||
shouldPublish: false,
|
||||
},
|
||||
{
|
||||
name: "all events allowed when empty",
|
||||
key: "/test/test.txt",
|
||||
eventTypes: []string{},
|
||||
notification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "test.txt"},
|
||||
},
|
||||
expectedType: eventTypeCreate,
|
||||
shouldPublish: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := &config{eventTypes: tt.eventTypes}
|
||||
f := newFilter(cfg)
|
||||
|
||||
eventType := detectEventType(tt.key, tt.notification)
|
||||
if eventType != tt.expectedType {
|
||||
t.Errorf("detectEventType() = %v, want %v", eventType, tt.expectedType)
|
||||
}
|
||||
|
||||
shouldPublish := f.shouldPublish(tt.key, tt.notification)
|
||||
if shouldPublish != tt.shouldPublish {
|
||||
t.Errorf("shouldPublish() = %v, want %v", shouldPublish, tt.shouldPublish)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterPathPrefixes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pathPrefixes []string
|
||||
eventTypes []string
|
||||
key string
|
||||
notification *filer_pb.EventNotification
|
||||
shouldPublish bool
|
||||
}{
|
||||
{
|
||||
name: "matches single prefix",
|
||||
pathPrefixes: []string{"/data/"},
|
||||
eventTypes: []string{"create"},
|
||||
key: "/data/file.txt",
|
||||
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "file.txt"}},
|
||||
shouldPublish: true,
|
||||
},
|
||||
{
|
||||
name: "matches one of multiple prefixes",
|
||||
pathPrefixes: []string{"/data/", "/logs/", "/tmp/"},
|
||||
eventTypes: []string{"create"},
|
||||
key: "/logs/app.log",
|
||||
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "app.log"}},
|
||||
shouldPublish: true,
|
||||
},
|
||||
{
|
||||
name: "no match",
|
||||
pathPrefixes: []string{"/data/", "/logs/"},
|
||||
eventTypes: []string{"create"},
|
||||
key: "/other/file.txt",
|
||||
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "file.txt"}},
|
||||
shouldPublish: false,
|
||||
},
|
||||
{
|
||||
name: "empty prefixes allows all",
|
||||
pathPrefixes: []string{},
|
||||
eventTypes: []string{"create"},
|
||||
key: "/any/path/file.txt",
|
||||
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "file.txt"}},
|
||||
shouldPublish: true,
|
||||
},
|
||||
{
|
||||
name: "exact prefix match",
|
||||
pathPrefixes: []string{"/data"},
|
||||
eventTypes: []string{"create"},
|
||||
key: "/data",
|
||||
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "data"}},
|
||||
shouldPublish: true,
|
||||
},
|
||||
{
|
||||
name: "partial match not allowed",
|
||||
pathPrefixes: []string{"/data/"},
|
||||
eventTypes: []string{"create"},
|
||||
key: "/database/file.txt",
|
||||
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "file.txt"}},
|
||||
shouldPublish: false,
|
||||
},
|
||||
{
|
||||
name: "rename matches destination prefix",
|
||||
pathPrefixes: []string{"/watched/"},
|
||||
eventTypes: []string{"rename"},
|
||||
key: "/outside/old.txt",
|
||||
notification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{Name: "old.txt"},
|
||||
NewEntry: &filer_pb.Entry{Name: "new.txt"},
|
||||
NewParentPath: "/watched",
|
||||
},
|
||||
shouldPublish: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := &config{
|
||||
pathPrefixes: tt.pathPrefixes,
|
||||
eventTypes: tt.eventTypes,
|
||||
}
|
||||
f := newFilter(cfg)
|
||||
|
||||
shouldPublish := f.shouldPublish(tt.key, tt.notification)
|
||||
if shouldPublish != tt.shouldPublish {
|
||||
t.Errorf("shouldPublish() = %v, want %v", shouldPublish, tt.shouldPublish)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterCombined(t *testing.T) {
|
||||
cfg := &config{
|
||||
eventTypes: []string{"create", "update"},
|
||||
pathPrefixes: []string{"/data/", "/logs/"},
|
||||
}
|
||||
f := newFilter(cfg)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
notification *filer_pb.EventNotification
|
||||
shouldPublish bool
|
||||
}{
|
||||
{
|
||||
name: "allowed event and path",
|
||||
key: "/data/file.txt",
|
||||
notification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "file.txt"},
|
||||
},
|
||||
shouldPublish: true,
|
||||
},
|
||||
{
|
||||
name: "allowed event but wrong path",
|
||||
key: "/other/file.txt",
|
||||
notification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "file.txt"},
|
||||
},
|
||||
shouldPublish: false,
|
||||
},
|
||||
{
|
||||
name: "wrong event but allowed path",
|
||||
key: "/data/file.txt",
|
||||
notification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{Name: "file.txt"},
|
||||
},
|
||||
shouldPublish: false,
|
||||
},
|
||||
{
|
||||
name: "wrong event and wrong path",
|
||||
key: "/other/file.txt",
|
||||
notification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{Name: "file.txt"},
|
||||
},
|
||||
shouldPublish: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
shouldPublish := f.shouldPublish(tt.key, tt.notification)
|
||||
if shouldPublish != tt.shouldPublish {
|
||||
t.Errorf("shouldPublish() = %v, want %v", shouldPublish, tt.shouldPublish)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
184
weed/notification/webhook/http.go
Normal file
184
weed/notification/webhook/http.go
Normal file
@@ -0,0 +1,184 @@
|
||||
package webhook
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
)
|
||||
|
||||
const (
|
||||
maxWebhookRetryDepth = 2
|
||||
)
|
||||
|
||||
type httpClient struct {
|
||||
endpoint string
|
||||
token string
|
||||
timeout time.Duration
|
||||
client *http.Client // Reused HTTP client with redirect prevention
|
||||
endpointMu sync.RWMutex
|
||||
finalURL string // Cached final URL after following redirects
|
||||
}
|
||||
|
||||
func newHTTPClient(cfg *config) (*httpClient, error) {
|
||||
return &httpClient{
|
||||
endpoint: cfg.endpoint,
|
||||
token: cfg.authBearerToken,
|
||||
timeout: time.Duration(cfg.timeoutSeconds) * time.Second,
|
||||
client: &http.Client{
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (h *httpClient) sendMessage(message *webhookMessage) error {
|
||||
return h.sendMessageWithRetry(message, 0)
|
||||
}
|
||||
|
||||
func (h *httpClient) sendMessageWithRetry(message *webhookMessage, depth int) error {
|
||||
// Prevent infinite recursion
|
||||
if depth > maxWebhookRetryDepth {
|
||||
return fmt.Errorf("webhook max retry depth exceeded")
|
||||
}
|
||||
|
||||
// Serialize the protobuf message to JSON for HTTP payload
|
||||
notificationData, err := json.Marshal(message.Notification)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal notification: %w", err)
|
||||
}
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"key": message.Key,
|
||||
"event_type": message.EventType,
|
||||
"message": json.RawMessage(notificationData),
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal message: %w", err)
|
||||
}
|
||||
|
||||
// Use cached final URL if available, otherwise use original endpoint
|
||||
h.endpointMu.RLock()
|
||||
targetURL := h.finalURL
|
||||
usingCachedURL := targetURL != ""
|
||||
if targetURL == "" {
|
||||
targetURL = h.endpoint
|
||||
}
|
||||
h.endpointMu.RUnlock()
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, targetURL, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if h.token != "" {
|
||||
req.Header.Set("Authorization", security.BearerPrefix+h.token)
|
||||
}
|
||||
|
||||
// Apply timeout via context (not on client) to avoid redundancy
|
||||
if h.timeout > 0 {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), h.timeout)
|
||||
defer cancel()
|
||||
req = req.WithContext(ctx)
|
||||
}
|
||||
|
||||
resp, err := h.client.Do(req)
|
||||
if err != nil {
|
||||
if drainErr := drainResponse(resp); drainErr != nil {
|
||||
glog.Errorf("failed to drain response: %v", drainErr)
|
||||
}
|
||||
|
||||
// If using cached URL and request failed, clear cache and retry with original endpoint
|
||||
if usingCachedURL && depth == 0 {
|
||||
glog.V(1).Infof("Webhook request to cached URL %s failed, clearing cache and retrying with original endpoint", targetURL)
|
||||
h.setFinalURL("")
|
||||
return h.sendMessageWithRetry(message, depth+1)
|
||||
}
|
||||
|
||||
return fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
|
||||
// Handle redirects by caching the final destination and recreating POST request
|
||||
if resp.StatusCode >= 300 && resp.StatusCode < 400 {
|
||||
// Drain and close response body to enable connection reuse
|
||||
util_http.CloseResponse(resp)
|
||||
|
||||
location := resp.Header.Get("Location")
|
||||
if location == "" {
|
||||
return fmt.Errorf("webhook returned redirect status %d without Location header", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Resolve relative URLs against the request URL
|
||||
reqURL := req.URL
|
||||
finalURL, err := reqURL.Parse(location)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse redirect location: %w", err)
|
||||
}
|
||||
|
||||
// Update finalURL to follow the redirect for this attempt
|
||||
finalURLStr := finalURL.String()
|
||||
h.setFinalURL(finalURLStr)
|
||||
|
||||
if depth == 0 {
|
||||
glog.V(1).Infof("Webhook endpoint redirected from %s to %s, caching final destination", targetURL, finalURLStr)
|
||||
} else {
|
||||
glog.V(1).Infof("Webhook endpoint redirected from %s to %s (following redirect on retry)", targetURL, finalURLStr)
|
||||
}
|
||||
|
||||
// Recreate the POST request to the final destination (increment depth to prevent infinite loops)
|
||||
return h.sendMessageWithRetry(message, depth+1)
|
||||
}
|
||||
|
||||
// If using cached URL and got an error response, clear cache and retry with original endpoint
|
||||
if resp.StatusCode >= 400 && usingCachedURL && depth == 0 {
|
||||
// Drain and close response body to enable connection reuse
|
||||
util_http.CloseResponse(resp)
|
||||
|
||||
glog.V(1).Infof("Webhook request to cached URL %s returned error %d, clearing cache and retrying with original endpoint", targetURL, resp.StatusCode)
|
||||
h.setFinalURL("")
|
||||
return h.sendMessageWithRetry(message, depth+1)
|
||||
}
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
// Drain and close response body before returning error to enable connection reuse
|
||||
util_http.CloseResponse(resp)
|
||||
return fmt.Errorf("webhook returned status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Drain and close response body on success to enable connection reuse
|
||||
util_http.CloseResponse(resp)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *httpClient) setFinalURL(url string) {
|
||||
h.endpointMu.Lock()
|
||||
h.finalURL = url
|
||||
h.endpointMu.Unlock()
|
||||
}
|
||||
|
||||
func drainResponse(resp *http.Response) error {
|
||||
if resp == nil || resp.Body == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err := io.ReadAll(resp.Body)
|
||||
|
||||
return errors.Join(
|
||||
err,
|
||||
resp.Body.Close(),
|
||||
)
|
||||
}
|
||||
522
weed/notification/webhook/http_test.go
Normal file
522
weed/notification/webhook/http_test.go
Normal file
@@ -0,0 +1,522 @@
|
||||
package webhook
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
)
|
||||
|
||||
func init() {
|
||||
util_http.InitGlobalHttpClient()
|
||||
}
|
||||
|
||||
func TestHttpClientSendMessage(t *testing.T) {
|
||||
var receivedPayload map[string]interface{}
|
||||
var receivedHeaders http.Header
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
receivedHeaders = r.Header
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if err := json.Unmarshal(body, &receivedPayload); err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
cfg := &config{
|
||||
endpoint: server.URL,
|
||||
authBearerToken: "test-token",
|
||||
}
|
||||
|
||||
client, err := newHTTPClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create HTTP client: %v", err)
|
||||
}
|
||||
|
||||
message := &filer_pb.EventNotification{
|
||||
OldEntry: nil,
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
IsDirectory: false,
|
||||
},
|
||||
}
|
||||
|
||||
err = client.sendMessage(newWebhookMessage("/test/path", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send message: %v", err)
|
||||
}
|
||||
|
||||
if receivedPayload["key"] != "/test/path" {
|
||||
t.Errorf("Expected key '/test/path', got %v", receivedPayload["key"])
|
||||
}
|
||||
|
||||
if receivedPayload["event_type"] != "create" {
|
||||
t.Errorf("Expected event_type 'create', got %v", receivedPayload["event_type"])
|
||||
}
|
||||
|
||||
if receivedPayload["message"] == nil {
|
||||
t.Error("Expected message to be present")
|
||||
}
|
||||
|
||||
if receivedHeaders.Get("Content-Type") != "application/json" {
|
||||
t.Errorf("Expected Content-Type 'application/json', got %s", receivedHeaders.Get("Content-Type"))
|
||||
}
|
||||
|
||||
expectedAuth := "Bearer test-token"
|
||||
if receivedHeaders.Get("Authorization") != expectedAuth {
|
||||
t.Errorf("Expected Authorization '%s', got %s", expectedAuth, receivedHeaders.Get("Authorization"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHttpClientSendMessageWithoutToken(t *testing.T) {
|
||||
var receivedHeaders http.Header
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
receivedHeaders = r.Header
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
cfg := &config{
|
||||
endpoint: server.URL,
|
||||
authBearerToken: "",
|
||||
}
|
||||
|
||||
client, err := newHTTPClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create HTTP client: %v", err)
|
||||
}
|
||||
|
||||
message := &filer_pb.EventNotification{}
|
||||
|
||||
err = client.sendMessage(newWebhookMessage("/test/path", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send message: %v", err)
|
||||
}
|
||||
|
||||
if receivedHeaders.Get("Authorization") != "" {
|
||||
t.Errorf("Expected no Authorization header, got %s", receivedHeaders.Get("Authorization"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHttpClientSendMessageServerError(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
cfg := &config{
|
||||
endpoint: server.URL,
|
||||
authBearerToken: "test-token",
|
||||
}
|
||||
|
||||
client, err := newHTTPClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create HTTP client: %v", err)
|
||||
}
|
||||
|
||||
message := &filer_pb.EventNotification{}
|
||||
|
||||
err = client.sendMessage(newWebhookMessage("/test/path", message))
|
||||
if err == nil {
|
||||
t.Error("Expected error for server error response")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHttpClientSendMessageNetworkError(t *testing.T) {
|
||||
cfg := &config{
|
||||
endpoint: "http://localhost:99999",
|
||||
authBearerToken: "",
|
||||
}
|
||||
|
||||
client, err := newHTTPClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create HTTP client: %v", err)
|
||||
}
|
||||
|
||||
message := &filer_pb.EventNotification{}
|
||||
|
||||
err = client.sendMessage(newWebhookMessage("/test/path", message))
|
||||
if err == nil {
|
||||
t.Error("Expected error for network failure")
|
||||
}
|
||||
}
|
||||
|
||||
// TestHttpClientFollowsRedirectAsPost verifies that redirects are followed with POST method preserved
|
||||
func TestHttpClientFollowsRedirectAsPost(t *testing.T) {
|
||||
redirectCalled := false
|
||||
finalCalled := false
|
||||
var finalMethod string
|
||||
var finalBody map[string]interface{}
|
||||
|
||||
// Create final destination server
|
||||
finalServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
finalCalled = true
|
||||
finalMethod = r.Method
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
json.Unmarshal(body, &finalBody)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer finalServer.Close()
|
||||
|
||||
// Create redirect server
|
||||
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
redirectCalled = true
|
||||
// Return 301 redirect to final server
|
||||
http.Redirect(w, r, finalServer.URL, http.StatusMovedPermanently)
|
||||
}))
|
||||
defer redirectServer.Close()
|
||||
|
||||
cfg := &config{
|
||||
endpoint: redirectServer.URL,
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 5,
|
||||
}
|
||||
|
||||
client, err := newHTTPClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create HTTP client: %v", err)
|
||||
}
|
||||
|
||||
message := &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
},
|
||||
}
|
||||
|
||||
// Send message - should follow redirect and recreate POST request
|
||||
err = client.sendMessage(newWebhookMessage("/test/path", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send message: %v", err)
|
||||
}
|
||||
|
||||
if !redirectCalled {
|
||||
t.Error("Expected redirect server to be called")
|
||||
}
|
||||
|
||||
if !finalCalled {
|
||||
t.Error("Expected final server to be called after redirect")
|
||||
}
|
||||
|
||||
if finalMethod != "POST" {
|
||||
t.Errorf("Expected POST method at final destination, got %s", finalMethod)
|
||||
}
|
||||
|
||||
if finalBody["key"] != "/test/path" {
|
||||
t.Errorf("Expected key '/test/path' at final destination, got %v", finalBody["key"])
|
||||
}
|
||||
|
||||
// Verify the final URL is cached
|
||||
client.endpointMu.RLock()
|
||||
cachedURL := client.finalURL
|
||||
client.endpointMu.RUnlock()
|
||||
|
||||
if cachedURL != finalServer.URL {
|
||||
t.Errorf("Expected cached URL %s, got %s", finalServer.URL, cachedURL)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHttpClientUsesCachedRedirect verifies that subsequent requests use the cached redirect destination
|
||||
func TestHttpClientUsesCachedRedirect(t *testing.T) {
|
||||
redirectCount := 0
|
||||
finalCount := 0
|
||||
|
||||
// Create final destination server
|
||||
finalServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
finalCount++
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer finalServer.Close()
|
||||
|
||||
// Create redirect server
|
||||
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
redirectCount++
|
||||
http.Redirect(w, r, finalServer.URL, http.StatusMovedPermanently)
|
||||
}))
|
||||
defer redirectServer.Close()
|
||||
|
||||
cfg := &config{
|
||||
endpoint: redirectServer.URL,
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 5,
|
||||
}
|
||||
|
||||
client, err := newHTTPClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create HTTP client: %v", err)
|
||||
}
|
||||
|
||||
message := &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
},
|
||||
}
|
||||
|
||||
// First request - should hit redirect server
|
||||
err = client.sendMessage(newWebhookMessage("/test/path1", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send first message: %v", err)
|
||||
}
|
||||
|
||||
if redirectCount != 1 {
|
||||
t.Errorf("Expected 1 redirect call, got %d", redirectCount)
|
||||
}
|
||||
if finalCount != 1 {
|
||||
t.Errorf("Expected 1 final call, got %d", finalCount)
|
||||
}
|
||||
|
||||
// Second request - should use cached URL and skip redirect server
|
||||
err = client.sendMessage(newWebhookMessage("/test/path2", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send second message: %v", err)
|
||||
}
|
||||
|
||||
if redirectCount != 1 {
|
||||
t.Errorf("Expected redirect server to be called only once (cached), got %d calls", redirectCount)
|
||||
}
|
||||
if finalCount != 2 {
|
||||
t.Errorf("Expected 2 final calls, got %d", finalCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHttpClientPreservesPostMethod verifies POST method is preserved and not converted to GET
|
||||
func TestHttpClientPreservesPostMethod(t *testing.T) {
|
||||
var receivedMethod string
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
receivedMethod = r.Method
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
cfg := &config{
|
||||
endpoint: server.URL,
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 5,
|
||||
}
|
||||
|
||||
client, err := newHTTPClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create HTTP client: %v", err)
|
||||
}
|
||||
|
||||
message := &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
},
|
||||
}
|
||||
|
||||
err = client.sendMessage(newWebhookMessage("/test/path", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send message: %v", err)
|
||||
}
|
||||
|
||||
if receivedMethod != "POST" {
|
||||
t.Errorf("Expected POST method, got %s", receivedMethod)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHttpClientInvalidatesCacheOnError verifies that cache is invalidated when cached URL fails
|
||||
func TestHttpClientInvalidatesCacheOnError(t *testing.T) {
|
||||
finalServerDown := false // Start with server UP
|
||||
originalCallCount := 0
|
||||
finalCallCount := 0
|
||||
|
||||
// Create final destination server that can be toggled
|
||||
finalServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
finalCallCount++
|
||||
if finalServerDown {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
} else {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
}))
|
||||
defer finalServer.Close()
|
||||
|
||||
// Create redirect server
|
||||
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
originalCallCount++
|
||||
http.Redirect(w, r, finalServer.URL, http.StatusMovedPermanently)
|
||||
}))
|
||||
defer redirectServer.Close()
|
||||
|
||||
cfg := &config{
|
||||
endpoint: redirectServer.URL,
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 5,
|
||||
}
|
||||
|
||||
client, err := newHTTPClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create HTTP client: %v", err)
|
||||
}
|
||||
|
||||
message := &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
},
|
||||
}
|
||||
|
||||
// First request - should follow redirect and cache the final URL
|
||||
err = client.sendMessage(newWebhookMessage("/test/path1", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send first message: %v", err)
|
||||
}
|
||||
|
||||
if originalCallCount != 1 {
|
||||
t.Errorf("Expected 1 original call, got %d", originalCallCount)
|
||||
}
|
||||
if finalCallCount != 1 {
|
||||
t.Errorf("Expected 1 final call, got %d", finalCallCount)
|
||||
}
|
||||
|
||||
// Verify cache was set
|
||||
client.endpointMu.RLock()
|
||||
cachedURL := client.finalURL
|
||||
client.endpointMu.RUnlock()
|
||||
if cachedURL != finalServer.URL {
|
||||
t.Errorf("Expected cached URL %s, got %s", finalServer.URL, cachedURL)
|
||||
}
|
||||
|
||||
// Second request with cached URL working - should use cache
|
||||
err = client.sendMessage(newWebhookMessage("/test/path2", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send second message: %v", err)
|
||||
}
|
||||
|
||||
if originalCallCount != 1 {
|
||||
t.Errorf("Expected still 1 original call (using cache), got %d", originalCallCount)
|
||||
}
|
||||
if finalCallCount != 2 {
|
||||
t.Errorf("Expected 2 final calls, got %d", finalCallCount)
|
||||
}
|
||||
|
||||
// Third request - bring final server DOWN, should invalidate cache and retry with original
|
||||
// Flow: cached URL (fail, depth=0) -> clear cache -> retry original (depth=1) -> redirect -> final (fail, depth=2)
|
||||
finalServerDown = true
|
||||
err = client.sendMessage(newWebhookMessage("/test/path3", message))
|
||||
if err == nil {
|
||||
t.Error("Expected error when cached URL fails and retry also fails")
|
||||
}
|
||||
|
||||
// originalCallCount: 1 (initial) + 1 (retry after cache invalidation) = 2
|
||||
if originalCallCount != 2 {
|
||||
t.Errorf("Expected 2 original calls, got %d", originalCallCount)
|
||||
}
|
||||
// finalCallCount: 2 (previous) + 1 (cached fail) + 1 (retry after redirect) = 4
|
||||
if finalCallCount != 4 {
|
||||
t.Errorf("Expected 4 final calls, got %d", finalCallCount)
|
||||
}
|
||||
|
||||
// Verify final URL is still set (to the failed destination from the redirect)
|
||||
client.endpointMu.RLock()
|
||||
finalURLAfterError := client.finalURL
|
||||
client.endpointMu.RUnlock()
|
||||
if finalURLAfterError != finalServer.URL {
|
||||
t.Errorf("Expected finalURL to be %s after error, got %s", finalServer.URL, finalURLAfterError)
|
||||
}
|
||||
|
||||
// Fourth request - bring final server back UP
|
||||
// Since cache still has the final URL, it should use it directly
|
||||
finalServerDown = false
|
||||
err = client.sendMessage(newWebhookMessage("/test/path4", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send fourth message after recovery: %v", err)
|
||||
}
|
||||
|
||||
// Should have used the cached URL directly (no new original call)
|
||||
// originalCallCount: still 2
|
||||
if originalCallCount != 2 {
|
||||
t.Errorf("Expected 2 original calls (using cache), got %d", originalCallCount)
|
||||
}
|
||||
// finalCallCount: 4 + 1 = 5
|
||||
if finalCallCount != 5 {
|
||||
t.Errorf("Expected 5 final calls, got %d", finalCallCount)
|
||||
}
|
||||
|
||||
// Verify cache was re-established
|
||||
client.endpointMu.RLock()
|
||||
reestablishedCache := client.finalURL
|
||||
client.endpointMu.RUnlock()
|
||||
if reestablishedCache != finalServer.URL {
|
||||
t.Errorf("Expected cache to be re-established to %s, got %s", finalServer.URL, reestablishedCache)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHttpClientInvalidatesCacheOnNetworkError verifies cache invalidation on network errors
|
||||
func TestHttpClientInvalidatesCacheOnNetworkError(t *testing.T) {
|
||||
originalCallCount := 0
|
||||
var finalServer *httptest.Server
|
||||
// Create redirect server
|
||||
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
originalCallCount++
|
||||
if finalServer != nil {
|
||||
http.Redirect(w, r, finalServer.URL, http.StatusMovedPermanently)
|
||||
} else {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
}))
|
||||
defer redirectServer.Close()
|
||||
|
||||
// Create final destination server
|
||||
finalServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
cfg := &config{
|
||||
endpoint: redirectServer.URL,
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 5,
|
||||
}
|
||||
|
||||
client, err := newHTTPClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create HTTP client: %v", err)
|
||||
}
|
||||
|
||||
message := &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
},
|
||||
}
|
||||
|
||||
// First request - establish cache
|
||||
err = client.sendMessage(newWebhookMessage("/test/path1", message))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send first message: %v", err)
|
||||
}
|
||||
|
||||
if originalCallCount != 1 {
|
||||
t.Errorf("Expected 1 original call, got %d", originalCallCount)
|
||||
}
|
||||
|
||||
// Close final server to simulate network error
|
||||
cachedURL := finalServer.URL
|
||||
finalServer.Close()
|
||||
finalServer = nil
|
||||
|
||||
// Second request - cached URL is down, should invalidate and retry with original
|
||||
err = client.sendMessage(newWebhookMessage("/test/path2", message))
|
||||
if err == nil {
|
||||
t.Error("Expected error when network fails")
|
||||
}
|
||||
|
||||
if originalCallCount != 2 {
|
||||
t.Errorf("Expected 2 original calls (retry after cache invalidation), got %d", originalCallCount)
|
||||
}
|
||||
|
||||
// Verify cache was cleared
|
||||
client.endpointMu.RLock()
|
||||
clearedCache := client.finalURL
|
||||
client.endpointMu.RUnlock()
|
||||
if clearedCache == cachedURL {
|
||||
t.Errorf("Expected cache to be invalidated, but still has %s", clearedCache)
|
||||
}
|
||||
}
|
||||
186
weed/notification/webhook/types.go
Normal file
186
weed/notification/webhook/types.go
Normal file
@@ -0,0 +1,186 @@
|
||||
package webhook
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
const (
|
||||
queueName = "webhook"
|
||||
pubSubTopicName = "webhook_topic"
|
||||
deadLetterTopic = "webhook_dead_letter"
|
||||
)
|
||||
|
||||
type eventType string
|
||||
|
||||
const (
|
||||
eventTypeCreate eventType = "create"
|
||||
eventTypeDelete eventType = "delete"
|
||||
eventTypeUpdate eventType = "update"
|
||||
eventTypeRename eventType = "rename"
|
||||
)
|
||||
|
||||
func (e eventType) valid() bool {
|
||||
return slices.Contains([]eventType{
|
||||
eventTypeCreate,
|
||||
eventTypeDelete,
|
||||
eventTypeUpdate,
|
||||
eventTypeRename,
|
||||
},
|
||||
e,
|
||||
)
|
||||
}
|
||||
|
||||
var (
|
||||
pubSubHandlerNameTemplate = func(n int) string {
|
||||
return "webhook_handler_" + strconv.Itoa(n)
|
||||
}
|
||||
)
|
||||
|
||||
type client interface {
|
||||
sendMessage(message *webhookMessage) error
|
||||
}
|
||||
|
||||
type webhookMessage struct {
|
||||
Key string `json:"key"`
|
||||
EventType string `json:"event_type"`
|
||||
Notification *filer_pb.EventNotification `json:"message_data"`
|
||||
}
|
||||
|
||||
func newWebhookMessage(key string, message proto.Message) *webhookMessage {
|
||||
notification, ok := message.(*filer_pb.EventNotification)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
eventType := string(detectEventType(key, notification))
|
||||
|
||||
return &webhookMessage{
|
||||
Key: key,
|
||||
EventType: eventType,
|
||||
Notification: notification,
|
||||
}
|
||||
}
|
||||
|
||||
type config struct {
|
||||
endpoint string
|
||||
authBearerToken string
|
||||
timeoutSeconds int
|
||||
|
||||
maxRetries int
|
||||
backoffSeconds int
|
||||
maxBackoffSeconds int
|
||||
nWorkers int
|
||||
bufferSize int
|
||||
|
||||
eventTypes []string
|
||||
pathPrefixes []string
|
||||
}
|
||||
|
||||
func newConfigWithDefaults(configuration util.Configuration, prefix string) *config {
|
||||
c := &config{
|
||||
endpoint: configuration.GetString(prefix + "endpoint"),
|
||||
authBearerToken: configuration.GetString(prefix + "bearer_token"),
|
||||
timeoutSeconds: 10,
|
||||
maxRetries: 3,
|
||||
backoffSeconds: 3,
|
||||
maxBackoffSeconds: 30,
|
||||
nWorkers: 5,
|
||||
bufferSize: 10_000,
|
||||
}
|
||||
|
||||
if bufferSize := configuration.GetInt(prefix + "buffer_size"); bufferSize > 0 {
|
||||
c.bufferSize = bufferSize
|
||||
}
|
||||
if workers := configuration.GetInt(prefix + "workers"); workers > 0 {
|
||||
c.nWorkers = workers
|
||||
}
|
||||
if maxRetries := configuration.GetInt(prefix + "max_retries"); maxRetries > 0 {
|
||||
c.maxRetries = maxRetries
|
||||
}
|
||||
if backoffSeconds := configuration.GetInt(prefix + "backoff_seconds"); backoffSeconds > 0 {
|
||||
c.backoffSeconds = backoffSeconds
|
||||
}
|
||||
if maxBackoffSeconds := configuration.GetInt(prefix + "max_backoff_seconds"); maxBackoffSeconds > 0 {
|
||||
c.maxBackoffSeconds = maxBackoffSeconds
|
||||
}
|
||||
if timeout := configuration.GetInt(prefix + "timeout_seconds"); timeout > 0 {
|
||||
c.timeoutSeconds = timeout
|
||||
}
|
||||
|
||||
c.eventTypes = configuration.GetStringSlice(prefix + "event_types")
|
||||
c.pathPrefixes = configuration.GetStringSlice(prefix + "path_prefixes")
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *config) validate() error {
|
||||
if c.endpoint == "" {
|
||||
return fmt.Errorf("webhook endpoint is required")
|
||||
}
|
||||
|
||||
_, err := url.Parse(c.endpoint)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid webhook endpoint: %w", err)
|
||||
}
|
||||
|
||||
if c.timeoutSeconds < 1 || c.timeoutSeconds > 300 {
|
||||
return fmt.Errorf("timeout must be between 1 and 300 seconds, got %d", c.timeoutSeconds)
|
||||
}
|
||||
|
||||
if c.maxRetries < 0 || c.maxRetries > 10 {
|
||||
return fmt.Errorf("max retries must be between 0 and 10, got %d", c.maxRetries)
|
||||
}
|
||||
|
||||
if c.backoffSeconds < 1 || c.backoffSeconds > 60 {
|
||||
return fmt.Errorf("backoff seconds must be between 1 and 60, got %d", c.backoffSeconds)
|
||||
}
|
||||
|
||||
if c.maxBackoffSeconds < c.backoffSeconds || c.maxBackoffSeconds > 300 {
|
||||
return fmt.Errorf("max backoff seconds must be between %d and 300, got %d", c.backoffSeconds, c.maxBackoffSeconds)
|
||||
}
|
||||
|
||||
if c.nWorkers < 1 || c.nWorkers > 100 {
|
||||
return fmt.Errorf("workers must be between 1 and 100, got %d", c.nWorkers)
|
||||
}
|
||||
|
||||
if c.bufferSize < 100 || c.bufferSize > 1_000_000 {
|
||||
return fmt.Errorf("buffer size must be between 100 and 1,000,000, got %d", c.bufferSize)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func detectEventType(key string, notification *filer_pb.EventNotification) eventType {
|
||||
hasOldEntry := notification.OldEntry != nil
|
||||
hasNewEntry := notification.NewEntry != nil
|
||||
|
||||
if !hasOldEntry && hasNewEntry {
|
||||
return eventTypeCreate
|
||||
}
|
||||
|
||||
if hasOldEntry && !hasNewEntry {
|
||||
return eventTypeDelete
|
||||
}
|
||||
|
||||
if hasOldEntry && hasNewEntry {
|
||||
oldDir, _ := util.FullPath(key).DirAndName()
|
||||
newDir := notification.NewParentPath
|
||||
if newDir == "" {
|
||||
newDir = oldDir
|
||||
}
|
||||
if oldDir != newDir || notification.OldEntry.Name != notification.NewEntry.Name {
|
||||
return eventTypeRename
|
||||
}
|
||||
|
||||
return eventTypeUpdate
|
||||
}
|
||||
|
||||
return eventTypeUpdate
|
||||
}
|
||||
243
weed/notification/webhook/webhook_queue.go
Normal file
243
weed/notification/webhook/webhook_queue.go
Normal file
@@ -0,0 +1,243 @@
|
||||
package webhook
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/ThreeDotsLabs/watermill"
|
||||
"github.com/ThreeDotsLabs/watermill/message"
|
||||
"github.com/ThreeDotsLabs/watermill/message/router/middleware"
|
||||
"github.com/ThreeDotsLabs/watermill/message/router/plugin"
|
||||
"github.com/ThreeDotsLabs/watermill/pubsub/gochannel"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/notification"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
func init() {
|
||||
notification.MessageQueues = append(notification.MessageQueues, &Queue{})
|
||||
}
|
||||
|
||||
type Queue struct {
|
||||
router *message.Router
|
||||
queueChannel *gochannel.GoChannel
|
||||
config *config
|
||||
client client
|
||||
filter *filter
|
||||
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
|
||||
// Semaphore for controlling concurrent webhook requests
|
||||
sem chan struct{}
|
||||
}
|
||||
|
||||
func (w *Queue) GetName() string {
|
||||
return queueName
|
||||
}
|
||||
|
||||
func (w *Queue) SendMessage(key string, msg proto.Message) error {
|
||||
eventNotification, ok := msg.(*filer_pb.EventNotification)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
if w.filter != nil && !w.filter.shouldPublish(key, eventNotification) {
|
||||
return nil
|
||||
}
|
||||
|
||||
m := newWebhookMessage(key, msg)
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
wMsg, err := m.toWaterMillMessage()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return w.queueChannel.Publish(pubSubTopicName, wMsg)
|
||||
}
|
||||
|
||||
func (w *webhookMessage) toWaterMillMessage() (*message.Message, error) {
|
||||
payload, err := proto.Marshal(w.Notification)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
msg := message.NewMessage(watermill.NewUUID(), payload)
|
||||
// Set event type and key as metadata
|
||||
msg.Metadata.Set("event_type", w.EventType)
|
||||
msg.Metadata.Set("key", w.Key)
|
||||
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
func (w *Queue) Initialize(configuration util.Configuration, prefix string) error {
|
||||
c := newConfigWithDefaults(configuration, prefix)
|
||||
|
||||
if err := c.validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return w.initialize(c)
|
||||
}
|
||||
|
||||
func (w *Queue) initialize(cfg *config) error {
|
||||
w.ctx, w.cancel = context.WithCancel(context.Background())
|
||||
w.config = cfg
|
||||
w.filter = newFilter(cfg)
|
||||
|
||||
// Initialize semaphore for controlling concurrent webhook requests
|
||||
w.sem = make(chan struct{}, cfg.nWorkers)
|
||||
|
||||
hClient, err := newHTTPClient(cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create webhook http client: %w", err)
|
||||
}
|
||||
w.client = hClient
|
||||
|
||||
if err = w.setupWatermillQueue(cfg); err != nil {
|
||||
return fmt.Errorf("failed to setup watermill queue: %w", err)
|
||||
}
|
||||
if err = w.logDeadLetterMessages(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Queue) setupWatermillQueue(cfg *config) error {
|
||||
logger := watermill.NewStdLogger(false, false)
|
||||
pubSubConfig := gochannel.Config{
|
||||
OutputChannelBuffer: int64(cfg.bufferSize),
|
||||
Persistent: false,
|
||||
}
|
||||
w.queueChannel = gochannel.NewGoChannel(pubSubConfig, logger)
|
||||
|
||||
router, err := message.NewRouter(
|
||||
message.RouterConfig{
|
||||
CloseTimeout: 60 * time.Second,
|
||||
},
|
||||
logger,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create router: %w", err)
|
||||
}
|
||||
w.router = router
|
||||
|
||||
retryMiddleware := middleware.Retry{
|
||||
MaxRetries: cfg.maxRetries,
|
||||
InitialInterval: time.Duration(cfg.backoffSeconds) * time.Second,
|
||||
MaxInterval: time.Duration(cfg.maxBackoffSeconds) * time.Second,
|
||||
Multiplier: 2.0,
|
||||
RandomizationFactor: 0.3,
|
||||
Logger: logger,
|
||||
}.Middleware
|
||||
|
||||
poisonQueue, err := middleware.PoisonQueue(w.queueChannel, deadLetterTopic)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create poison queue: %w", err)
|
||||
}
|
||||
|
||||
router.AddPlugin(plugin.SignalsHandler)
|
||||
router.AddMiddleware(retryMiddleware, poisonQueue)
|
||||
|
||||
// Add a single handler to avoid duplicate message delivery.
|
||||
// With gochannel's default behavior, each handler call creates
|
||||
// a separate subscription, and all subscriptions receive their own copy of each message.
|
||||
// Using a single handler ensures each webhook is sent only once.
|
||||
// Concurrency is controlled via semaphore in handleWebhook based on nWorkers config.
|
||||
router.AddConsumerHandler(
|
||||
"webhook_handler",
|
||||
pubSubTopicName,
|
||||
w.queueChannel,
|
||||
w.handleWebhook,
|
||||
)
|
||||
|
||||
go func() {
|
||||
// cancels the queue context so the dead letter logger exists in case context not canceled by the shutdown signal already
|
||||
defer w.cancel()
|
||||
|
||||
if err := router.Run(w.ctx); err != nil && !errors.Is(err, context.Canceled) {
|
||||
glog.Errorf("webhook pubsub worker stopped with error: %v", err)
|
||||
}
|
||||
|
||||
glog.Info("webhook pubsub worker stopped")
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Queue) handleWebhook(msg *message.Message) error {
|
||||
// Acquire semaphore slot (blocks if at capacity)
|
||||
w.sem <- struct{}{}
|
||||
defer func() { <-w.sem }()
|
||||
|
||||
var n filer_pb.EventNotification
|
||||
if err := proto.Unmarshal(msg.Payload, &n); err != nil {
|
||||
glog.Errorf("failed to unmarshal protobuf message: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Reconstruct webhook message from metadata and payload
|
||||
webhookMsg := &webhookMessage{
|
||||
Key: msg.Metadata.Get("key"),
|
||||
EventType: msg.Metadata.Get("event_type"),
|
||||
Notification: &n,
|
||||
}
|
||||
|
||||
if err := w.client.sendMessage(webhookMsg); err != nil {
|
||||
glog.Errorf("failed to send message to webhook %s: %v", webhookMsg.Key, err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Queue) logDeadLetterMessages() error {
|
||||
ch, err := w.queueChannel.Subscribe(w.ctx, deadLetterTopic)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case msg, ok := <-ch:
|
||||
if !ok {
|
||||
glog.Info("dead letter channel closed")
|
||||
return
|
||||
}
|
||||
if msg == nil {
|
||||
glog.Errorf("received nil message from dead letter channel")
|
||||
continue
|
||||
}
|
||||
key := "unknown"
|
||||
if msg.Metadata != nil {
|
||||
if keyValue, exists := msg.Metadata["key"]; exists {
|
||||
key = keyValue
|
||||
}
|
||||
}
|
||||
payload := ""
|
||||
if msg.Payload != nil {
|
||||
var n filer_pb.EventNotification
|
||||
if err := proto.Unmarshal(msg.Payload, &n); err != nil {
|
||||
payload = fmt.Sprintf("failed to unmarshal payload: %v", err)
|
||||
} else {
|
||||
payload = n.String()
|
||||
}
|
||||
}
|
||||
glog.Errorf("received dead letter message: %s, key: %s", payload, key)
|
||||
case <-w.ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
609
weed/notification/webhook/webhook_queue_test.go
Normal file
609
weed/notification/webhook/webhook_queue_test.go
Normal file
@@ -0,0 +1,609 @@
|
||||
package webhook
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
func TestConfigValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config *config
|
||||
wantErr bool
|
||||
errMsg string
|
||||
}{
|
||||
{
|
||||
name: "valid config",
|
||||
config: &config{
|
||||
endpoint: "https://example.com/webhook",
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 30,
|
||||
maxRetries: 3,
|
||||
backoffSeconds: 5,
|
||||
maxBackoffSeconds: 30,
|
||||
nWorkers: 5,
|
||||
bufferSize: 10000,
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "empty endpoint",
|
||||
config: &config{
|
||||
endpoint: "",
|
||||
timeoutSeconds: 30,
|
||||
maxRetries: 3,
|
||||
backoffSeconds: 5,
|
||||
maxBackoffSeconds: 30,
|
||||
nWorkers: 5,
|
||||
bufferSize: 10000,
|
||||
},
|
||||
wantErr: true,
|
||||
errMsg: "endpoint is required",
|
||||
},
|
||||
{
|
||||
name: "invalid URL",
|
||||
config: &config{
|
||||
endpoint: "://invalid-url",
|
||||
timeoutSeconds: 30,
|
||||
maxRetries: 3,
|
||||
backoffSeconds: 5,
|
||||
maxBackoffSeconds: 30,
|
||||
nWorkers: 5,
|
||||
bufferSize: 10000,
|
||||
},
|
||||
wantErr: true,
|
||||
errMsg: "invalid webhook endpoint",
|
||||
},
|
||||
{
|
||||
name: "timeout too large",
|
||||
config: &config{
|
||||
endpoint: "https://example.com/webhook",
|
||||
timeoutSeconds: 301,
|
||||
maxRetries: 3,
|
||||
backoffSeconds: 5,
|
||||
maxBackoffSeconds: 30,
|
||||
nWorkers: 5,
|
||||
bufferSize: 10000,
|
||||
},
|
||||
wantErr: true,
|
||||
errMsg: "timeout must be between",
|
||||
},
|
||||
{
|
||||
name: "too many retries",
|
||||
config: &config{
|
||||
endpoint: "https://example.com/webhook",
|
||||
timeoutSeconds: 30,
|
||||
maxRetries: 11,
|
||||
backoffSeconds: 5,
|
||||
maxBackoffSeconds: 30,
|
||||
nWorkers: 5,
|
||||
bufferSize: 10000,
|
||||
},
|
||||
wantErr: true,
|
||||
errMsg: "max retries must be between",
|
||||
},
|
||||
{
|
||||
name: "too many workers",
|
||||
config: &config{
|
||||
endpoint: "https://example.com/webhook",
|
||||
timeoutSeconds: 30,
|
||||
maxRetries: 3,
|
||||
backoffSeconds: 5,
|
||||
maxBackoffSeconds: 30,
|
||||
nWorkers: 101,
|
||||
bufferSize: 10000,
|
||||
},
|
||||
wantErr: true,
|
||||
errMsg: "workers must be between",
|
||||
},
|
||||
{
|
||||
name: "buffer too large",
|
||||
config: &config{
|
||||
endpoint: "https://example.com/webhook",
|
||||
timeoutSeconds: 30,
|
||||
maxRetries: 3,
|
||||
backoffSeconds: 5,
|
||||
maxBackoffSeconds: 30,
|
||||
nWorkers: 5,
|
||||
bufferSize: 1000001,
|
||||
},
|
||||
wantErr: true,
|
||||
errMsg: "buffer size must be between",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := tt.config.validate()
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("validate() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
if err != nil && tt.errMsg != "" {
|
||||
if err.Error() == "" || !strings.Contains(err.Error(), tt.errMsg) {
|
||||
t.Errorf("validate() error message = %v, want to contain %v", err.Error(), tt.errMsg)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebhookMessageSerialization(t *testing.T) {
|
||||
msg := &filer_pb.EventNotification{
|
||||
OldEntry: nil,
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
IsDirectory: false,
|
||||
},
|
||||
}
|
||||
|
||||
webhookMsg := newWebhookMessage("/test/path", msg)
|
||||
|
||||
wmMsg, err := webhookMsg.toWaterMillMessage()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to convert to watermill message: %v", err)
|
||||
}
|
||||
|
||||
// Unmarshal the protobuf payload directly
|
||||
var eventNotification filer_pb.EventNotification
|
||||
err = proto.Unmarshal(wmMsg.Payload, &eventNotification)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to unmarshal protobuf message: %v", err)
|
||||
}
|
||||
|
||||
// Check metadata
|
||||
if wmMsg.Metadata.Get("key") != "/test/path" {
|
||||
t.Errorf("Expected key '/test/path', got %v", wmMsg.Metadata.Get("key"))
|
||||
}
|
||||
|
||||
if wmMsg.Metadata.Get("event_type") != "create" {
|
||||
t.Errorf("Expected event type 'create', got %v", wmMsg.Metadata.Get("event_type"))
|
||||
}
|
||||
|
||||
if eventNotification.NewEntry.Name != "test.txt" {
|
||||
t.Errorf("Expected file name 'test.txt', got %v", eventNotification.NewEntry.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueInitialize(t *testing.T) {
|
||||
cfg := &config{
|
||||
endpoint: "https://example.com/webhook",
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 10,
|
||||
maxRetries: 3,
|
||||
backoffSeconds: 3,
|
||||
maxBackoffSeconds: 60,
|
||||
nWorkers: 1,
|
||||
bufferSize: 100,
|
||||
}
|
||||
|
||||
q := &Queue{}
|
||||
err := q.initialize(cfg)
|
||||
if err != nil {
|
||||
t.Errorf("Initialize() error = %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if q.cancel != nil {
|
||||
q.cancel()
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if q.router != nil {
|
||||
q.router.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
if q.router == nil {
|
||||
t.Error("Expected router to be initialized")
|
||||
}
|
||||
if q.queueChannel == nil {
|
||||
t.Error("Expected queueChannel to be initialized")
|
||||
}
|
||||
if q.client == nil {
|
||||
t.Error("Expected client to be initialized")
|
||||
}
|
||||
if q.config == nil {
|
||||
t.Error("Expected config to be initialized")
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueueSendMessage test sending messages to the queue
|
||||
func TestQueueSendMessage(t *testing.T) {
|
||||
cfg := &config{
|
||||
endpoint: "https://example.com/webhook",
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 1,
|
||||
maxRetries: 1,
|
||||
backoffSeconds: 1,
|
||||
maxBackoffSeconds: 1,
|
||||
nWorkers: 1,
|
||||
bufferSize: 10,
|
||||
}
|
||||
|
||||
q := &Queue{}
|
||||
err := q.initialize(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to initialize queue: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if q.cancel != nil {
|
||||
q.cancel()
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if q.router != nil {
|
||||
q.router.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
msg := &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
},
|
||||
}
|
||||
|
||||
err = q.SendMessage("/test/path", msg)
|
||||
if err != nil {
|
||||
t.Errorf("SendMessage() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueHandleWebhook(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
cfg := &config{
|
||||
endpoint: server.URL,
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 1,
|
||||
maxRetries: 0,
|
||||
backoffSeconds: 1,
|
||||
maxBackoffSeconds: 1,
|
||||
nWorkers: 1,
|
||||
bufferSize: 10,
|
||||
}
|
||||
|
||||
client, _ := newHTTPClient(cfg)
|
||||
q := &Queue{
|
||||
client: client,
|
||||
sem: make(chan struct{}, cfg.nWorkers),
|
||||
}
|
||||
|
||||
message := newWebhookMessage("/test/path", &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
},
|
||||
})
|
||||
|
||||
wmMsg, err := message.toWaterMillMessage()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create watermill message: %v", err)
|
||||
}
|
||||
|
||||
err = q.handleWebhook(wmMsg)
|
||||
if err != nil {
|
||||
t.Errorf("handleWebhook() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueEndToEnd(t *testing.T) {
|
||||
// Simplified test - just verify the queue can be created and message can be sent
|
||||
// without needing full end-to-end processing
|
||||
cfg := &config{
|
||||
endpoint: "https://example.com/webhook",
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 1,
|
||||
maxRetries: 0,
|
||||
backoffSeconds: 1,
|
||||
maxBackoffSeconds: 1,
|
||||
nWorkers: 1,
|
||||
bufferSize: 10,
|
||||
}
|
||||
|
||||
q := &Queue{}
|
||||
err := q.initialize(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to initialize queue: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if q.cancel != nil {
|
||||
q.cancel()
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if q.router != nil {
|
||||
q.router.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
msg := &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
},
|
||||
}
|
||||
|
||||
err = q.SendMessage("/test/path", msg)
|
||||
if err != nil {
|
||||
t.Errorf("SendMessage() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueRetryMechanism(t *testing.T) {
|
||||
cfg := &config{
|
||||
endpoint: "https://example.com/webhook",
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 1,
|
||||
maxRetries: 3, // Test that this config is used
|
||||
backoffSeconds: 2,
|
||||
maxBackoffSeconds: 10,
|
||||
nWorkers: 1,
|
||||
bufferSize: 10,
|
||||
}
|
||||
|
||||
q := &Queue{}
|
||||
err := q.initialize(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to initialize queue: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if q.cancel != nil {
|
||||
q.cancel()
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if q.router != nil {
|
||||
q.router.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
// Verify that the queue is properly configured for retries
|
||||
if q.config.maxRetries != 3 {
|
||||
t.Errorf("Expected maxRetries=3, got %d", q.config.maxRetries)
|
||||
}
|
||||
|
||||
if q.config.backoffSeconds != 2 {
|
||||
t.Errorf("Expected backoffSeconds=2, got %d", q.config.backoffSeconds)
|
||||
}
|
||||
|
||||
if q.config.maxBackoffSeconds != 10 {
|
||||
t.Errorf("Expected maxBackoffSeconds=10, got %d", q.config.maxBackoffSeconds)
|
||||
}
|
||||
|
||||
// Test that we can send a message (retry behavior is handled by Watermill middleware)
|
||||
msg := &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "test.txt"},
|
||||
}
|
||||
|
||||
err = q.SendMessage("/test/retry", msg)
|
||||
if err != nil {
|
||||
t.Errorf("SendMessage() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueueNoDuplicateWebhooks verifies that webhooks are sent only once regardless of worker count
|
||||
func TestQueueNoDuplicateWebhooks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
nWorkers int
|
||||
expected int
|
||||
}{
|
||||
{"1 worker", 1, 1},
|
||||
{"5 workers", 5, 1},
|
||||
{"10 workers", 10, 1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
callCount := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
callCount++
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
cfg := &config{
|
||||
endpoint: server.URL,
|
||||
authBearerToken: "test-token",
|
||||
timeoutSeconds: 5,
|
||||
maxRetries: 0,
|
||||
backoffSeconds: 1,
|
||||
maxBackoffSeconds: 1,
|
||||
nWorkers: tt.nWorkers,
|
||||
bufferSize: 10,
|
||||
}
|
||||
|
||||
q := &Queue{}
|
||||
err := q.initialize(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to initialize queue: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if q.cancel != nil {
|
||||
q.cancel()
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if q.router != nil {
|
||||
q.router.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
// Wait for router and subscriber to be fully ready
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
msg := &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "test.txt",
|
||||
},
|
||||
}
|
||||
|
||||
err = q.SendMessage("/test/path", msg)
|
||||
if err != nil {
|
||||
t.Errorf("SendMessage() error = %v", err)
|
||||
}
|
||||
|
||||
// Wait for message processing
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
if callCount != tt.expected {
|
||||
t.Errorf("Expected %d webhook call(s), got %d with %d workers", tt.expected, callCount, tt.nWorkers)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueSendMessageWithFilter(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg *config
|
||||
key string
|
||||
notification *filer_pb.EventNotification
|
||||
shouldPublish bool
|
||||
}{
|
||||
{
|
||||
name: "allowed event type",
|
||||
cfg: &config{
|
||||
endpoint: "https://example.com/webhook",
|
||||
timeoutSeconds: 10,
|
||||
maxRetries: 1,
|
||||
backoffSeconds: 1,
|
||||
maxBackoffSeconds: 1,
|
||||
nWorkers: 1,
|
||||
bufferSize: 10,
|
||||
eventTypes: []string{"create"},
|
||||
},
|
||||
key: "/data/file.txt",
|
||||
notification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "file.txt"},
|
||||
},
|
||||
shouldPublish: true,
|
||||
},
|
||||
{
|
||||
name: "filtered event type",
|
||||
cfg: &config{
|
||||
endpoint: "https://example.com/webhook",
|
||||
timeoutSeconds: 10,
|
||||
maxRetries: 1,
|
||||
backoffSeconds: 1,
|
||||
maxBackoffSeconds: 1,
|
||||
nWorkers: 1,
|
||||
bufferSize: 10,
|
||||
eventTypes: []string{"update", "rename"},
|
||||
},
|
||||
key: "/data/file.txt",
|
||||
notification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "file.txt"},
|
||||
},
|
||||
shouldPublish: false,
|
||||
},
|
||||
{
|
||||
name: "allowed path prefix",
|
||||
cfg: &config{
|
||||
endpoint: "https://example.com/webhook",
|
||||
timeoutSeconds: 10,
|
||||
maxRetries: 1,
|
||||
backoffSeconds: 1,
|
||||
maxBackoffSeconds: 1,
|
||||
nWorkers: 1,
|
||||
bufferSize: 10,
|
||||
pathPrefixes: []string{"/data/"},
|
||||
},
|
||||
key: "/data/file.txt",
|
||||
notification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "file.txt"},
|
||||
},
|
||||
shouldPublish: true,
|
||||
},
|
||||
{
|
||||
name: "filtered path prefix",
|
||||
cfg: &config{
|
||||
endpoint: "https://example.com/webhook",
|
||||
timeoutSeconds: 10,
|
||||
maxRetries: 1,
|
||||
backoffSeconds: 1,
|
||||
maxBackoffSeconds: 1,
|
||||
nWorkers: 1,
|
||||
bufferSize: 10,
|
||||
pathPrefixes: []string{"/logs/"},
|
||||
},
|
||||
key: "/data/file.txt",
|
||||
notification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "file.txt"},
|
||||
},
|
||||
shouldPublish: false,
|
||||
},
|
||||
{
|
||||
name: "combined filters - both pass",
|
||||
cfg: &config{
|
||||
endpoint: "https://example.com/webhook",
|
||||
timeoutSeconds: 10,
|
||||
maxRetries: 1,
|
||||
backoffSeconds: 1,
|
||||
maxBackoffSeconds: 1,
|
||||
nWorkers: 1,
|
||||
bufferSize: 10,
|
||||
eventTypes: []string{"create", "delete"},
|
||||
pathPrefixes: []string{"/data/", "/logs/"},
|
||||
},
|
||||
key: "/data/file.txt",
|
||||
notification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "file.txt"},
|
||||
},
|
||||
shouldPublish: true,
|
||||
},
|
||||
{
|
||||
name: "combined filters - event fails",
|
||||
cfg: &config{
|
||||
endpoint: "https://example.com/webhook",
|
||||
timeoutSeconds: 10,
|
||||
maxRetries: 1,
|
||||
backoffSeconds: 1,
|
||||
maxBackoffSeconds: 1,
|
||||
nWorkers: 1,
|
||||
bufferSize: 10,
|
||||
eventTypes: []string{"update", "delete"},
|
||||
pathPrefixes: []string{"/data/", "/logs/"},
|
||||
},
|
||||
key: "/data/file.txt",
|
||||
notification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "file.txt"},
|
||||
},
|
||||
shouldPublish: false,
|
||||
},
|
||||
{
|
||||
name: "combined filters - path fails",
|
||||
cfg: &config{
|
||||
endpoint: "https://example.com/webhook",
|
||||
timeoutSeconds: 10,
|
||||
maxRetries: 1,
|
||||
backoffSeconds: 1,
|
||||
maxBackoffSeconds: 1,
|
||||
nWorkers: 1,
|
||||
bufferSize: 10,
|
||||
eventTypes: []string{"create", "delete"},
|
||||
pathPrefixes: []string{"/logs/"},
|
||||
},
|
||||
key: "/data/file.txt",
|
||||
notification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "file.txt"},
|
||||
},
|
||||
shouldPublish: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
shouldPublish := newFilter(tt.cfg).shouldPublish(tt.key, tt.notification)
|
||||
if shouldPublish != tt.shouldPublish {
|
||||
t.Errorf("Expected shouldPublish=%v, got %v", tt.shouldPublish, shouldPublish)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user