snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/kafka/scripts/kafka-gateway-start.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+
+# Kafka Gateway Startup Script for Integration Testing
+
+set -e
+
+echo "Starting Kafka Gateway..."
+
+SEAWEEDFS_MASTERS=${SEAWEEDFS_MASTERS:-seaweedfs-master:9333}
+SEAWEEDFS_FILER=${SEAWEEDFS_FILER:-seaweedfs-filer:8888}
+SEAWEEDFS_MQ_BROKER=${SEAWEEDFS_MQ_BROKER:-seaweedfs-mq-broker:17777}
+SEAWEEDFS_FILER_GROUP=${SEAWEEDFS_FILER_GROUP:-}
+
+# Wait for dependencies
+echo "Waiting for SeaweedFS master(s)..."
+OLD_IFS="$IFS"
+IFS=','
+for MASTER in $SEAWEEDFS_MASTERS; do
+  MASTER_HOST=${MASTER%:*}
+  MASTER_PORT=${MASTER#*:}
+  while ! nc -z "$MASTER_HOST" "$MASTER_PORT"; do
+    sleep 1
+  done
+  echo "SeaweedFS master $MASTER is ready"
+done
+IFS="$OLD_IFS"
+
+echo "Waiting for SeaweedFS Filer..."
+while ! nc -z "${SEAWEEDFS_FILER%:*}" "${SEAWEEDFS_FILER#*:}"; do
+  sleep 1
+done
+echo "SeaweedFS Filer is ready"
+
+echo "Waiting for SeaweedFS MQ Broker..."
+while ! nc -z "${SEAWEEDFS_MQ_BROKER%:*}" "${SEAWEEDFS_MQ_BROKER#*:}"; do
+  sleep 1
+done
+echo "SeaweedFS MQ Broker is ready"
+
+echo "Waiting for Schema Registry..."
+while ! curl -f "${SCHEMA_REGISTRY_URL}/subjects" > /dev/null 2>&1; do
+  sleep 1
+done
+echo "Schema Registry is ready"
+
+# Start Kafka Gateway
+echo "Starting Kafka Gateway on port ${KAFKA_PORT:-9093}..."
+exec /usr/bin/weed mq.kafka.gateway \
+  -master=${SEAWEEDFS_MASTERS} \
+  -filerGroup=${SEAWEEDFS_FILER_GROUP} \
+  -port=${KAFKA_PORT:-9093} \
+  -port.pprof=${PPROF_PORT:-10093} \
+  -schema-registry-url=${SCHEMA_REGISTRY_URL} \
+  -ip=0.0.0.0

test/kafka/scripts/test-broker-discovery.sh

@@ -0,0 +1,129 @@
+#!/bin/bash
+
+# Test script to verify broker discovery works end-to-end
+
+set -e
+
+echo "=== Testing SeaweedFS Broker Discovery ==="
+
+cd /Users/chrislu/go/src/github.com/seaweedfs/seaweedfs
+
+# Build weed binary
+echo "Building weed binary..."
+go build -o /tmp/weed-discovery ./weed
+
+# Setup data directory
+WEED_DATA_DIR="/tmp/seaweedfs-discovery-test-$$"
+mkdir -p "$WEED_DATA_DIR"
+echo "Using data directory: $WEED_DATA_DIR"
+
+# Cleanup function
+cleanup() {
+    echo "Cleaning up..."
+    pkill -f "weed.*server" || true
+    pkill -f "weed.*mq.broker" || true
+    sleep 2
+    rm -rf "$WEED_DATA_DIR"
+    rm -f /tmp/weed-discovery* /tmp/broker-discovery-test*
+}
+trap cleanup EXIT
+
+# Start SeaweedFS server with consistent IP configuration
+echo "Starting SeaweedFS server..."
+/tmp/weed-discovery -v 1 server \
+  -ip="127.0.0.1" \
+  -ip.bind="127.0.0.1" \
+  -dir="$WEED_DATA_DIR" \
+  -master.raftHashicorp \
+  -master.port=9333 \
+  -volume.port=8081 \
+  -filer.port=8888 \
+  -filer=true \
+  -metricsPort=9325 \
+  > /tmp/weed-discovery-server.log 2>&1 &
+
+SERVER_PID=$!
+echo "Server PID: $SERVER_PID"
+
+# Wait for master
+echo "Waiting for master..."
+for i in $(seq 1 30); do
+  if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
+    echo "✓ Master is up"
+    break
+  fi
+  echo "  Waiting for master... ($i/30)"
+  sleep 1
+done
+
+# Give components time to initialize
+echo "Waiting for components to initialize..."
+sleep 10
+
+# Start MQ broker
+echo "Starting MQ broker..."
+/tmp/weed-discovery -v 2 mq.broker \
+  -master="127.0.0.1:9333" \
+  -port=17777 \
+  > /tmp/weed-discovery-broker.log 2>&1 &
+
+BROKER_PID=$!
+echo "Broker PID: $BROKER_PID"
+
+# Wait for broker
+echo "Waiting for broker to register..."
+sleep 15
+broker_ready=false
+for i in $(seq 1 20); do
+  if nc -z 127.0.0.1 17777; then
+    echo "✓ MQ broker is accepting connections"
+    broker_ready=true
+    break
+  fi
+  echo "  Waiting for MQ broker... ($i/20)"
+  sleep 1
+done
+
+if [ "$broker_ready" = false ]; then
+  echo "[FAIL] MQ broker failed to start"
+  echo "Server logs:"
+  cat /tmp/weed-discovery-server.log
+  echo "Broker logs:"  
+  cat /tmp/weed-discovery-broker.log
+  exit 1
+fi
+
+# Additional wait for broker registration
+echo "Allowing broker to register with master..."
+sleep 15
+
+# Check cluster status
+echo "Checking cluster status..."
+CLUSTER_STATUS=$(curl -s "http://127.0.0.1:9333/cluster/status")
+echo "Cluster status: $CLUSTER_STATUS"
+
+# Now test broker discovery using the same approach as the Kafka gateway
+echo "Testing broker discovery..."
+cd test/kafka
+SEAWEEDFS_MASTERS=127.0.0.1:9333 timeout 30s go test -v -run "TestOffsetManagement" -timeout 25s ./e2e/... > /tmp/broker-discovery-test.log 2>&1 && discovery_success=true || discovery_success=false
+
+if [ "$discovery_success" = true ]; then
+  echo "[OK] Broker discovery test PASSED!"
+  echo "Gateway was able to discover and connect to MQ brokers"
+else
+  echo "[FAIL] Broker discovery test FAILED"
+  echo "Last few lines of test output:"
+  tail -20 /tmp/broker-discovery-test.log || echo "No test logs available"
+fi
+
+echo
+echo "📊 Test Results:"
+echo "  Broker startup: ✅"
+echo "  Broker registration: ✅"  
+echo "  Gateway discovery: $([ "$discovery_success" = true ] && echo "✅" || echo "❌")"
+
+echo
+echo "📁 Logs available:"
+echo "  Server: /tmp/weed-discovery-server.log"
+echo "  Broker: /tmp/weed-discovery-broker.log"
+echo "  Discovery test: /tmp/broker-discovery-test.log"

test/kafka/scripts/test-broker-startup.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+
+# Script to test SeaweedFS MQ broker startup locally
+# This helps debug broker startup issues before running CI
+
+set -e
+
+echo "=== Testing SeaweedFS MQ Broker Startup ==="
+
+# Build weed binary
+echo "Building weed binary..."
+cd "$(dirname "$0")/../../.."
+go build -o /tmp/weed ./weed
+
+# Setup data directory
+WEED_DATA_DIR="/tmp/seaweedfs-broker-test-$$"
+mkdir -p "$WEED_DATA_DIR"
+echo "Using data directory: $WEED_DATA_DIR"
+
+# Cleanup function
+cleanup() {
+    echo "Cleaning up..."
+    pkill -f "weed.*server" || true
+    pkill -f "weed.*mq.broker" || true
+    sleep 2
+    rm -rf "$WEED_DATA_DIR"
+    rm -f /tmp/weed-*.log
+}
+trap cleanup EXIT
+
+# Start SeaweedFS server  
+echo "Starting SeaweedFS server..."
+/tmp/weed -v 1 server \
+  -ip="127.0.0.1" \
+  -ip.bind="0.0.0.0" \
+  -dir="$WEED_DATA_DIR" \
+  -master.raftHashicorp \
+  -master.port=9333 \
+  -volume.port=8081 \
+  -filer.port=8888 \
+  -filer=true \
+  -metricsPort=9325 \
+  > /tmp/weed-server-test.log 2>&1 &
+
+SERVER_PID=$!
+echo "Server PID: $SERVER_PID"
+
+# Wait for master
+echo "Waiting for master..."
+for i in $(seq 1 30); do
+  if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
+    echo "✓ Master is up"
+    break
+  fi
+  echo "  Waiting for master... ($i/30)"
+  sleep 1
+done
+
+# Wait for filer
+echo "Waiting for filer..."
+for i in $(seq 1 30); do
+  if nc -z 127.0.0.1 8888; then
+    echo "✓ Filer is up"
+    break
+  fi
+  echo "  Waiting for filer... ($i/30)"
+  sleep 1
+done
+
+# Start MQ broker  
+echo "Starting MQ broker..."
+/tmp/weed -v 2 mq.broker \
+  -master="127.0.0.1:9333" \
+  -ip="127.0.0.1" \
+  -port=17777 \
+  > /tmp/weed-mq-broker-test.log 2>&1 &
+
+BROKER_PID=$!
+echo "Broker PID: $BROKER_PID"
+
+# Wait for broker
+echo "Waiting for broker..."
+broker_ready=false
+for i in $(seq 1 30); do
+  if nc -z 127.0.0.1 17777; then
+    echo "✓ MQ broker is up"
+    broker_ready=true
+    break
+  fi
+  echo "  Waiting for MQ broker... ($i/30)"
+  sleep 1
+done
+
+if [ "$broker_ready" = false ]; then
+  echo "❌ MQ broker failed to start"
+  echo
+  echo "=== Server logs ==="
+  cat /tmp/weed-server-test.log
+  echo
+  echo "=== Broker logs ==="
+  cat /tmp/weed-mq-broker-test.log
+  exit 1
+fi
+
+# Broker started successfully - discovery will be tested by Kafka gateway
+echo "✓ Broker started successfully and accepting connections"
+
+echo
+echo "[OK] All tests passed!"
+echo "Server logs: /tmp/weed-server-test.log"  
+echo "Broker logs: /tmp/weed-mq-broker-test.log"

test/kafka/scripts/test_schema_registry.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Test script for schema registry E2E testing
+# This script sets up a mock schema registry and runs the E2E tests
+
+set -e
+
+echo "🚀 Starting Schema Registry E2E Test"
+
+# Check if we have a real schema registry URL
+if [ -n "$SCHEMA_REGISTRY_URL" ]; then
+    echo "📡 Using real Schema Registry: $SCHEMA_REGISTRY_URL"
+else
+    echo "🔧 No SCHEMA_REGISTRY_URL set, using mock registry"
+    # For now, we'll skip the test if no real registry is available
+    # In the future, we could start a mock registry here
+    export SCHEMA_REGISTRY_URL="http://localhost:8081"
+    echo "⚠️  Mock registry not implemented yet, test will be skipped"
+fi
+
+# Start SeaweedFS infrastructure
+echo "🌱 Starting SeaweedFS infrastructure..."
+cd /Users/chrislu/go/src/github.com/seaweedfs/seaweedfs
+
+# Clean up any existing processes
+pkill -f "weed server" || true
+pkill -f "weed mq.broker" || true
+sleep 2
+
+# Start SeaweedFS server
+echo "🗄️  Starting SeaweedFS server..."
+/tmp/weed server -dir=/tmp/seaweedfs-test -master.port=9333 -volume.port=8080 -filer.port=8888 -ip=localhost -master.peers=none > /tmp/seaweed-server.log 2>&1 &
+SERVER_PID=$!
+
+# Wait for server to be ready
+sleep 5
+
+# Start MQ broker
+echo "📨 Starting SeaweedMQ broker..."
+/tmp/weed mq.broker -master=localhost:9333 -port=17777 > /tmp/seaweed-broker.log 2>&1 &
+BROKER_PID=$!
+
+# Wait for broker to be ready
+sleep 3
+
+# Check if services are running
+if ! curl -s http://localhost:9333/cluster/status > /dev/null; then
+    echo "[FAIL] SeaweedFS server not ready"
+    exit 1
+fi
+
+echo "[OK] SeaweedFS infrastructure ready"
+
+# Run the schema registry E2E tests
+echo "🧪 Running Schema Registry E2E tests..."
+cd /Users/chrislu/go/src/github.com/seaweedfs/seaweedfs/test/kafka
+
+export SEAWEEDFS_MASTERS=127.0.0.1:9333
+
+# Run the tests
+if go test -v ./integration -run TestSchemaRegistryE2E -timeout 5m; then
+    echo "[OK] Schema Registry E2E tests PASSED!"
+    TEST_RESULT=0
+else
+    echo "[FAIL] Schema Registry E2E tests FAILED!"
+    TEST_RESULT=1
+fi
+
+# Cleanup
+echo "🧹 Cleaning up..."
+kill $BROKER_PID $SERVER_PID 2>/dev/null || true
+sleep 2
+pkill -f "weed server" || true
+pkill -f "weed mq.broker" || true
+
+echo "🏁 Schema Registry E2E Test completed"
+exit $TEST_RESULT

test/kafka/scripts/wait-for-services.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+
+# Wait for Services Script for Kafka Integration Tests
+
+set -e
+
+echo "Waiting for services to be ready..."
+
+# Configuration
+KAFKA_HOST=${KAFKA_HOST:-localhost}
+KAFKA_PORT=${KAFKA_PORT:-9092}
+SCHEMA_REGISTRY_URL=${SCHEMA_REGISTRY_URL:-http://localhost:8081}
+KAFKA_GATEWAY_HOST=${KAFKA_GATEWAY_HOST:-localhost}
+KAFKA_GATEWAY_PORT=${KAFKA_GATEWAY_PORT:-9093}
+SEAWEEDFS_MASTER_URL=${SEAWEEDFS_MASTER_URL:-http://localhost:9333}
+MAX_WAIT=${MAX_WAIT:-300}  # 5 minutes
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Helper function to wait for a service
+wait_for_service() {
+    local service_name=$1
+    local check_command=$2
+    local timeout=${3:-60}
+    
+    echo -e "${BLUE}Waiting for ${service_name}...${NC}"
+    
+    local count=0
+    while [ $count -lt $timeout ]; do
+        if eval "$check_command" > /dev/null 2>&1; then
+            echo -e "${GREEN}[OK] ${service_name} is ready${NC}"
+            return 0
+        fi
+        
+        if [ $((count % 10)) -eq 0 ]; then
+            echo -e "${YELLOW}Still waiting for ${service_name}... (${count}s)${NC}"
+        fi
+        
+        sleep 1
+        count=$((count + 1))
+    done
+    
+    echo -e "${RED}[FAIL] ${service_name} failed to start within ${timeout} seconds${NC}"
+    return 1
+}
+
+# Wait for Zookeeper
+echo "=== Checking Zookeeper ==="
+wait_for_service "Zookeeper" "nc -z localhost 2181" 30
+
+# Wait for Kafka
+echo "=== Checking Kafka ==="
+wait_for_service "Kafka" "nc -z ${KAFKA_HOST} ${KAFKA_PORT}" 60
+
+# Test Kafka broker API
+echo "=== Testing Kafka API ==="
+wait_for_service "Kafka API" "timeout 5 kafka-broker-api-versions --bootstrap-server ${KAFKA_HOST}:${KAFKA_PORT}" 30
+
+# Wait for Schema Registry
+echo "=== Checking Schema Registry ==="
+wait_for_service "Schema Registry" "curl -f ${SCHEMA_REGISTRY_URL}/subjects" 60
+
+# Wait for SeaweedFS Master
+echo "=== Checking SeaweedFS Master ==="
+wait_for_service "SeaweedFS Master" "curl -f ${SEAWEEDFS_MASTER_URL}/cluster/status" 30
+
+# Wait for SeaweedFS Volume
+echo "=== Checking SeaweedFS Volume ==="
+wait_for_service "SeaweedFS Volume" "curl -f http://localhost:8080/status" 30
+
+# Wait for SeaweedFS Filer
+echo "=== Checking SeaweedFS Filer ==="
+wait_for_service "SeaweedFS Filer" "curl -f http://localhost:8888/" 30
+
+# Wait for SeaweedFS MQ Broker
+echo "=== Checking SeaweedFS MQ Broker ==="
+wait_for_service "SeaweedFS MQ Broker" "nc -z localhost 17777" 30
+
+# Wait for SeaweedFS MQ Agent
+echo "=== Checking SeaweedFS MQ Agent ==="
+wait_for_service "SeaweedFS MQ Agent" "nc -z localhost 16777" 30
+
+# Wait for Kafka Gateway
+echo "=== Checking Kafka Gateway ==="
+wait_for_service "Kafka Gateway" "nc -z ${KAFKA_GATEWAY_HOST} ${KAFKA_GATEWAY_PORT}" 60
+
+# Final verification
+echo "=== Final Verification ==="
+
+# Test Kafka topic creation
+echo "Testing Kafka topic operations..."
+TEST_TOPIC="health-check-$(date +%s)"
+if kafka-topics --create --topic "$TEST_TOPIC" --bootstrap-server "${KAFKA_HOST}:${KAFKA_PORT}" --partitions 1 --replication-factor 1 > /dev/null 2>&1; then
+    echo -e "${GREEN}[OK] Kafka topic creation works${NC}"
+    kafka-topics --delete --topic "$TEST_TOPIC" --bootstrap-server "${KAFKA_HOST}:${KAFKA_PORT}" > /dev/null 2>&1 || true
+else
+    echo -e "${RED}[FAIL] Kafka topic creation failed${NC}"
+    exit 1
+fi
+
+# Test Schema Registry
+echo "Testing Schema Registry..."
+if curl -f "${SCHEMA_REGISTRY_URL}/subjects" > /dev/null 2>&1; then
+    echo -e "${GREEN}[OK] Schema Registry is accessible${NC}"
+else
+    echo -e "${RED}[FAIL] Schema Registry is not accessible${NC}"
+    exit 1
+fi
+
+# Test Kafka Gateway connectivity
+echo "Testing Kafka Gateway..."
+if nc -z "${KAFKA_GATEWAY_HOST}" "${KAFKA_GATEWAY_PORT}"; then
+    echo -e "${GREEN}[OK] Kafka Gateway is accessible${NC}"
+else
+    echo -e "${RED}[FAIL] Kafka Gateway is not accessible${NC}"
+    exit 1
+fi
+
+echo -e "${GREEN}All services are ready!${NC}"
+echo ""
+echo "Service endpoints:"
+echo "  Kafka: ${KAFKA_HOST}:${KAFKA_PORT}"
+echo "  Schema Registry: ${SCHEMA_REGISTRY_URL}"
+echo "  Kafka Gateway: ${KAFKA_GATEWAY_HOST}:${KAFKA_GATEWAY_PORT}"
+echo "  SeaweedFS Master: ${SEAWEEDFS_MASTER_URL}"
+echo "  SeaweedFS Filer: http://localhost:8888"
+echo "  SeaweedFS MQ Broker: localhost:17777"
+echo "  SeaweedFS MQ Agent: localhost:16777"
+echo ""
+echo "Ready to run integration tests!"