upstream-seaweedfs/test/kafka/kafka-client-loadtest/scripts/run-loadtest.sh

#!/bin/bash

# Kafka Client Load Test Runner Script
# This script helps run various load test scenarios against SeaweedFS Kafka Gateway

set -euo pipefail

# Default configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
DOCKER_COMPOSE_FILE="$PROJECT_DIR/docker-compose.yml"
CONFIG_FILE="$PROJECT_DIR/config/loadtest.yaml"

# Default test parameters
TEST_MODE="comprehensive"
TEST_DURATION="300s"
PRODUCER_COUNT=10
CONSUMER_COUNT=5
MESSAGE_RATE=1000
MESSAGE_SIZE=1024
TOPIC_COUNT=5
PARTITIONS_PER_TOPIC=3

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Function to print colored output
log_info() {
    echo -e "${BLUE}[INFO]${NC} $1"
}

log_success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1"
}

log_warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# Function to show usage
show_usage() {
    cat << EOF
Kafka Client Load Test Runner

Usage: $0 [OPTIONS] [COMMAND]

Commands:
  start               Start the load test infrastructure and run tests
  stop                Stop all services
  restart             Restart all services
  status              Show service status
  logs                Show logs from all services
  clean               Clean up all resources (volumes, networks, etc.)
  monitor             Start monitoring stack (Prometheus + Grafana)
  scenarios           Run predefined test scenarios

Options:
  -m, --mode MODE           Test mode: producer, consumer, comprehensive (default: comprehensive)
  -d, --duration DURATION   Test duration (default: 300s)
  -p, --producers COUNT     Number of producers (default: 10)
  -c, --consumers COUNT     Number of consumers (default: 5)
  -r, --rate RATE          Messages per second per producer (default: 1000)
  -s, --size SIZE          Message size in bytes (default: 1024)
  -t, --topics COUNT       Number of topics (default: 5)
  --partitions COUNT       Partitions per topic (default: 3)
  --config FILE           Configuration file (default: config/loadtest.yaml)
  --monitoring            Enable monitoring stack
  --wait-ready            Wait for services to be ready before starting tests
  -v, --verbose           Verbose output
  -h, --help              Show this help message

Examples:
  # Run comprehensive test for 5 minutes
  $0 start -m comprehensive -d 5m

  # Run producer-only test with high throughput
  $0 start -m producer -p 20 -r 2000 -d 10m

  # Run consumer-only test
  $0 start -m consumer -c 10

  # Run with monitoring
  $0 start --monitoring -d 15m

  # Clean up everything
  $0 clean

Predefined Scenarios:
  quick              Quick smoke test (1 min, low load)
  standard           Standard load test (5 min, medium load) 
  stress             Stress test (10 min, high load)
  endurance          Endurance test (30 min, sustained load)
  burst              Burst test (variable load)

EOF
}

# Parse command line arguments
parse_args() {
    while [[ $# -gt 0 ]]; do
        case $1 in
            -m|--mode)
                TEST_MODE="$2"
                shift 2
                ;;
            -d|--duration)
                TEST_DURATION="$2"
                shift 2
                ;;
            -p|--producers)
                PRODUCER_COUNT="$2"
                shift 2
                ;;
            -c|--consumers)
                CONSUMER_COUNT="$2"
                shift 2
                ;;
            -r|--rate)
                MESSAGE_RATE="$2"
                shift 2
                ;;
            -s|--size)
                MESSAGE_SIZE="$2"
                shift 2
                ;;
            -t|--topics)
                TOPIC_COUNT="$2"
                shift 2
                ;;
            --partitions)
                PARTITIONS_PER_TOPIC="$2"
                shift 2
                ;;
            --config)
                CONFIG_FILE="$2"
                shift 2
                ;;
            --monitoring)
                ENABLE_MONITORING=1
                shift
                ;;
            --wait-ready)
                WAIT_READY=1
                shift
                ;;
            -v|--verbose)
                VERBOSE=1
                shift
                ;;
            -h|--help)
                show_usage
                exit 0
                ;;
            -*)
                log_error "Unknown option: $1"
                show_usage
                exit 1
                ;;
            *)
                if [[ -z "${COMMAND:-}" ]]; then
                    COMMAND="$1"
                else
                    log_error "Multiple commands specified"
                    show_usage
                    exit 1
                fi
                shift
                ;;
        esac
    done
}

# Check if Docker and Docker Compose are available
check_dependencies() {
    if ! command -v docker &> /dev/null; then
        log_error "Docker is not installed or not in PATH"
        exit 1
    fi
    
    if ! command -v docker-compose &> /dev/null && ! docker compose version &> /dev/null; then
        log_error "Docker Compose is not installed or not in PATH"
        exit 1
    fi
    
    # Use docker compose if available, otherwise docker-compose
    if docker compose version &> /dev/null; then
        DOCKER_COMPOSE="docker compose"
    else
        DOCKER_COMPOSE="docker-compose"
    fi
}

# Wait for services to be ready
wait_for_services() {
    log_info "Waiting for services to be ready..."
    
    local timeout=300  # 5 minutes timeout
    local elapsed=0
    local check_interval=5
    
    while [[ $elapsed -lt $timeout ]]; do
        if $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" ps --format table | grep -q "healthy"; then
            if check_service_health; then
                log_success "All services are ready!"
                return 0
            fi
        fi
        
        sleep $check_interval
        elapsed=$((elapsed + check_interval))
        log_info "Waiting... ($elapsed/${timeout}s)"
    done
    
    log_error "Services did not become ready within $timeout seconds"
    return 1
}

# Check health of critical services
check_service_health() {
    # Check Kafka Gateway
    if ! curl -s http://localhost:9093 >/dev/null 2>&1; then
        return 1
    fi
    
    # Check Schema Registry
    if ! curl -s http://localhost:8081/subjects >/dev/null 2>&1; then
        return 1
    fi
    
    return 0
}

# Start the load test infrastructure
start_services() {
    log_info "Starting SeaweedFS Kafka load test infrastructure..."
    
    # Set environment variables
    export TEST_MODE="$TEST_MODE"
    export TEST_DURATION="$TEST_DURATION"
    export PRODUCER_COUNT="$PRODUCER_COUNT"
    export CONSUMER_COUNT="$CONSUMER_COUNT"
    export MESSAGE_RATE="$MESSAGE_RATE"
    export MESSAGE_SIZE="$MESSAGE_SIZE"
    export TOPIC_COUNT="$TOPIC_COUNT"
    export PARTITIONS_PER_TOPIC="$PARTITIONS_PER_TOPIC"
    
    # Start core services
    $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" up -d \
        seaweedfs-master \
        seaweedfs-volume \
        seaweedfs-filer \
        seaweedfs-mq-broker \
        kafka-gateway \
        schema-registry
    
    # Start monitoring if enabled
    if [[ "${ENABLE_MONITORING:-0}" == "1" ]]; then
        log_info "Starting monitoring stack..."
        $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" --profile monitoring up -d
    fi
    
    # Wait for services to be ready if requested
    if [[ "${WAIT_READY:-0}" == "1" ]]; then
        wait_for_services
    fi
    
    log_success "Infrastructure started successfully"
}

# Run the load test
run_loadtest() {
    log_info "Starting Kafka client load test..."
    log_info "Mode: $TEST_MODE, Duration: $TEST_DURATION"
    log_info "Producers: $PRODUCER_COUNT, Consumers: $CONSUMER_COUNT"
    log_info "Message Rate: $MESSAGE_RATE msgs/sec, Size: $MESSAGE_SIZE bytes"
    
    # Run the load test
    $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" --profile loadtest up --abort-on-container-exit kafka-client-loadtest
    
    # Show test results
    show_results
}

# Show test results
show_results() {
    log_info "Load test completed! Gathering results..."
    
    # Get final metrics from the load test container
    if $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" ps kafka-client-loadtest-runner &>/dev/null; then
        log_info "Final test statistics:"
        $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" exec -T kafka-client-loadtest-runner curl -s http://localhost:8080/stats || true
    fi
    
    # Show Prometheus metrics if monitoring is enabled
    if [[ "${ENABLE_MONITORING:-0}" == "1" ]]; then
        log_info "Monitoring dashboards available at:"
        log_info "  Prometheus: http://localhost:9090"
        log_info "  Grafana:    http://localhost:3000 (admin/admin)"
    fi
    
    # Show where results are stored
    if [[ -d "$PROJECT_DIR/test-results" ]]; then
        log_info "Test results saved to: $PROJECT_DIR/test-results/"
    fi
}

# Stop services
stop_services() {
    log_info "Stopping all services..."
    $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" --profile loadtest --profile monitoring down
    log_success "Services stopped"
}

# Show service status
show_status() {
    log_info "Service status:"
    $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" ps
}

# Show logs
show_logs() {
    $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" logs -f "${1:-}"
}

# Clean up all resources
clean_all() {
    log_warning "This will remove all volumes, networks, and containers. Are you sure? (y/N)"
    read -r response
    if [[ "$response" =~ ^[Yy]$ ]]; then
        log_info "Cleaning up all resources..."
        $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" --profile loadtest --profile monitoring down -v --remove-orphans
        
        # Remove any remaining volumes
        docker volume ls -q | grep -E "(kafka-client-loadtest|seaweedfs)" | xargs -r docker volume rm
        
        # Remove networks
        docker network ls -q | grep -E "kafka-client-loadtest" | xargs -r docker network rm
        
        log_success "Cleanup completed"
    else
        log_info "Cleanup cancelled"
    fi
}

# Run predefined scenarios
run_scenario() {
    local scenario="$1"
    
    case "$scenario" in
        quick)
            TEST_MODE="comprehensive"
            TEST_DURATION="1m"
            PRODUCER_COUNT=2
            CONSUMER_COUNT=2
            MESSAGE_RATE=100
            MESSAGE_SIZE=512
            TOPIC_COUNT=2
            ;;
        standard)
            TEST_MODE="comprehensive"
            TEST_DURATION="5m"
            PRODUCER_COUNT=5
            CONSUMER_COUNT=3
            MESSAGE_RATE=500
            MESSAGE_SIZE=1024
            TOPIC_COUNT=3
            ;;
        stress)
            TEST_MODE="comprehensive"
            TEST_DURATION="10m"
            PRODUCER_COUNT=20
            CONSUMER_COUNT=10
            MESSAGE_RATE=2000
            MESSAGE_SIZE=2048
            TOPIC_COUNT=10
            ;;
        endurance)
            TEST_MODE="comprehensive"
            TEST_DURATION="30m"
            PRODUCER_COUNT=10
            CONSUMER_COUNT=5
            MESSAGE_RATE=1000
            MESSAGE_SIZE=1024
            TOPIC_COUNT=5
            ;;
        burst)
            TEST_MODE="comprehensive"
            TEST_DURATION="10m"
            PRODUCER_COUNT=10
            CONSUMER_COUNT=5
            MESSAGE_RATE=1000
            MESSAGE_SIZE=1024
            TOPIC_COUNT=5
            # Note: Burst behavior would be configured in the load test config
            ;;
        *)
            log_error "Unknown scenario: $scenario"
            log_info "Available scenarios: quick, standard, stress, endurance, burst"
            exit 1
            ;;
    esac
    
    log_info "Running $scenario scenario..."
    start_services
    if [[ "${WAIT_READY:-0}" == "1" ]]; then
        wait_for_services
    fi
    run_loadtest
}

# Main execution
main() {
    if [[ $# -eq 0 ]]; then
        show_usage
        exit 0
    fi
    
    parse_args "$@"
    check_dependencies
    
    case "${COMMAND:-}" in
        start)
            start_services
            run_loadtest
            ;;
        stop)
            stop_services
            ;;
        restart)
            stop_services
            start_services
            ;;
        status)
            show_status
            ;;
        logs)
            show_logs
            ;;
        clean)
            clean_all
            ;;
        monitor)
            ENABLE_MONITORING=1
            $DOCKER_COMPOSE -f "$DOCKER_COMPOSE_FILE" --profile monitoring up -d
            log_success "Monitoring stack started"
            log_info "Prometheus: http://localhost:9090"
            log_info "Grafana:    http://localhost:3000 (admin/admin)"
            ;;
        scenarios)
            if [[ -n "${2:-}" ]]; then
                run_scenario "$2"
            else
                log_error "Please specify a scenario"
                log_info "Available scenarios: quick, standard, stress, endurance, burst"
                exit 1
            fi
            ;;
        *)
            log_error "Unknown command: ${COMMAND:-}"
            show_usage
            exit 1
            ;;
    esac
}

# Set default values
ENABLE_MONITORING=0
WAIT_READY=0
VERBOSE=0

# Run main function
main "$@"