upstream-seaweedfs/.github/workflows/e2e.yml

name: "End to End"

on:
  push:
    branches: [ master ]
    paths:
      - 'weed/**'
      - 'docker/**'
      - 'go.mod'
      - 'go.sum'
      - '.github/workflows/e2e.yml'
  pull_request:
    branches: [ master ]
    paths:
      - 'weed/**'
      - 'docker/**'
      - 'go.mod'
      - 'go.sum'
      - '.github/workflows/e2e.yml'

concurrency:
  group: ${{ github.head_ref }}/e2e
  cancel-in-progress: true

permissions:
  contents: read

defaults:
  run:
    working-directory: docker

jobs:
  e2e:
    name: FUSE Mount
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    steps:
    - name: Check out code into the Go module directory
      uses: actions/checkout@v7

    - name: Set up Go
      uses: actions/setup-go@v7
      with:
        go-version-file: 'go.mod'

    - name: Configure Docker Hub mirror
      run: |
        echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
        sudo systemctl restart docker

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v4
      
    - name: Cache Docker layers
      uses: actions/cache@v6
      with:
        path: /tmp/.buildx-cache
        key: ${{ runner.os }}-buildx-e2e-${{ github.sha }}
        restore-keys: |
          ${{ runner.os }}-buildx-e2e-

    - name: Install dependencies
      run: |
        sudo rm -f /etc/apt/sources.list.d/azure-cli.list /etc/apt/sources.list.d/microsoft-prod.list
        # Same helper the e2e image installs through: the runner's own list is
        # azure-only too, and an outage there fails this step outright.
        sudo ./apt-install fuse
        
        # Verify FUSE installation
        echo "FUSE version: $(fusermount --version 2>&1 || echo 'fusermount not found')"
        echo "FUSE device: $(ls -la /dev/fuse 2>&1 || echo '/dev/fuse not found')"

    - name: Start SeaweedFS
      timeout-minutes: 15
      run: |
        # Enable Docker buildkit for better caching
        export DOCKER_BUILDKIT=1
        export COMPOSE_DOCKER_CLI_BUILD=1

        # Build with retry logic
        for i in {1..3}; do
          echo "Build attempt $i/3"
          if make build_e2e; then
            echo "Build successful on attempt $i"
            break
          elif [ $i -eq 3 ]; then
            echo "Build failed after 3 attempts"
            exit 1
          else
            echo "Build attempt $i failed, retrying in 30 seconds..."
            sleep 30
          fi
        done

        # Start services with wait
        docker compose -f ./compose/e2e-mount.yml up --wait

    - name: Rotate buildx cache
      if: always()
      run: |
        # Without this, --cache-to writes to .buildx-cache-new but actions/cache only
        # uploads .buildx-cache, so layers (notably the slow apt RUN) never persist.
        rm -rf /tmp/.buildx-cache
        if [ -d /tmp/.buildx-cache-new ]; then mv /tmp/.buildx-cache-new /tmp/.buildx-cache; fi

    - name: Run FIO 4k
      timeout-minutes: 15
      run: |
        echo "Starting FIO at: $(date)"
        # Concurrent r/w
        echo 'Run randrw with size=16M bs=4k'
        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --group_reporting --runtime=30 --time_based=1

        echo "Verify FIO at: $(date)"
        # Verified write
        echo 'Run randwrite with size=16M bs=4k'
        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1

    - name: Run FIO 128k
      timeout-minutes: 15
      run: |
        echo "Starting FIO at: $(date)"
        # Concurrent r/w
        echo 'Run randrw with size=16M bs=128k'
        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1

        echo "Verify FIO at: $(date)"
        # Verified write
        echo 'Run randwrite with size=16M bs=128k'
        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1

    - name: Run FIO 1MB
      timeout-minutes: 15
      run: |
        echo "Starting FIO at: $(date)"
        # Concurrent r/w
        echo 'Run randrw with size=16M bs=1m'
        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
        
        echo "Verify FIO at: $(date)"
        # Verified write
        echo 'Run randwrite with size=16M bs=1m'
        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1

    - name: Save logs
      if: always()
      run: |
        docker compose -f ./compose/e2e-mount.yml logs > output.log
        echo 'Showing last 500 log lines of mount service:'
        docker compose -f ./compose/e2e-mount.yml logs --tail 500 mount

    - name: Check for data races
      if: always()
      continue-on-error: true # TODO: remove this comment to enable build failure on data races (after all are fixed)
      run: grep -A50 'DATA RACE' output.log && exit 1 || exit 0

    - name: Archive logs
      if: always()
      uses: actions/upload-artifact@v7
      with:
        name: output-logs
        path: docker/output.log

    - name: Cleanup
      if: always()
      run: docker compose -f ./compose/e2e-mount.yml down --volumes --remove-orphans --rmi all