upstream-mattermost/.github/workflows/e2e-tests-playwright-rolling-upgrades-template.yml

---
name: E2E Tests - Playwright Rolling Upgrades Template

# Tests ONE from-version, given the from-image (upgrade_from_server_image) and the to-image
# (SERVER_IMAGE, built from server_image_* inputs). The caller resolves the from-version matrix
# and invokes this workflow once per entry, so everything here — the test system io run, the
# commit status, the workers — belongs to a single from-version.
#
# Shape mirrors e2e-tests-playwright-template.yml, with the upgrade harness prepended to each
# worker: upgrade-from → upgrade-to leaves that worker's server on the to-image, and dispatch-run
# then draws normal specs against it. The harness itself never goes through test system io —
# upgrade-specs/ lives outside the Playwright testDir, so dispatch-begin cannot enqueue it.

on:
  workflow_call:
    inputs:
      upgrade_from_server_image:
        description: "Full from-image reference to upgrade FROM (e.g. mattermostdevelopment/mattermost-enterprise-edition:release-11.9)"
        type: string
        required: true
      upgrade_from_context_label:
        description: "From-version label for job/artifact names (e.g. release-11.9, release-11.7-esr)"
        type: string
        required: true
      context_name:
        description: "GitHub commit status context name for this from-version"
        type: string
        required: true
      workers:
        description: "Number of parallel test system io dispatch workers"
        type: number
        required: false
        default: 20

      commit_sha:
        type: string
        required: true
      branch:
        type: string
        required: true
      build_id:
        type: string
        required: true
      server_image_tag:
        description: "Server image tag to upgrade TO (e.g., master or short SHA)"
        type: string
        required: true
      server_edition:
        description: "Server edition: enterprise (default), fips, or team"
        type: string
        required: false
        default: enterprise
      server_image_repo:
        description: "Docker registry: mattermostdevelopment (default) or mattermost"
        type: string
        required: false
        default: mattermostdevelopment
      server_image_aliases:
        description: "Comma-separated alias tags for description"
        type: string
        required: false

      enable_reporting:
        type: boolean
        required: false
        default: false
      report_type:
        type: string
        required: false
      ref_branch:
        type: string
        required: false
      pr_number:
        type: string
        required: false

      playwright_project:
        description: "Playwright project name (passed to dispatch-begin metadata and dispatch-run --project=)."
        type: string
        required: false
        default: chrome
      playwright_retries:
        description: "Playwright --retries=N (per-spec, in-process retry of flaky tests)"
        type: number
        required: false
        default: 1
      retest_on_fail:
        description: "Re-dispatch failed dispatch units once (whole-spec retry, on top of Playwright --retries)"
        type: boolean
        required: false
        default: true

    secrets:
      MM_LICENSE:
        required: false
      REPORT_WEBHOOK_URL:
        required: false

# Callers must grant: contents: read, statuses: write, id-token: write
permissions:
  contents: read
  statuses: write
  id-token: write

env:
  SERVER_IMAGE: "${{ inputs.server_image_repo }}/${{ inputs.server_edition == 'fips' && 'mattermost-enterprise-fips-edition' || inputs.server_edition == 'team' && 'mattermost-team-edition' || 'mattermost-enterprise-edition' }}:${{ inputs.server_image_tag }}"
  # Job/workflow-level so ci/publish-webhook's `if` can see it (step env is applied after `if`).
  REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }}

jobs:
  prepare-run:
    runs-on: ubuntu-24.04
    permissions:
      contents: read
      id-token: write
      statuses: write
    outputs:
      composite-identity-json: ${{ steps.composite-identity.outputs.composite-identity-json }}
      workers-matrix: ${{ steps.matrix.outputs.workers }}
      start_time: ${{ steps.matrix.outputs.start_time }}
    steps:
      - name: ci/checkout-repo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          persist-credentials: false
          ref: ${{ inputs.commit_sha }}
          fetch-depth: 1
      - name: ci/composite-identity
        id: composite-identity
        env:
          CONTEXT_NAME: ${{ inputs.context_name }}
          GITHUB_REPOSITORY: ${{ github.repository }}
          MM_BRANCH: ${{ inputs.branch }}
          MM_SHA: ${{ inputs.commit_sha }}
          PR_NUMBER: ${{ inputs.pr_number }}
        run: |
          # Same derivation as e2e-tests-playwright-template.yml: drop the `e2e-test/` prefix (the
          # framework name already implies E2E in the dashboard) and swap remaining `/` for `-` so
          # the dashboard URL is path-safe. The from-version is already part of the context, so it
          # carries into the run name for free. The commit-status context itself stays unchanged —
          # branch protection and the override workflows depend on it.
          NAME="${CONTEXT_NAME#e2e-test/}"
          NAME="${NAME//\//-}"
          if [ -n "$PR_NUMBER" ]; then
            COMPOSITE_IDENTITY=$(jq -nc \
              --arg repo "${GITHUB_REPOSITORY}" \
              --arg sha "${MM_SHA}" \
              --arg run_id "${GITHUB_RUN_ID}" \
              --arg name "${NAME}" \
              --arg attempt "${GITHUB_RUN_ATTEMPT}" \
              --arg branch "${MM_BRANCH}" \
              --arg pr "${PR_NUMBER}" \
              '{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch, gh_pr_number:$pr}')
          else
            COMPOSITE_IDENTITY=$(jq -nc \
              --arg repo "${GITHUB_REPOSITORY}" \
              --arg sha "${MM_SHA}" \
              --arg run_id "${GITHUB_RUN_ID}" \
              --arg name "${NAME}" \
              --arg attempt "${GITHUB_RUN_ATTEMPT}" \
              --arg branch "${MM_BRANCH}" \
              '{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch}')
          fi
          echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> "$GITHUB_OUTPUT"
      - name: ci/matrix
        id: matrix
        env:
          INPUT_WORKERS: ${{ inputs.workers }}
        run: |
          echo "workers=$(jq -nc --argjson n "${INPUT_WORKERS}" '[range(1; $n+1)]')" >> "$GITHUB_OUTPUT"
          echo "start_time=$(date +%s)" >> "$GITHUB_OUTPUT"

  # Install webapp node_modules once via the shared webapp-setup action, then
  # workers restore the same stable cache. The node_modules cache is keyed only
  # on webapp/package-lock.json and is shared with webapp-ci.yml jobs.
  prep-deps:
    name: prep-deps
    runs-on: ubuntu-24.04
    timeout-minutes: 15
    permissions:
      contents: read
    steps:
      - name: ci/checkout-repo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          persist-credentials: false
          ref: ${{ inputs.commit_sha }}
          fetch-depth: 1
      - name: ci/setup-webapp-node-modules
        uses: ./.github/actions/webapp-setup
      - name: ci/cache-playwright-deps
        id: cache-playwright
        uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
        with:
          path: |
            e2e-tests/playwright/node_modules
            e2e-tests/playwright/lib/dist
            e2e-tests/playwright/lib/node_modules
          key: e2e-playwright-deps-${{ runner.os }}-${{ hashFiles('e2e-tests/playwright/package-lock.json', 'e2e-tests/playwright/lib/src/**', 'e2e-tests/playwright/lib/package.json', 'e2e-tests/playwright/lib/rollup.config.js', 'e2e-tests/playwright/lib/tsconfig.json') }}
      - name: ci/install-playwright-deps
        if: steps.cache-playwright.outputs.cache-hit != 'true'
        working-directory: e2e-tests/playwright
        env:
          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1"
        run: npm ci
      - name: ci/cache-playwright-browsers
        id: cache-pw-browsers
        uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
        with:
          path: ~/.cache/ms-playwright
          key: playwright-browsers-${{ runner.os }}-${{ hashFiles('e2e-tests/playwright/package-lock.json') }}
      - name: ci/install-playwright-chromium
        if: steps.cache-pw-browsers.outputs.cache-hit != 'true'
        working-directory: e2e-tests/playwright
        run: npx playwright install chromium

  # Register this from-version's run AFTER prep-deps so workers reach dispatch-run within test
  # system io's inactivity window. Workers here have the upgrade harness to get through first, so
  # the gap is wider than the full suite's — see the harness note on the workers job.
  dispatch-begin:
    runs-on: ubuntu-24.04
    needs: [prepare-run, prep-deps]
    permissions:
      contents: read
      id-token: write
      statuses: write
    steps:
      - name: ci/checkout-repo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          persist-credentials: false
          ref: ${{ inputs.commit_sha }}
          fetch-depth: 1
      - name: ci/dispatch-begin
        uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-dispatch-begin@1631d8fcea24f4545a0b3b7f77e41c2fe0be4418 # 2026-07-28
        with:
          use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }}
          framework: playwright
          repo-dir: ${{ github.workspace }}
          composite-identity: ${{ needs.prepare-run.outputs.composite-identity-json }}
          total-reports-expected: ${{ inputs.workers }}
          retest-on-fail: ${{ inputs.retest_on_fail }}
          playwright-project: ${{ inputs.playwright_project }}
          github-token: ${{ secrets.GITHUB_TOKEN }}
          commit-status-context: ${{ inputs.context_name }}
          image-tag: ${{ inputs.server_image_tag }}
          image-aliases: ${{ inputs.server_image_aliases }}

  workers:
    name: rolling-upgrade-${{ inputs.upgrade_from_context_label }}-${{ matrix.worker_index }}
    runs-on: ubuntu-24.04
    # Longer than the full suite's 30m workers: each one upgrades a server before it draws its
    # first spec, which costs two image pulls plus the swap.
    timeout-minutes: 60
    needs: [prepare-run, dispatch-begin]
    permissions:
      contents: read
      id-token: write
    strategy:
      fail-fast: false
      matrix:
        worker_index: ${{ fromJSON(needs.prepare-run.outputs.workers-matrix) }}
    env:
      COMPOSITE_IDENTITY: ${{ needs.prepare-run.outputs.composite-identity-json }}
      MM_LICENSE: "${{ secrets.MM_LICENSE }}"
      TEST: playwright
      BRANCH: "${{ inputs.branch }}"
      BUILD_ID: "${{ inputs.build_id }}"
      CI_BASE_URL: "rolling-upgrade-${{ inputs.upgrade_from_context_label }}-${{ matrix.worker_index }}"
      # The from-image this worker boots on. The to-image it ends up on is SERVER_IMAGE, which
      # testConfig picks up for the swap.
      PW_UPGRADE_FROM_SERVER_IMAGE: "${{ inputs.upgrade_from_server_image }}"
      PW_USE_TESTCONTAINERS: "true"
      # Same fixed set as the full suite's workers: dispatch pulls specs from a shared queue, so no
      # worker knows in advance which service-tagged spec (e.g. @ldap, @saml) it will draw.
      PW_TESTCONTAINERS_SERVICES: "openldap,keycloak,elasticsearch,opensearch,minio,azurite"
      # Load-bearing three times over here: upgrade-to has to adopt the stack upgrade-from left
      # running, the swap has to keep Postgres and file storage while only the server is replaced,
      # and dispatch-run's per-spec invocations all have to land on that same upgraded server.
      PW_TESTCONTAINERS_REUSE: "true"
    steps:
      - name: ci/checkout-actions
        # Sparse-checkout just .github/actions from the triggering ref (master)
        # so the composite action below is available before the full checkout
        # overwrites the workspace with inputs.commit_sha.
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          persist-credentials: false
          sparse-checkout: .github/actions
          sparse-checkout-cone-mode: true
      - name: ci/runner-prep-for-openldap
        uses: ./.github/actions/runner-prep-openldap
      - name: ci/checkout-repo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          persist-credentials: false
          ref: ${{ inputs.commit_sha }}
          fetch-depth: 0
      - name: ci/setup-webapp-node-modules
        uses: ./.github/actions/webapp-setup
        with:
          read-only: "true"
      - name: ci/restore-playwright-deps
        uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
        with:
          path: |
            e2e-tests/playwright/node_modules
            e2e-tests/playwright/lib/dist
            e2e-tests/playwright/lib/node_modules
          key: e2e-playwright-deps-${{ runner.os }}-${{ hashFiles('e2e-tests/playwright/package-lock.json', 'e2e-tests/playwright/lib/src/**', 'e2e-tests/playwright/lib/package.json', 'e2e-tests/playwright/lib/rollup.config.js', 'e2e-tests/playwright/lib/tsconfig.json') }}
          fail-on-cache-miss: true
      - name: ci/restore-playwright-browsers
        uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
        with:
          path: ~/.cache/ms-playwright
          key: playwright-browsers-${{ runner.os }}-${{ hashFiles('e2e-tests/playwright/package-lock.json') }}
          fail-on-cache-miss: true
      # Boots the stack on the from-image (global setup) and seeds the baseline. Runs the `setup`
      # project as a project dependency. No testcontainers:down first — the runner is fresh, so
      # upgrade-from's clean-slate guard has nothing to trip over.
      - name: ci/upgrade-from
        working-directory: e2e-tests/playwright
        run: npx playwright test --project=upgrade-from
      # Swaps this worker's server to SERVER_IMAGE in place and re-verifies the baseline.
      - name: ci/upgrade-to
        working-directory: e2e-tests/playwright
        run: npx playwright test --project=upgrade-to
      # From here the server is the to-image, upgraded in place, and the rest of the job is the
      # normal full suite against it. Nothing is re-prepared in between: an upgrade that needed the
      # suite's setup re-run to be usable would not be a passing upgrade. The stack is already up
      # and seeded from upgrade-from (whose own `setup` dependency ran there), and each dispatched
      # spec brings its own config through pw.initSetup().
      #
      # Skipped automatically when the harness above failed, since a server that did not survive
      # its own upgrade has nothing useful to say about the suite.
      - name: ci/dispatch-run
        uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-dispatch-run@1631d8fcea24f4545a0b3b7f77e41c2fe0be4418 # 2026-07-28
        with:
          use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }}
          framework: playwright
          composite-identity: ${{ needs.prepare-run.outputs.composite-identity-json }}
          repo-dir: ${{ github.workspace }}
          artifacts-root: ${{ github.workspace }}/worker-artifacts
          github-token: ${{ secrets.GITHUB_TOKEN }}
          gh-job-name: rolling-upgrade-${{ inputs.upgrade_from_context_label }}-${{ matrix.worker_index }}
          playwright-retries: ${{ inputs.playwright_retries }}
          playwright-project: ${{ inputs.playwright_project }}
      - name: ci/upload-debug-artifacts
        if: always()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          # Includes logs/upgrade/{from,to}-mattermost.log (+ highlights, postgres, from-baseline.json)
          # captured by saveUpgradePhaseLogs during upgrade-from / upgrade-swap-to / upgrade-to.
          name: playwright-rolling-upgrades-${{ inputs.server_edition }}-from-${{ inputs.upgrade_from_context_label }}-debug-${{ matrix.worker_index }}
          path: |
            e2e-tests/playwright/logs/
            e2e-tests/playwright/results/
            worker-artifacts/
          retention-days: 5
          if-no-files-found: ignore

  report:
    runs-on: ubuntu-24.04
    needs: [prepare-run, dispatch-begin, workers]
    if: always()
    permissions:
      contents: read
      id-token: write
      statuses: write
    outputs:
      commit_status_description: ${{ steps.summary.outputs.commit_status_description }}
      webhook_payload: ${{ steps.summary.outputs.webhook_payload }}
    steps:
      - name: ci/checkout-repo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          persist-credentials: false
      - name: ci/run-summary
        id: summary
        continue-on-error: true
        uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-summary@e2d5032cfa71a3ad11b0975b135cd221564f2787 # 2026-08-06
        with:
          use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }}
          composite-identity: ${{ needs.prepare-run.outputs.composite-identity-json }}
          framework: playwright
          report-type: ${{ inputs.report_type }}
          image-tag: ${{ inputs.server_image_tag }}
          image-aliases: ${{ inputs.server_image_aliases }}
          server-image: ${{ env.SERVER_IMAGE }}
          pr-number: ${{ inputs.pr_number }}
          ref-branch: ${{ inputs.ref_branch }}
          commit-status-context: ${{ inputs.context_name }}
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - name: ci/publish-webhook
        if: inputs.enable_reporting && env.REPORT_WEBHOOK_URL != ''
        env:
          PAYLOAD: ${{ steps.summary.outputs.webhook_payload }}
        run: |
          curl -X POST -H "Content-Type: application/json" -d "$PAYLOAD" "$REPORT_WEBHOOK_URL"
      - name: ci/assert-results
        env:
          SUMMARY_OUTCOME: ${{ steps.summary.outcome }}
        run: |
          [ "$SUMMARY_OUTCOME" = "success" ]