grafana/.github/workflows/pr-e2e-tests.yml

name: End-to-end tests

on:
  pull_request:
  push:
    branches:
      - main
      - release-*.*.*

# TODO: re-enable this before merging
# concurrency:
#   group: ${{ github.workflow }}-${{ github.ref }}
#   cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}

permissions: {}

env:
  ACTIONS_STEP_DEBUG: true
  RUNNER_DEBUG: 1

jobs:
  detect-changes:
    name: Detect whether code changed
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      changed: ${{ steps.detect-changes.outputs.e2e }}
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: true # required to get more history in the changed-files action
          fetch-depth: 2
      - name: Detect changes
        id: detect-changes
        uses: ./.github/actions/change-detection
        with:
          self: .github/workflows/pr-e2e-tests.yml

  build-grafana:
    needs: detect-changes
    if: needs.detect-changes.outputs.changed == 'true'
    name: Build & Package Grafana
    runs-on: ubuntu-latest-16-cores
    permissions:
      contents: read
    outputs:
      artifact: ${{ steps.artifact.outputs.artifact }}
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false

      # TODO: add a cleanup workflow to remove the cache when the PR is closed
      # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#force-deletion-of-caches-overriding-default-cache-eviction-policy
      # TODO: maybe we could just use the cache to store the build, instead of uploading as an artifact?
      - uses: actions/cache@v4
        id: cache
        with:
          key: "build-grafana-${{ runner.os }}-${{ hashFiles('yarn.lock', 'public/*',  'packages/*', 'pkg/**/*.go', '**/go.mod', '**/go.sum', '!**_test.go', '!**.test.ts', '!**.test.tsx') }}"
          path: |
            build-dir

      # If no cache hit, build Grafana
      - name: Build Grafana
        if: steps.cache.outputs.cache-hit != 'true'
        uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
        with:
          verb: run
          args: go run ./pkg/build/cmd artifacts -a targz:grafana:linux/amd64 --grafana-dir="${PWD}" > out.txt
      - name: Cat built artifact
        if: steps.cache.outputs.cache-hit != 'true'
        run: cat out.txt
      - name: Move built artifact
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          mkdir -p build-dir
          mv "$(cat out.txt)" build-dir/grafana.tar.gz

      # If cache hit, validate the artifact is present
      - name: Validate artifact
        if: steps.cache.outputs.cache-hit == 'true'
        run: |
          if [ ! -f build-dir/grafana.tar.gz ]; then
            echo "Error: build-dir/grafana.tar.gz not found in cache"
            exit 1
          fi

      - name: Set artifact name
        run: echo "artifact=grafana-server-${{github.run_number}}" >> "$GITHUB_OUTPUT"
        id: artifact

      - name: Upload artifact
        uses: actions/upload-artifact@v4
        id: upload
        with:
          retention-days: 1
          name: ${{ steps.artifact.outputs.artifact }}
          path: build-dir/grafana.tar.gz

  # TODO: we won't need this when we only have playwright
  build-e2e-runner:
    needs: detect-changes
    if: needs.detect-changes.outputs.changed == 'true'
    name: Build E2E test runner
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      artifact: ${{ steps.artifact.outputs.artifact }}
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache: ${{ !github.event.pull_request.head.repo.fork }}
      - name: Build E2E test runner
        id: artifact
        run: |
          set -euo pipefail
          # We want a static binary, so we need to set CGO_ENABLED=0
          CGO_ENABLED=0 go build -o ./e2e-runner ./e2e/
          echo "artifact=e2e-runner-${{github.run_number}}" >> "$GITHUB_OUTPUT"
      - uses: actions/upload-artifact@v4
        id: upload
        with:
          retention-days: 1
          name: ${{ steps.artifact.outputs.artifact }}
          path: e2e-runner

  run-e2e-tests:
    needs:
      - build-grafana
      - build-e2e-runner
    strategy:
      fail-fast: false
      matrix:
        include:
          - suite: various-suite
            path: e2e/various-suite
          - suite: dashboards-suite
            path: e2e/dashboards-suite
          - suite: smoke-tests-suite
            path: e2e/smoke-tests-suite
          - suite: panels-suite
            path: e2e/panels-suite
          - suite: various-suite (old arch)
            path: e2e/old-arch/various-suite
            flags: --flags="--env dashboardScene=false"
          - suite: dashboards-suite (old arch)
            path: e2e/old-arch/dashboards-suite
            flags: --flags="--env dashboardScene=false"
          - suite: smoke-tests-suite (old arch)
            path: e2e/old-arch/smoke-tests-suite
            flags: --flags="--env dashboardScene=false"
          - suite: panels-suite (old arch)
            path: e2e/old-arch/panels-suite
            flags: --flags="--env dashboardScene=false"
    name: ${{ matrix.suite }}
    runs-on: ubuntu-latest-8-cores
    permissions:
      contents: read

    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: actions/download-artifact@v4
        with:
          name: ${{ needs.build-grafana.outputs.artifact }}
      - uses: actions/download-artifact@v4
        with:
          name: ${{ needs.build-e2e-runner.outputs.artifact }}
      - name: chmod +x
        run: chmod +x ./e2e-runner
      - name: Run E2E tests
        uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
        with:
          verb: run
          args: go run ./pkg/build/e2e --package=grafana.tar.gz
            --suite=${{ matrix.path }}
            ${{ matrix.flags }}
      - name: Set suite name
        id: set-suite-name
        if: success() || failure()
        env:
          SUITE: ${{ matrix.path }}
        run: |
          set -euo pipefail
          echo "suite=$(echo "$SUITE" | sed 's/\//-/g')" >> "$GITHUB_OUTPUT"
      - uses: actions/upload-artifact@v4
        if: success() || failure()
        with:
          name: ${{ steps.set-suite-name.outputs.suite }}-${{ github.run_number }}
          path: videos
          retention-days: 1

  run-playwright-tests:
    needs:
      - build-grafana
    name: Playwright E2E tests (${{ matrix.shard }}/${{ matrix.shardTotal }})
    runs-on: ubuntu-latest-8-cores
    permissions:
      contents: read

    strategy:
      fail-fast: false
      matrix:
        shard: [1, 2, 3, 4, 5, 6, 7, 8]
        shardTotal: [8]

    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: actions/download-artifact@v4
        with:
          name: ${{ needs.build-grafana.outputs.artifact }}
      - name: Run E2E tests
        uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
        with:
          verb: run
          args: go run ./pkg/build/e2e-playwright --package=grafana.tar.gz --shard=${{ matrix.shard }}/${{ matrix.shardTotal }} --blob-dir=./blob-report
      - uses: actions/upload-artifact@v4
        if: success() || failure()
        with:
          name: playwright-blob-${{ github.run_number }}-${{ matrix.shard }}
          path: ./blob-report
          retention-days: 1

  required-playwright-tests:
    needs:
      - run-playwright-tests
      - build-grafana
    if: ${{ !cancelled() }}
    name: All Playwright tests complete
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false

      - uses: actions/setup-node@v4
        with:
          node-version-file: '.nvmrc'

      - name: Download blob reports from GitHub Actions Artifacts
        uses: actions/download-artifact@v4
        with:
          path: blobs
          pattern: playwright-blob-*
          merge-multiple: true

      - name: Check blob reports
        run: |
          if [ ! "$(ls -A ./blobs)" ]; then
            echo "Error: No blob reports found in ./blobs directory"
            echo "Did the Playwright tests run at all?"
            exit 1
          fi
          echo "Found blob reports in ./blobs:"
          ls -lah ./blobs

      - name: Merge into HTML Report
        run: npx playwright merge-reports --reporter html ./blobs

      - name: Merge into JSON Report
        env:
          PLAYWRIGHT_JSON_OUTPUT_NAME: /tmp/playwright-results.json
        run: npx playwright merge-reports --reporter=json ./blobs

      - name: Bench report
        run: |
          docker run --rm \
            --volume="/tmp/playwright-results.json:/home/bench/tests/playwright-results.json" \
            us-docker.pkg.dev/grafanalabs-global/docker-grafana-bench-prod/grafana-bench:v0.5.1 report \
              --grafana-url "http://localhost:3000" \
              --grafana-version "CI- ${{ github.sha }}" \
              --test-suite-name "FrontendCore" \
              --report-input playwright \
              --report-output log \
              --log-level DEBUG \
              /home/bench/tests/playwright-results.json

      - name: Upload HTML report
        uses: actions/upload-artifact@v4
        with:
          name: playwright-html-${{ github.run_number }}
          path: playwright-report
          retention-days: 7

      - name: Check test suites
        id: check-jobs
        uses: ./.github/actions/check-jobs
        continue-on-error: true # Failure will be reported on Show test results step
        with:
          needs: ${{ toJson(needs) }}
          failure-message: "One or more E2E test suites have failed"
          success-message: "All E2E test suites completed successfully"

      - name: Show test results
        env:
          FAILED: ${{ steps.check-jobs.outputs.any-failed }}
        # sed removes the leading `../../src/` from the paths
        run: |
          npx playwright merge-reports --reporter list ./all-blob-reports | sed 's|\(\.\./\)\{1,\}src/|/|g'
          if [ "$FAILED" = "true" ]; then
            exit 1
          fi

  run-a11y-test:
    needs:
      - build-grafana
    name: A11y test
    runs-on: ubuntu-latest-8-cores
    permissions:
      contents: read

    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: actions/download-artifact@v4
        with:
          name: ${{ needs.build-grafana.outputs.artifact }}
      - name: Run PR a11y test
        if: github.event_name == 'pull_request'
        uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
        with:
          verb: run
          args: go run ./pkg/build/a11y --package=grafana.tar.gz
      - name: Run non-PR a11y test
        if: github.event_name != 'pull_request'
        uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
        with:
          verb: run
          args: go run ./pkg/build/a11y --package=grafana.tar.gz --no-threshold-fail

  # This is the job that is actually required by rulesets.
  # We want to only require one job instead of all the individual tests.
  # Future work also allows us to start skipping some tests based on changed files.
  required-e2e-tests:
    needs:
      - run-e2e-tests
      - build-grafana
      # a11y test is not listed on purpose: it is not an important E2E test.
      # It is also totally fine to fail right now.
    # always() is the best function here.
    # success() || failure() will skip this function if any need is also skipped.
    # That means conditional test suites will fail the entire requirement check.
    if: always()

    name: All E2E tests complete
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false

      - name: Check test suites
        uses: ./.github/actions/check-jobs
        with:
          needs: ${{ toJson(needs) }}
          failure-message: "One or more E2E test suites have failed"
          success-message: "All E2E test suites completed successfully"