From cea1f8443738e9e4b375c065dd4a1e466036bb74 Mon Sep 17 00:00:00 2001 From: joshhunt Date: Mon, 30 Jun 2025 16:17:46 +0100 Subject: [PATCH] wip --- .github/workflows/playwright-e2e.yml | 146 ++++++++++++++++++++++ pkg/build/e2e-playwright/README.md | 22 ++++ pkg/build/e2e-playwright/main.go | 177 +++++++++++++++++++++++++++ pkg/build/e2e-playwright/run-e2e.go | 79 ++++++++++++ pkg/build/e2e-playwright/service.go | 51 ++++++++ 5 files changed, 475 insertions(+) create mode 100644 .github/workflows/playwright-e2e.yml create mode 100644 pkg/build/e2e-playwright/README.md create mode 100644 pkg/build/e2e-playwright/main.go create mode 100644 pkg/build/e2e-playwright/run-e2e.go create mode 100644 pkg/build/e2e-playwright/service.go diff --git a/.github/workflows/playwright-e2e.yml b/.github/workflows/playwright-e2e.yml new file mode 100644 index 00000000000..35ce28c0a23 --- /dev/null +++ b/.github/workflows/playwright-e2e.yml @@ -0,0 +1,146 @@ +name: Playwright E2E Tests + +on: + pull_request: + push: + branches: + - main + - release-*.*.* + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +permissions: {} + +jobs: + build-grafana: + name: Build & Package Grafana + runs-on: ubuntu-latest-16-cores + permissions: + contents: read + outputs: + artifact: ${{ steps.artifact.outputs.artifact }} + steps: + - uses: actions/checkout@v4 + with: + path: ./grafana + persist-credentials: false + - uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e + with: + verb: run + args: go -C grafana run ./pkg/build/cmd artifacts -a targz:grafana:linux/amd64 --grafana-dir="${PWD}/grafana" > out.txt + - run: mv "$(cat out.txt)" grafana.tar.gz + - run: echo "artifact=grafana-playwright-${{github.run_number}}" >> "$GITHUB_OUTPUT" + id: artifact + - uses: actions/upload-artifact@v4 + id: upload + with: + retention-days: 1 + name: ${{ steps.artifact.outputs.artifact }} + path: grafana.tar.gz + + run-playwright-tests: + needs: + - build-grafana + strategy: + fail-fast: false + matrix: + include: + - suite: plugin-e2e-api-tests + project: admin + - suite: plugin-e2e-api-tests + project: viewer + - suite: plugin-e2e + project: elasticsearch + - suite: plugin-e2e + project: mysql + - suite: plugin-e2e + project: mssql + - suite: plugin-e2e + project: cloudwatch + - suite: plugin-e2e + project: azuremonitor + - suite: plugin-e2e + project: cloudmonitoring + - suite: plugin-e2e + project: graphite + - suite: plugin-e2e + project: influxdb + - suite: plugin-e2e + project: jaeger + - suite: plugin-e2e + project: zipkin + - suite: core-tests + project: panels + - suite: core-tests + project: smoke + - suite: core-tests + project: dashboards + - suite: core-tests + project: loki + - suite: test-plugins + project: extensions-test-app + - suite: test-plugins + project: grafana-e2etest-datasource + name: ${{ matrix.suite }}-${{ matrix.project }} + runs-on: ubuntu-latest-8-cores + permissions: + contents: read + + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/download-artifact@v4 + with: + name: ${{ needs.build-grafana.outputs.artifact }} + - name: Run Playwright E2E tests + uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e + with: + verb: run + args: go run ./pkg/build/daggerbuild/playwright run --package=grafana.tar.gz --project=${{ matrix.project }} + - name: Set suite name + id: set-suite-name + if: success() || failure() + env: + SUITE: ${{ matrix.suite }}-${{ matrix.project }} + run: | + set -euo pipefail + echo "suite=$(echo "$SUITE" | sed 's/\//-/g')" >> "$GITHUB_OUTPUT" + - uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: playwright-report-${{ steps.set-suite-name.outputs.suite }}-${{ github.run_number }} + path: playwright-report + retention-days: 3 + - uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: test-results-${{ steps.set-suite-name.outputs.suite }}-${{ github.run_number }} + path: test-results + retention-days: 3 + + # This is the job that is actually required by rulesets. + # We want to only require one job instead of all the individual tests. + required-playwright-tests: + needs: + - run-playwright-tests + # always() is the best function here. + # success() || failure() will skip this function if any need is also skipped. + # That means conditional test suites will fail the entire requirement check. + if: always() + + name: All Playwright E2E tests complete + runs-on: ubuntu-latest + steps: + - name: Check test suites + env: + NEEDS: ${{ toJson(needs) }} + run: | + FAILURES="$(echo "$NEEDS" | jq 'with_entries(select(.value.result == "failure")) | map_values(.result)')" + echo "$FAILURES" + if [ "$(echo "$FAILURES" | jq '. | length')" != "0" ]; then + exit 1 + fi + echo "All OK!" \ No newline at end of file diff --git a/pkg/build/e2e-playwright/README.md b/pkg/build/e2e-playwright/README.md new file mode 100644 index 00000000000..d5fc311d696 --- /dev/null +++ b/pkg/build/e2e-playwright/README.md @@ -0,0 +1,22 @@ +# Pa11y accessability tests + +We use pa11y to run some automated simple accessability tests. They're ran with dagger to help orchestrate starting server + tests in a reproducable manner. + +To run the tests locally: + +1. Install dagger locally https://docs.dagger.io/install/ +2. Grab the grafana.tar.gz artifact by either + 1. Downloading it from the Github Action artifact from your PR + 1. Build it locally with: + ```sh + dagger run go run ./pkg/build/cmd artifacts -a targz:grafana:linux/amd64 --grafana-dir="$PWD" > dist/files.txt + cat dist/files.txt # Will output the path to the grafana.tar.gz + ``` +3. Run the dagger pipeline with: + ```sh + dagger -v run go run ./pkg/build/a11y --package=(full path to .tar.gz) --results=./pa11y-ci-results.json + ``` + The JSON results file will be saved to the file from the `--results` arg +4. If they fail and you want to see the full output + 1. Run the dagger command with `dagger -vE [...]` + 2. At the end, arrow up to the exec pa11y-ci segment and hit Enter \ No newline at end of file diff --git a/pkg/build/e2e-playwright/main.go b/pkg/build/e2e-playwright/main.go new file mode 100644 index 00000000000..2f5855f1f82 --- /dev/null +++ b/pkg/build/e2e-playwright/main.go @@ -0,0 +1,177 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "os/signal" + + "dagger.io/dagger" + "github.com/urfave/cli/v3" +) + +var ( + grafanaHost = "grafana" + grafanaPort = 3001 +) + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + if err := NewApp().Run(ctx, os.Args); err != nil { + cancel() + fmt.Println(err) + os.Exit(1) + } +} + +func NewApp() *cli.Command { + return &cli.Command{ + Name: "a11y", + Usage: "Run Grafana playwright e2e tests", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "grafana-dir", + Usage: "Path to the grafana/grafana clone directory", + Value: ".", + Validator: mustBeDir("grafana-dir"), + TakesFile: true, + }, + &cli.StringFlag{ + Name: "package", + Usage: "Path to the grafana tar.gz package", + Value: "grafana.tar.gz", + Validator: mustBeFile("package", false), + TakesFile: true, + }, + &cli.StringFlag{ + Name: "license", + Usage: "Path to the Grafana Enterprise license file (optional)", + Validator: mustBeFile("license", true), + TakesFile: true, + }, + // &cli.StringFlag{ + // Name: "config", + // Usage: "Path to the pa11y config file to use", + // Value: "e2e/pa11yci.conf.js", + // Validator: mustBeFile("config", true), + // TakesFile: true, + // }, + // &cli.StringFlag{ + // Name: "results", + // Usage: "Path to the pa11y results file to export", + // TakesFile: true, + // }, + // &cli.BoolFlag{ + // Name: "no-threshold-fail", + // Usage: "Don't fail the task if any of the tests fail. Use this in combination with --results to list all violations even if they're within thresholds", + // Value: false, + // }, + }, + Action: run, + } +} + +func run(ctx context.Context, cmd *cli.Command) error { + grafanaDir := cmd.String("grafana-dir") + targzPath := cmd.String("package") + licensePath := cmd.String("license") + // pa11yConfigPath := cmd.String("config") + // pa11yResultsPath := cmd.String("results") + // noThresholdFail := cmd.Bool("no-threshold-fail") + + d, err := dagger.Connect(ctx) + if err != nil { + return fmt.Errorf("failed to connect to Dagger: %w", err) + } + + // Explicitly only the files used by the grafana-server service + hostSrc := d.Host().Directory(grafanaDir, dagger.HostDirectoryOpts{ + Include: []string{ + "./devenv", + "./e2e/test-plugins", // Directory is included so provisioning works, but they're not actually build + "./scripts/grafana-server/custom.ini", + "./scripts/grafana-server/start-server", + "./scripts/grafana-server/kill-server", + "./scripts/grafana-server/variables", + }, + }) + + targz := d.Host().File(targzPath) + // pa11yConfig := d.Host().File(pa11yConfigPath) + + var license *dagger.File + if licensePath != "" { + license = d.Host().File(licensePath) + } + + svc, err := GrafanaService(ctx, d, GrafanaServiceOpts{ + HostSrc: hostSrc, + GrafanaTarGz: targz, + License: license, + }) + if err != nil { + return fmt.Errorf("failed to create Grafana service: %w", err) + } + + c, runErr := RunTest(ctx, d, svc) + if runErr != nil { + return fmt.Errorf("failed to run a11y test suite: %w", runErr) + } + + c, syncErr := c.Sync(ctx) + if syncErr != nil { + return fmt.Errorf("failed to sync a11y test suite: %w", syncErr) + } + + code, codeErr := c.ExitCode(ctx) + if codeErr != nil { + return fmt.Errorf("failed to get exit code of a11y test suite: %w", codeErr) + } + + if code == 0 { + log.Printf("a11y tests passed with exit code %d", code) + } else { + return fmt.Errorf("a11y tests failed with exit code %d", code) + } + + log.Println("a11y tests completed successfully") + return nil +} + +func mustBeFile(arg string, emptyOk bool) func(string) error { + return func(s string) error { + if s == "" { + if emptyOk { + return nil + } + return cli.Exit(arg+" cannot be empty", 1) + } + stat, err := os.Stat(s) + if err != nil { + return cli.Exit(arg+" does not exist or cannot be read: "+s, 1) + } + if stat.IsDir() { + return cli.Exit(arg+" must be a file, not a directory: "+s, 1) + } + return nil + } +} + +func mustBeDir(arg string) func(string) error { + return func(s string) error { + if s == "" { + return cli.Exit(arg+" cannot be empty", 1) + } + stat, err := os.Stat(s) + if err != nil { + return cli.Exit(arg+" does not exist or cannot be read: "+s, 1) + } + if !stat.IsDir() { + return cli.Exit(arg+" must be a directory: "+s, 1) + } + return nil + } +} diff --git a/pkg/build/e2e-playwright/run-e2e.go b/pkg/build/e2e-playwright/run-e2e.go new file mode 100644 index 00000000000..883a6eced2e --- /dev/null +++ b/pkg/build/e2e-playwright/run-e2e.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "fmt" + "strings" + + "dagger.io/dagger" +) + +// NodeVersionContainer returns a container whose `stdout` will return the node version from the '.nvmrc' file in the directory 'src'. +func NodeVersion(d *dagger.Client, src *dagger.File) *dagger.Container { + return d.Container().From("alpine:3"). + WithMountedFile("/src/.nvmrc", src). + WithWorkdir("/src"). + WithExec([]string{"cat", ".nvmrc"}) +} + +func NodeImage(version string) string { + return fmt.Sprintf("node:%s-slim", strings.TrimPrefix(strings.TrimSpace(version), "v")) +} + +func RunTest( + ctx context.Context, + d *dagger.Client, + grafanaService *dagger.Service, +) (*dagger.Container, error) { + + // Explicitly only the files u'sed by e2e tests + hostSrc := d.Host().Directory(".", dagger.HostDirectoryOpts{ + Include: []string{ + // Include all files for a valid yarn workspace install + "package.json", + "yarn.lock", + ".yarnrc.yml", + ".yarn", + "packages/*/package.json", + "public/app/plugins/*/*/package.json", + "e2e/test-plugins/*/package.json", + ".nvmrc", + "public/app/types/*.d.ts", + + // packages we use in playwright tests + "packages", + + // e2e files + "e2e-playwright", + }, + Exclude: []string{ + "packages/*/dist", + }, + }) + + nodeVersion, err := NodeVersion(d, hostSrc.File(".nvmrc")).Stdout(ctx) + if err != nil { + return nil, err + } + + yarnCache := d.CacheVolume("yarn-cache") + yarnCacheDir := "/yarn-cache" + + pa11yContainer := d.Container().From(NodeImage(nodeVersion)). + WithExec([]string{"npx", "-y", "playwright@1.52.0", "install", "--with-deps"}). // TODO: sync version from package.json + WithWorkdir("/src"). + WithDirectory("/src", hostSrc). + WithMountedCache(yarnCacheDir, yarnCache). + WithExec([]string{"corepack", "enable"}). + WithExec([]string{"corepack", "install"}). + WithEnvVariable("YARN_CACHE_FOLDER", yarnCacheDir). + WithExec([]string{"yarn", "config", "get", "cacheFolder"}). + WithExec([]string{"yarn", "install", "--immutable"}). + // WithExec([]string{"yarn", "e2e:plugin:build"}). + WithEnvVariable("HOST", grafanaHost). + WithEnvVariable("PORT", fmt.Sprint(grafanaPort)). + WithExec([]string{"yarn", "packages:build"}). + WithExec([]string{"yarn", "e2e:playwright"}) + + return pa11yContainer, nil +} diff --git a/pkg/build/e2e-playwright/service.go b/pkg/build/e2e-playwright/service.go new file mode 100644 index 00000000000..c866bc60520 --- /dev/null +++ b/pkg/build/e2e-playwright/service.go @@ -0,0 +1,51 @@ +package main + +import ( + "context" + "fmt" + "os" + "strings" + + "dagger.io/dagger" +) + +type GrafanaServiceOpts struct { + HostSrc *dagger.Directory + GrafanaTarGz *dagger.File + License *dagger.File +} + +func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOpts) (*dagger.Service, error) { + container := d.Container().From("alpine:3"). + WithExec([]string{"apk", "add", "--no-cache", "bash", "tar", "netcat-openbsd"}). + WithMountedFile("/src/grafana.tar.gz", opts.GrafanaTarGz). + WithExec([]string{"mkdir", "-p", "/src/grafana"}). + WithExec([]string{"tar", "--strip-components=1", "-xzf", "/src/grafana.tar.gz", "-C", "/src/grafana"}). + WithDirectory("/src/grafana/devenv", opts.HostSrc.Directory("./devenv")). + WithDirectory("/src/grafana/e2e/test-plugins", opts.HostSrc.Directory("./e2e/test-plugins")). + WithDirectory("/src/grafana/scripts", opts.HostSrc.Directory("./scripts")). + WithWorkdir("/src/grafana"). + WithEnvVariable("GF_APP_MODE", "development"). + WithEnvVariable("GF_SERVER_HTTP_PORT", fmt.Sprint(grafanaPort)). + WithEnvVariable("GF_SERVER_ROUTER_LOGGING", "1"). + WithExposedPort(grafanaPort) + + var licenseArg string + if opts.License != nil { + licenseArg = "/src/license.jwt" + container = container.WithMountedFile(licenseArg, opts.License) + } + + // We add all GF_ environment variables to allow for overriding Grafana configuration. + // It is unlikely the runner has any such otherwise. + for _, env := range os.Environ() { + if strings.HasPrefix(env, "GF_") { + parts := strings.SplitN(env, "=", 2) + container = container.WithEnvVariable(parts[0], parts[1]) + } + } + + svc := container.AsService(dagger.ContainerAsServiceOpts{Args: []string{"bash", "-x", "scripts/grafana-server/start-server", licenseArg}}) + + return svc, nil +}