CI: Fix pa11y check by running in docker-puppeteer image (#107084)

* Change pa11y dagger to run in docker-puppeteer image

* export json results

* re-enable pa11y ci check

* update gha workflow to new flags

* add no-threshold-fail, use single pa11y config

* fix codeowners

* readme

* fix drone config

pkg/build/a11y/README.md

@@ -0,0 +1,22 @@
+# Pa11y accessability tests
+
+We use pa11y to run some automated simple accessability tests. They're ran with dagger to help orchestrate starting server + tests in a reproducable manner.
+
+To run the tests locally:
+
+1. Install dagger locally https://docs.dagger.io/install/
+2. Grab the grafana.tar.gz artifact by either
+   1. Downloading it from the Github Action artifact from your PR
+   1. Build it locally with:
+      ```sh
+      dagger run go run ./pkg/build/cmd artifacts -a targz:grafana:linux/amd64 --grafana-dir="$PWD" > dist/files.txt
+      cat dist/files.txt # Will output the path to the grafana.tar.gz 
+      ```
+3. Run the dagger pipeline with:
+   ```sh
+   dagger -v run go run ./pkg/build/a11y --package=(full path to .tar.gz) --results=./pa11y-ci-results.json
+   ```
+   The JSON results file will be saved to the file from the `--results` arg 
+4. If they fail and you want to see the full output
+   1. Run the dagger command with `dagger -vE [...]`
+   2. At the end, arrow up to the exec pa11y-ci segment and hit Enter

pkg/build/a11y/main.go

@@ -6,12 +6,16 @@ import (
 	"log"
 	"os"
 	"os/signal"
-	"path/filepath"
 
 	"dagger.io/dagger"
 	"github.com/urfave/cli/v3"
 )
 
+var (
+	grafanaHost = "grafana"
+	grafanaPort = 3001
+)
+
 func main() {
 	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
 	defer cancel()
@@ -49,8 +53,21 @@ func NewApp() *cli.Command {
 				TakesFile: true,
 			},
 			&cli.StringFlag{
-				Name:  "flags",
-				Usage: "Flags to pass through to the e2e runner",
+				Name:      "config",
+				Usage:     "Path to the pa11y config file to use",
+				Value:     "e2e/pa11yci.conf.js",
+				Validator: mustBeFile("config", true),
+				TakesFile: true,
+			},
+			&cli.StringFlag{
+				Name:      "results",
+				Usage:     "Path to the pa11y results file to export",
+				TakesFile: true,
+			},
+			&cli.BoolFlag{
+				Name:  "no-threshold-fail",
+				Usage: "Don't fail the task if any of the tests fail. Use this in combination with --results to list all violations even if they're within thresholds",
+				Value: false,
 			},
 		},
 		Action: run,
@@ -61,26 +78,29 @@ func run(ctx context.Context, cmd *cli.Command) error {
 	grafanaDir := cmd.String("grafana-dir")
 	targzPath := cmd.String("package")
 	licensePath := cmd.String("license")
-	runnerFlags := cmd.String("flags")
+	pa11yConfigPath := cmd.String("config")
+	pa11yResultsPath := cmd.String("results")
+	noThresholdFail := cmd.Bool("no-threshold-fail")
 
 	d, err := dagger.Connect(ctx)
 	if err != nil {
 		return fmt.Errorf("failed to connect to Dagger: %w", err)
 	}
 
-	yarnCache := d.CacheVolume("yarn")
-
-	//nolint:gosec
-	nvmrcContents, err := os.ReadFile(filepath.Join(grafanaDir, ".nvmrc"))
-	if err != nil {
-		return fmt.Errorf("failed to read .nvmrc file: %w", err)
-	}
-	nodeVersion := string(nvmrcContents)
-
-	grafana := d.Host().Directory(grafanaDir, dagger.HostDirectoryOpts{
-		Exclude: []string{"node_modules", "*.tar.gz"},
+	// Explicitly only the files used by the grafana-server service
+	hostSrc := d.Host().Directory(grafanaDir, dagger.HostDirectoryOpts{
+		Include: []string{
+			"./devenv",
+			"./e2e/test-plugins", // Directory is included so provisioning works, but they're not actually build
+			"./scripts/grafana-server/custom.ini",
+			"./scripts/grafana-server/start-server",
+			"./scripts/grafana-server/kill-server",
+			"./scripts/grafana-server/variables",
+		},
 	})
+
 	targz := d.Host().File(targzPath)
+	pa11yConfig := d.Host().File(pa11yConfigPath)
 
 	var license *dagger.File
 	if licensePath != "" {
@@ -88,27 +108,34 @@ func run(ctx context.Context, cmd *cli.Command) error {
 	}
 
 	svc, err := GrafanaService(ctx, d, GrafanaServiceOpts{
-		GrafanaDir:   grafana,
+		HostSrc:      hostSrc,
 		GrafanaTarGz: targz,
 		License:      license,
-		YarnCache:    yarnCache,
-		NodeVersion:  nodeVersion,
 	})
 	if err != nil {
 		return fmt.Errorf("failed to create Grafana service: %w", err)
 	}
 
-	c := RunTest(d, svc, grafana, yarnCache, nodeVersion, runnerFlags)
-	c, err = c.Sync(ctx)
-	if err != nil {
-		return fmt.Errorf("failed to run a11y test suite: %w", err)
+	c, runErr := RunTest(ctx, d, svc, pa11yConfig, noThresholdFail, pa11yResultsPath)
+	if runErr != nil {
+		return fmt.Errorf("failed to run a11y test suite: %w", runErr)
 	}
 
-	code, err := c.ExitCode(ctx)
-	if err != nil {
-		return fmt.Errorf("failed to get exit code of a11y test suite: %w", err)
+	c, syncErr := c.Sync(ctx)
+	if syncErr != nil {
+		return fmt.Errorf("failed to sync a11y test suite: %w", syncErr)
 	}
-	if code != 0 {
+
+	code, codeErr := c.ExitCode(ctx)
+	if codeErr != nil {
+		return fmt.Errorf("failed to get exit code of a11y test suite: %w", codeErr)
+	}
+
+	if code == 0 {
+		log.Printf("a11y tests passed with exit code %d", code)
+	} else if noThresholdFail {
+		log.Printf("a11y tests failed with exit code %d, but noFail is true", code)
+	} else {
 		return fmt.Errorf("a11y tests failed with exit code %d", code)
 	}
 

pkg/build/a11y/run.go

@@ -1,25 +1,48 @@
 package main
 
 import (
+	"context"
 	"fmt"
 
 	"dagger.io/dagger"
 )
 
 func RunTest(
+	ctx context.Context,
 	d *dagger.Client,
-	svc *dagger.Service,
-	src *dagger.Directory, cache *dagger.CacheVolume,
-	nodeVersion, runnerFlags string) *dagger.Container {
-	command := fmt.Sprintf(
-		"./e2e-runner a11y --start-grafana=false"+
-			" --grafana-host grafana --grafana-port 3001 %s", runnerFlags)
-
-	return GrafanaFrontend(d, cache, nodeVersion, src).
-		WithExec([]string{"/bin/sh", "-c", "apt-get update && apt-get install -y git curl"}).
-		WithExec([]string{"curl", "-LO", "https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb"}).
-		WithExec([]string{"apt-get", "install", "-y", "./google-chrome-stable_current_amd64.deb"}).
+	grafanaService *dagger.Service,
+	pa11yConfig *dagger.File,
+	noThresholdFail bool,
+	pa11yResultsPath string,
+) (*dagger.Container, error) {
+	// docker-puppeteer container already has Chrome and Pa11y installed in it
+	pa11yContainer := d.Container().From("grafana/docker-puppeteer:1.1.0").
 		WithWorkdir("/src").
-		WithServiceBinding("grafana", svc).
-		WithExec([]string{"/bin/bash", "-c", command}, dagger.ContainerWithExecOpts{Expect: dagger.ReturnTypeAny})
+		WithExec([]string{"mkdir", "-p", "./screenshots"}). // not yet exported
+		WithEnvVariable("HOST", grafanaHost).
+		WithEnvVariable("PORT", fmt.Sprint(grafanaPort))
+
+	if noThresholdFail {
+		// This logic is non-intuitive - --no-threshold-fail will make pa11y fail (by removing thresholds from the config)
+		// so it can write all violations to the results file. This failure is then ignored by the caller in main.go.
+		// Otherwise, pa11y ignores violations if they're within the thresholds and doesn't include them in the results file
+		pa11yContainer = pa11yContainer.
+			WithEnvVariable("NO_THRESHOLDS", "true")
+	}
+
+	pa11yContainer = pa11yContainer.
+		WithServiceBinding(grafanaHost, grafanaService).
+		WithMountedFile("pa11yci-config.js", pa11yConfig).
+		WithExec([]string{"pa11y-ci", "--config", "pa11yci-config.js"}, dagger.ContainerWithExecOpts{
+			Expect: dagger.ReturnTypeAny, // allow this to fail here so we can handle non-zero exit codes at the caller
+		})
+
+	if pa11yResultsPath != "" {
+		_, err := pa11yContainer.File("/src/pa11y-ci-results.json").Export(ctx, pa11yResultsPath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get pa11y results: %w", err)
+		}
+	}
+
+	return pa11yContainer, nil
 }

pkg/build/a11y/service.go

@@ -9,77 +9,31 @@ import (
 	"dagger.io/dagger"
 )
 
-func NodeImage(version string) string {
-	return fmt.Sprintf("node:%s-slim", strings.TrimPrefix(strings.TrimSpace(version), "v"))
-}
-
 type GrafanaServiceOpts struct {
-	GrafanaDir   *dagger.Directory
+	HostSrc      *dagger.Directory
 	GrafanaTarGz *dagger.File
 	License      *dagger.File
-	YarnCache    *dagger.CacheVolume
-	NodeVersion  string
-}
-
-func Frontend(src *dagger.Directory) *dagger.Directory {
-	return src.
-		WithoutFile("go.mod").
-		WithoutFile("go.sum").
-		WithoutFile("go.work").
-		WithoutFile("go.work.sum").
-		WithoutDirectory(".github").
-		WithoutDirectory("docs").
-		WithoutDirectory("pkg").
-		WithoutDirectory("apps").
-		WithoutDirectory("videos")
-}
-
-func WithGrafanaFrontend(c *dagger.Container, src *dagger.Directory) *dagger.Container {
-	return c.WithDirectory("/src", Frontend(src), dagger.ContainerWithDirectoryOpts{
-		Exclude: []string{
-			"*drone*",
-			"*.go",
-			"*.md",
-		},
-	})
-}
-
-func WithYarnCache(c *dagger.Container, cache *dagger.CacheVolume) *dagger.Container {
-	return c.
-		WithWorkdir("/src").
-		WithMountedCache("/yarn/cache", cache)
-}
-
-func GrafanaFrontend(d *dagger.Client, yarnCache *dagger.CacheVolume, nodeVersion string, grafanaDir *dagger.Directory) *dagger.Container {
-	container := d.Container().From(NodeImage(nodeVersion))
-	container = WithGrafanaFrontend(container, grafanaDir)
-	return WithYarnCache(container, yarnCache).
-		WithEnvVariable("YARN_CACHE_FOLDER", "/yarn/cache").
-		WithExec([]string{"yarn", "install", "--immutable"})
 }
 
 func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOpts) (*dagger.Service, error) {
-	src := GrafanaFrontend(d, opts.YarnCache, opts.NodeVersion, opts.GrafanaDir)
-
 	container := d.Container().From("alpine:3").
 		WithExec([]string{"apk", "add", "--no-cache", "bash", "tar", "netcat-openbsd"}).
 		WithMountedFile("/src/grafana.tar.gz", opts.GrafanaTarGz).
 		WithExec([]string{"mkdir", "-p", "/src/grafana"}).
 		WithExec([]string{"tar", "--strip-components=1", "-xzf", "/src/grafana.tar.gz", "-C", "/src/grafana"}).
-		WithDirectory("/src/grafana/devenv", src.Directory("/src/devenv")).
-		WithDirectory("/src/grafana/e2e", src.Directory("/src/e2e")).
-		WithDirectory("/src/grafana/scripts", src.Directory("/src/scripts")).
-		WithDirectory("/src/grafana/tools", src.Directory("/src/tools")).
+		WithDirectory("/src/grafana/devenv", opts.HostSrc.Directory("./devenv")).
+		WithDirectory("/src/grafana/e2e/test-plugins", opts.HostSrc.Directory("./e2e/test-plugins")).
+		WithDirectory("/src/grafana/scripts", opts.HostSrc.Directory("./scripts")).
 		WithWorkdir("/src/grafana").
 		WithEnvVariable("GF_APP_MODE", "development").
-		WithEnvVariable("GF_SERVER_HTTP_PORT", "3001").
+		WithEnvVariable("GF_SERVER_HTTP_PORT", fmt.Sprint(grafanaPort)).
 		WithEnvVariable("GF_SERVER_ROUTER_LOGGING", "1").
-		WithExposedPort(3001)
+		WithExposedPort(grafanaPort)
 
 	var licenseArg string
 	if opts.License != nil {
-		container = container.WithMountedFile("/src/license.jwt", opts.License)
 		licenseArg = "/src/license.jwt"
+		container = container.WithMountedFile(licenseArg, opts.License)
 	}
 
 	// We add all GF_ environment variables to allow for overriding Grafana configuration.