wip

.github/workflows/playwright-e2e.yml

@@ -0,0 +1,146 @@
+name: Playwright E2E Tests
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - release-*.*.*
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+permissions: {}
+
+jobs:
+  build-grafana:
+    name: Build & Package Grafana
+    runs-on: ubuntu-latest-16-cores
+    permissions:
+      contents: read
+    outputs:
+      artifact: ${{ steps.artifact.outputs.artifact }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: ./grafana
+          persist-credentials: false
+      - uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
+        with:
+          verb: run
+          args: go -C grafana run ./pkg/build/cmd artifacts -a targz:grafana:linux/amd64 --grafana-dir="${PWD}/grafana" > out.txt
+      - run: mv "$(cat out.txt)" grafana.tar.gz
+      - run: echo "artifact=grafana-playwright-${{github.run_number}}" >> "$GITHUB_OUTPUT"
+        id: artifact
+      - uses: actions/upload-artifact@v4
+        id: upload
+        with:
+          retention-days: 1
+          name: ${{ steps.artifact.outputs.artifact }}
+          path: grafana.tar.gz
+
+  run-playwright-tests:
+    needs:
+      - build-grafana
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - suite: plugin-e2e-api-tests
+            project: admin
+          - suite: plugin-e2e-api-tests
+            project: viewer
+          - suite: plugin-e2e
+            project: elasticsearch
+          - suite: plugin-e2e
+            project: mysql
+          - suite: plugin-e2e
+            project: mssql
+          - suite: plugin-e2e
+            project: cloudwatch
+          - suite: plugin-e2e
+            project: azuremonitor
+          - suite: plugin-e2e
+            project: cloudmonitoring
+          - suite: plugin-e2e
+            project: graphite
+          - suite: plugin-e2e
+            project: influxdb
+          - suite: plugin-e2e
+            project: jaeger
+          - suite: plugin-e2e
+            project: zipkin
+          - suite: core-tests
+            project: panels
+          - suite: core-tests
+            project: smoke
+          - suite: core-tests
+            project: dashboards
+          - suite: core-tests
+            project: loki
+          - suite: test-plugins
+            project: extensions-test-app
+          - suite: test-plugins
+            project: grafana-e2etest-datasource
+    name: ${{ matrix.suite }}-${{ matrix.project }}
+    runs-on: ubuntu-latest-8-cores
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/download-artifact@v4
+        with:
+          name: ${{ needs.build-grafana.outputs.artifact }}
+      - name: Run Playwright E2E tests
+        uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
+        with:
+          verb: run
+          args: go run ./pkg/build/daggerbuild/playwright run --package=grafana.tar.gz --project=${{ matrix.project }}
+      - name: Set suite name
+        id: set-suite-name
+        if: success() || failure()
+        env:
+          SUITE: ${{ matrix.suite }}-${{ matrix.project }}
+        run: |
+          set -euo pipefail
+          echo "suite=$(echo "$SUITE" | sed 's/\//-/g')" >> "$GITHUB_OUTPUT"
+      - uses: actions/upload-artifact@v4
+        if: success() || failure()
+        with:
+          name: playwright-report-${{ steps.set-suite-name.outputs.suite }}-${{ github.run_number }}
+          path: playwright-report
+          retention-days: 3
+      - uses: actions/upload-artifact@v4
+        if: success() || failure()
+        with:
+          name: test-results-${{ steps.set-suite-name.outputs.suite }}-${{ github.run_number }}
+          path: test-results
+          retention-days: 3
+
+  # This is the job that is actually required by rulesets.
+  # We want to only require one job instead of all the individual tests.
+  required-playwright-tests:
+    needs:
+      - run-playwright-tests
+    # always() is the best function here.
+    # success() || failure() will skip this function if any need is also skipped.
+    # That means conditional test suites will fail the entire requirement check.
+    if: always()
+
+    name: All Playwright E2E tests complete
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check test suites
+        env:
+          NEEDS: ${{ toJson(needs) }}
+        run: |
+          FAILURES="$(echo "$NEEDS" | jq 'with_entries(select(.value.result == "failure")) | map_values(.result)')"
+          echo "$FAILURES"
+          if [ "$(echo "$FAILURES" | jq '. | length')" != "0" ]; then
+            exit 1
+          fi
+          echo "All OK!"

pkg/build/e2e-playwright/README.md

@@ -0,0 +1,22 @@
+# Pa11y accessability tests
+
+We use pa11y to run some automated simple accessability tests. They're ran with dagger to help orchestrate starting server + tests in a reproducable manner.
+
+To run the tests locally:
+
+1. Install dagger locally https://docs.dagger.io/install/
+2. Grab the grafana.tar.gz artifact by either
+   1. Downloading it from the Github Action artifact from your PR
+   1. Build it locally with:
+      ```sh
+      dagger run go run ./pkg/build/cmd artifacts -a targz:grafana:linux/amd64 --grafana-dir="$PWD" > dist/files.txt
+      cat dist/files.txt # Will output the path to the grafana.tar.gz 
+      ```
+3. Run the dagger pipeline with:
+   ```sh
+   dagger -v run go run ./pkg/build/a11y --package=(full path to .tar.gz) --results=./pa11y-ci-results.json
+   ```
+   The JSON results file will be saved to the file from the `--results` arg 
+4. If they fail and you want to see the full output
+   1. Run the dagger command with `dagger -vE [...]`
+   2. At the end, arrow up to the exec pa11y-ci segment and hit Enter

pkg/build/e2e-playwright/main.go

@@ -0,0 +1,177 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+
+	"dagger.io/dagger"
+	"github.com/urfave/cli/v3"
+)
+
+var (
+	grafanaHost = "grafana"
+	grafanaPort = 3001
+)
+
+func main() {
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer cancel()
+
+	if err := NewApp().Run(ctx, os.Args); err != nil {
+		cancel()
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func NewApp() *cli.Command {
+	return &cli.Command{
+		Name:  "a11y",
+		Usage: "Run Grafana playwright e2e tests",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:      "grafana-dir",
+				Usage:     "Path to the grafana/grafana clone directory",
+				Value:     ".",
+				Validator: mustBeDir("grafana-dir"),
+				TakesFile: true,
+			},
+			&cli.StringFlag{
+				Name:      "package",
+				Usage:     "Path to the grafana tar.gz package",
+				Value:     "grafana.tar.gz",
+				Validator: mustBeFile("package", false),
+				TakesFile: true,
+			},
+			&cli.StringFlag{
+				Name:      "license",
+				Usage:     "Path to the Grafana Enterprise license file (optional)",
+				Validator: mustBeFile("license", true),
+				TakesFile: true,
+			},
+			// &cli.StringFlag{
+			// 	Name:      "config",
+			// 	Usage:     "Path to the pa11y config file to use",
+			// 	Value:     "e2e/pa11yci.conf.js",
+			// 	Validator: mustBeFile("config", true),
+			// 	TakesFile: true,
+			// },
+			// &cli.StringFlag{
+			// 	Name:      "results",
+			// 	Usage:     "Path to the pa11y results file to export",
+			// 	TakesFile: true,
+			// },
+			// &cli.BoolFlag{
+			// 	Name:  "no-threshold-fail",
+			// 	Usage: "Don't fail the task if any of the tests fail. Use this in combination with --results to list all violations even if they're within thresholds",
+			// 	Value: false,
+			// },
+		},
+		Action: run,
+	}
+}
+
+func run(ctx context.Context, cmd *cli.Command) error {
+	grafanaDir := cmd.String("grafana-dir")
+	targzPath := cmd.String("package")
+	licensePath := cmd.String("license")
+	// pa11yConfigPath := cmd.String("config")
+	// pa11yResultsPath := cmd.String("results")
+	// noThresholdFail := cmd.Bool("no-threshold-fail")
+
+	d, err := dagger.Connect(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to connect to Dagger: %w", err)
+	}
+
+	// Explicitly only the files used by the grafana-server service
+	hostSrc := d.Host().Directory(grafanaDir, dagger.HostDirectoryOpts{
+		Include: []string{
+			"./devenv",
+			"./e2e/test-plugins", // Directory is included so provisioning works, but they're not actually build
+			"./scripts/grafana-server/custom.ini",
+			"./scripts/grafana-server/start-server",
+			"./scripts/grafana-server/kill-server",
+			"./scripts/grafana-server/variables",
+		},
+	})
+
+	targz := d.Host().File(targzPath)
+	// pa11yConfig := d.Host().File(pa11yConfigPath)
+
+	var license *dagger.File
+	if licensePath != "" {
+		license = d.Host().File(licensePath)
+	}
+
+	svc, err := GrafanaService(ctx, d, GrafanaServiceOpts{
+		HostSrc:      hostSrc,
+		GrafanaTarGz: targz,
+		License:      license,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to create Grafana service: %w", err)
+	}
+
+	c, runErr := RunTest(ctx, d, svc)
+	if runErr != nil {
+		return fmt.Errorf("failed to run a11y test suite: %w", runErr)
+	}
+
+	c, syncErr := c.Sync(ctx)
+	if syncErr != nil {
+		return fmt.Errorf("failed to sync a11y test suite: %w", syncErr)
+	}
+
+	code, codeErr := c.ExitCode(ctx)
+	if codeErr != nil {
+		return fmt.Errorf("failed to get exit code of a11y test suite: %w", codeErr)
+	}
+
+	if code == 0 {
+		log.Printf("a11y tests passed with exit code %d", code)
+	} else {
+		return fmt.Errorf("a11y tests failed with exit code %d", code)
+	}
+
+	log.Println("a11y tests completed successfully")
+	return nil
+}
+
+func mustBeFile(arg string, emptyOk bool) func(string) error {
+	return func(s string) error {
+		if s == "" {
+			if emptyOk {
+				return nil
+			}
+			return cli.Exit(arg+" cannot be empty", 1)
+		}
+		stat, err := os.Stat(s)
+		if err != nil {
+			return cli.Exit(arg+" does not exist or cannot be read: "+s, 1)
+		}
+		if stat.IsDir() {
+			return cli.Exit(arg+" must be a file, not a directory: "+s, 1)
+		}
+		return nil
+	}
+}
+
+func mustBeDir(arg string) func(string) error {
+	return func(s string) error {
+		if s == "" {
+			return cli.Exit(arg+" cannot be empty", 1)
+		}
+		stat, err := os.Stat(s)
+		if err != nil {
+			return cli.Exit(arg+" does not exist or cannot be read: "+s, 1)
+		}
+		if !stat.IsDir() {
+			return cli.Exit(arg+" must be a directory: "+s, 1)
+		}
+		return nil
+	}
+}

pkg/build/e2e-playwright/run-e2e.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"dagger.io/dagger"
+)
+
+// NodeVersionContainer returns a container whose `stdout` will return the node version from the '.nvmrc' file in the directory 'src'.
+func NodeVersion(d *dagger.Client, src *dagger.File) *dagger.Container {
+	return d.Container().From("alpine:3").
+		WithMountedFile("/src/.nvmrc", src).
+		WithWorkdir("/src").
+		WithExec([]string{"cat", ".nvmrc"})
+}
+
+func NodeImage(version string) string {
+	return fmt.Sprintf("node:%s-slim", strings.TrimPrefix(strings.TrimSpace(version), "v"))
+}
+
+func RunTest(
+	ctx context.Context,
+	d *dagger.Client,
+	grafanaService *dagger.Service,
+) (*dagger.Container, error) {
+
+	// Explicitly only the files u'sed by e2e tests
+	hostSrc := d.Host().Directory(".", dagger.HostDirectoryOpts{
+		Include: []string{
+			// Include all files for a valid yarn workspace install
+			"package.json",
+			"yarn.lock",
+			".yarnrc.yml",
+			".yarn",
+			"packages/*/package.json",
+			"public/app/plugins/*/*/package.json",
+			"e2e/test-plugins/*/package.json",
+			".nvmrc",
+			"public/app/types/*.d.ts",
+
+			// packages we use in playwright tests
+			"packages",
+
+			// e2e files
+			"e2e-playwright",
+		},
+		Exclude: []string{
+			"packages/*/dist",
+		},
+	})
+
+	nodeVersion, err := NodeVersion(d, hostSrc.File(".nvmrc")).Stdout(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	yarnCache := d.CacheVolume("yarn-cache")
+	yarnCacheDir := "/yarn-cache"
+
+	pa11yContainer := d.Container().From(NodeImage(nodeVersion)).
+		WithExec([]string{"npx", "-y", "playwright@1.52.0", "install", "--with-deps"}). // TODO: sync version from package.json
+		WithWorkdir("/src").
+		WithDirectory("/src", hostSrc).
+		WithMountedCache(yarnCacheDir, yarnCache).
+		WithExec([]string{"corepack", "enable"}).
+		WithExec([]string{"corepack", "install"}).
+		WithEnvVariable("YARN_CACHE_FOLDER", yarnCacheDir).
+		WithExec([]string{"yarn", "config", "get", "cacheFolder"}).
+		WithExec([]string{"yarn", "install", "--immutable"}).
+		// WithExec([]string{"yarn", "e2e:plugin:build"}).
+		WithEnvVariable("HOST", grafanaHost).
+		WithEnvVariable("PORT", fmt.Sprint(grafanaPort)).
+		WithExec([]string{"yarn", "packages:build"}).
+		WithExec([]string{"yarn", "e2e:playwright"})
+
+	return pa11yContainer, nil
+}

pkg/build/e2e-playwright/service.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	"dagger.io/dagger"
+)
+
+type GrafanaServiceOpts struct {
+	HostSrc      *dagger.Directory
+	GrafanaTarGz *dagger.File
+	License      *dagger.File
+}
+
+func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOpts) (*dagger.Service, error) {
+	container := d.Container().From("alpine:3").
+		WithExec([]string{"apk", "add", "--no-cache", "bash", "tar", "netcat-openbsd"}).
+		WithMountedFile("/src/grafana.tar.gz", opts.GrafanaTarGz).
+		WithExec([]string{"mkdir", "-p", "/src/grafana"}).
+		WithExec([]string{"tar", "--strip-components=1", "-xzf", "/src/grafana.tar.gz", "-C", "/src/grafana"}).
+		WithDirectory("/src/grafana/devenv", opts.HostSrc.Directory("./devenv")).
+		WithDirectory("/src/grafana/e2e/test-plugins", opts.HostSrc.Directory("./e2e/test-plugins")).
+		WithDirectory("/src/grafana/scripts", opts.HostSrc.Directory("./scripts")).
+		WithWorkdir("/src/grafana").
+		WithEnvVariable("GF_APP_MODE", "development").
+		WithEnvVariable("GF_SERVER_HTTP_PORT", fmt.Sprint(grafanaPort)).
+		WithEnvVariable("GF_SERVER_ROUTER_LOGGING", "1").
+		WithExposedPort(grafanaPort)
+
+	var licenseArg string
+	if opts.License != nil {
+		licenseArg = "/src/license.jwt"
+		container = container.WithMountedFile(licenseArg, opts.License)
+	}
+
+	// We add all GF_ environment variables to allow for overriding Grafana configuration.
+	// It is unlikely the runner has any such otherwise.
+	for _, env := range os.Environ() {
+		if strings.HasPrefix(env, "GF_") {
+			parts := strings.SplitN(env, "=", 2)
+			container = container.WithEnvVariable(parts[0], parts[1])
+		}
+	}
+
+	svc := container.AsService(dagger.ContainerAsServiceOpts{Args: []string{"bash", "-x", "scripts/grafana-server/start-server", licenseArg}})
+
+	return svc, nil
+}