E2E: Rework the runner (#105712)

2025-06-11 08:43:06 +02:00
parent fd7b6091a2
commit 185d1a1530
14 changed files with 622 additions and 122 deletions
@@ -836,7 +836,6 @@ embed.go @grafana/grafana-as-code
 /.github/workflows/frontend-lint.yml @grafana/grafana-frontend-platform
 /.github/workflows/analytics-events-report.yml @grafana/grafana-frontend-platform
 /.github/workflows/pr-e2e-tests.yml @grafana/grafana-developer-enablement-squad
-/.github/workflows/run-e2e-suite.yml @grafana/grafana-developer-enablement-squad
 /.github/workflows/skye-add-to-project.yml @grafana/grafana-frontend-platform
 /.github/zizmor.yml @grafana/grafana-developer-enablement-squad
 /.github/license_finder.yaml @bergquist
@@ -39,39 +39,98 @@ jobs:
          retention-days: 1
          name: ${{ steps.artifact.outputs.artifact }}
          path: grafana.tar.gz
-  e2e-matrix:
+
+  build-e2e-runner:
+    name: Build E2E test runner
+    runs-on: ubuntu-latest
    permissions:
      contents: read
+    outputs:
+      artifact: ${{ steps.artifact.outputs.artifact }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: ${{ !github.event.pull_request.head.repo.fork }}
+      - name: Build E2E test runner
+        id: artifact
+        run: |
+          # We want a static binary, so we need to set CGO_ENABLED=0
+          CGO_ENABLED=0 go build -o ./e2e-runner ./e2e/
+          echo "artifact=e2e-runner-${{github.run_number}}" >> "$GITHUB_OUTPUT"
+      - uses: actions/upload-artifact@v4
+        id: upload
+        with:
+          retention-days: 1
+          name: ${{ steps.artifact.outputs.artifact }}
+          path: e2e-runner
+
+  run-e2e-tests:
+    needs:
+      - build-grafana
+      - build-e2e-runner
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - suite: various-suite
+            path: e2e/various-suite
+          - suite: dashboards-suite
+            path: e2e/dashboards-suite
+          - suite: smoke-tests-suite
+            path: e2e/smoke-tests-suite
+          - suite: panels-suite
+            path: e2e/panels-suite
+          - suite: various-suite (old arch)
+            path: e2e/old-arch/various-suite
+            flags: --flags="--env dashboardScene=false"
+          - suite: dashboards-suite (old arch)
+            path: e2e/old-arch/dashboards-suite
+            flags: --flags="--env dashboardScene=false"
+          - suite: smoke-tests-suite (old arch)
+            path: e2e/old-arch/smoke-tests-suite
+            flags: --flags="--env dashboardScene=false"
+          - suite: panels-suite (old arch)
+            path: e2e/old-arch/panels-suite
+            flags: --flags="--env dashboardScene=false"
    name: ${{ matrix.suite }}
-    strategy:
-      fail-fast: false
-      matrix:
-        suite:
-          - various-suite
-          - dashboards-suite
-          - smoke-tests-suite
-          - panels-suite
-    needs:
-      - build-grafana
-    uses: ./.github/workflows/run-e2e-suite.yml
-    with:
-      package: ${{ needs.build-grafana.outputs.artifact }}
-      suite: ${{ matrix.suite }}
-  e2e-matrix-old-arch:
+    runs-on: ubuntu-latest-8-cores
    permissions:
      contents: read
-    name: ${{ matrix.suite }} (old arch)
-    strategy:
-      fail-fast: false
-      matrix:
-        suite:
-          - old-arch/various-suite
-          - old-arch/dashboards-suite
-          - old-arch/smoke-tests-suite
-          - old-arch/panels-suite
-    needs:
-      - build-grafana
-    uses: ./.github/workflows/run-e2e-suite.yml
-    with:
-      package: ${{ needs.build-grafana.outputs.artifact }}
-      suite: ${{ matrix.suite }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/download-artifact@v4
+        with:
+          name: ${{ needs.build-grafana.outputs.artifact }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: ${{ needs.build-e2e-runner.outputs.artifact }}
+      - name: chmod +x
+        run: chmod +x ./e2e-runner
+      - name: Run E2E tests
+        uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
+        with:
+          verb: run
+          args: go run ./pkg/build/e2e --package=grafana.tar.gz
+            --suite=${{ matrix.path }}
+            ${{ matrix.flags }}
+      - name: Set suite name
+        id: set-suite-name
+        if: success() || failure()
+        env:
+          SUITE: ${{ matrix.path }}
+        run: |
+          echo "suite=$(echo "$SUITE" | sed 's/\//-/g')" >> "$GITHUB_OUTPUT"
+      - uses: actions/upload-artifact@v4
+        if: success() || failure()
+        with:
+          name: ${{ steps.set-suite-name.outputs.suite }}-${{ github.run_number }}
+          path: videos
+          retention-days: 1
@@ -1,39 +0,0 @@
-name: e2e suite
-
-on:
-  workflow_call:
-    inputs:
-      package:
-        type: string
-        required: true
-      suite:
-        type: string
-        required: true
-
-jobs:
-  main:
-    runs-on: ubuntu-latest-8-cores
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - uses: actions/download-artifact@v4
-        with:
-          name: ${{ inputs.package }}
-      - uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
-        with:
-          verb: run
-          args: go run ./pkg/build/e2e --package=grafana.tar.gz --suite=${{ inputs.suite }}
-      - name: Set suite name
-        id: set-suite-name
-        if: success() || failure()
-        env:
-          SUITE: ${{ inputs.suite }}
-        run: |
-          echo "suite=$(echo "$SUITE" | sed 's/\//-/g')" >> "$GITHUB_OUTPUT"
-      - uses: actions/upload-artifact@v4
-        if: success() || failure()
-        with:
-          name: e2e-${{ steps.set-suite-name.outputs.suite }}-${{github.run_number}}
-          path: videos
-          retention-days: 1
@@ -21,6 +21,7 @@ rules:
      - pr-frontend-unit-tests.yml
      - pr-test-integration.yml
      - publish-kinds-release.yml
+      - pr-e2e-tests.yml
  dangerous-triggers:
    ignore:
      - auto-milestone.yml
@@ -0,0 +1,351 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/urfave/cli/v3"
+)
+
+func main() {
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer cancel()
+
+	if err := Run().Run(ctx, os.Args); err != nil {
+		cancel()
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func Run() *cli.Command {
+	var suitePath string
+
+	return &cli.Command{
+		Name:  "e2e",
+		Usage: "Run the test suite",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  "command",
+				Usage: "Cypress command to run. 'open' can be useful for development (enum: run, open)",
+				Value: "run",
+				Validator: func(s string) error {
+					if s != "run" && s != "open" {
+						return fmt.Errorf("invalid command: %s, must be 'run' or 'open'", s)
+					}
+					return nil
+				},
+			},
+			&cli.StringFlag{
+				Name:  "browser",
+				Usage: "Browser to run tests with (e.g.: chrome, electron)",
+				Value: "chrome",
+			},
+			&cli.StringFlag{
+				Name:  "grafana-base-url",
+				Usage: "Base URL for Grafana",
+				Value: "http://localhost:3001",
+			},
+			&cli.BoolFlag{
+				Name:  "cypress-video",
+				Usage: "Enable Cypress video recordings",
+				Value: false,
+			},
+			&cli.BoolFlag{
+				Name:  "smtp-plugin",
+				Usage: "Enable SMTP plugin",
+				Value: false,
+			},
+			&cli.BoolFlag{
+				Name:  "benchmark-plugin",
+				Usage: "Enable Benchmark plugin",
+				Value: false,
+			},
+			&cli.BoolFlag{
+				Name:  "slowmo",
+				Usage: "Slow down the test run",
+				Value: false,
+			},
+			&cli.StringSliceFlag{
+				Name:  "env",
+				Usage: "Additional Cypress environment variables to set (format: KEY=VALUE)",
+				Validator: func(s []string) error {
+					pattern := regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*=.*`)
+					for _, v := range s {
+						if !pattern.MatchString(v) {
+							return fmt.Errorf("invalid environment variable format: %s, must be KEY=VALUE", v)
+						}
+					}
+					return nil
+				},
+			},
+			&cli.StringSliceFlag{
+				Name:  "parameters",
+				Usage: "Additional parameters to pass to the Cypress command (e.g. --headed)",
+			},
+			&cli.DurationFlag{
+				Name:  "timeout",
+				Usage: "Timeout for the Cypress command (precision: milliseconds)",
+				Value: time.Second * 30,
+				Validator: func(d time.Duration) error {
+					if d < 0 {
+						return fmt.Errorf("timeout must be a positive duration")
+					}
+					if d.Round(time.Millisecond) != d {
+						return fmt.Errorf("timeout must be a whole number of milliseconds")
+					}
+					return nil
+				},
+			},
+			&cli.StringFlag{
+				Name:  "timezone",
+				Usage: "Timezone to set for the Cypress run (e.g. 'America/New_York')",
+				Value: "Pacific/Honolulu",
+			},
+
+			&cli.BoolFlag{
+				Name:     "start-grafana",
+				Usage:    "Start and wait for Grafana before running the tests",
+				Value:    true,
+				Category: "Grafana Server",
+			},
+			&cli.StringFlag{
+				Name:      "license-path",
+				Usage:     "Path to the Grafana Enterprise license file (optional; requires --start-grafana)",
+				Value:     "",
+				TakesFile: true,
+				Category:  "Grafana Server",
+			},
+			&cli.BoolFlag{
+				Name:     "image-renderer",
+				Usage:    "Install the image renderer plugin (requires --start-grafana)",
+				Category: "Grafana Server",
+			},
+
+			&cli.StringFlag{
+				Name:        "suite",
+				Usage:       "Path to the suite to run (e.g. './e2e/dashboards-suite')",
+				TakesFile:   true,
+				Required:    true,
+				Destination: &suitePath,
+			},
+		},
+		Action: runAction,
+	}
+}
+
+func runAction(ctx context.Context, c *cli.Command) error {
+	suitePath := c.String("suite")
+	suitePath, err := normalisePath(suitePath)
+	if err != nil {
+		return fmt.Errorf("failed to normalise suite path: %w", err)
+	}
+
+	repoRoot, err := gitRepoRoot(ctx, suitePath)
+	if err != nil {
+		return fmt.Errorf("failed to get git repo root: %w", err)
+	}
+
+	screenshotsFolder := path.Join(suitePath, "screenshots")
+	videosFolder := path.Join(suitePath, "videos")
+	fileServerFolder := path.Join(repoRoot, "e2e", "cypress")
+	fixturesFolder := path.Join(fileServerFolder, "fixtures")
+	downloadsFolder := path.Join(fileServerFolder, "downloads")
+	benchmarkPluginResultsFolder := path.Join(suitePath, "benchmark-results")
+	reporter := path.Join(repoRoot, "e2e", "log-reporter.js")
+
+	env := map[string]string{
+		"BENCHMARK_PLUGIN_ENABLED":        fmt.Sprintf("%t", c.Bool("benchmark-plugin")),
+		"SMTP_PLUGIN_ENABLED":             fmt.Sprintf("%t", c.Bool("smtp-plugin")),
+		"BENCHMARK_PLUGIN_RESULTS_FOLDER": benchmarkPluginResultsFolder,
+		"SLOWMO":                          "0",
+		"BASE_URL":                        c.String("grafana-base-url"),
+	}
+	for _, v := range c.StringSlice("env") {
+		parts := strings.SplitN(v, "=", 2)
+		if len(parts) != 2 {
+			return fmt.Errorf("invalid environment variable format: %s, must be KEY=VALUE", v)
+		}
+		env[parts[0]] = parts[1]
+	}
+
+	cypressConfig := map[string]string{
+		"screenshotsFolder": screenshotsFolder,
+		"fixturesFolder":    fixturesFolder,
+		"videosFolder":      videosFolder,
+		"downloadsFolder":   downloadsFolder,
+		"fileServerFolder":  fileServerFolder,
+		"reporter":          reporter,
+
+		"specPattern":           path.Join(suitePath, "*.spec.ts"),
+		"defaultCommandTimeout": fmt.Sprintf("%d", c.Duration("timeout").Milliseconds()),
+		"viewportWidth":         "1920",
+		"viewportHeight":        "1080",
+		"trashAssetsBeforeRuns": "false",
+		"baseUrl":               c.String("grafana-base-url"),
+		"video":                 fmt.Sprintf("%t", c.Bool("cypress-video")),
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	if c.Bool("start-grafana") {
+		startServerPath := path.Join(repoRoot, "scripts", "grafana-server", "start-server")
+		waitForGrafanaPath := path.Join(repoRoot, "scripts", "grafana-server", "wait-for-grafana")
+		go func() {
+			var args []string
+			if c.String("license-path") != "" {
+				args = append(args, c.String("license-path"))
+			}
+			//nolint:gosec
+			cmd := exec.CommandContext(ctx, startServerPath, args...)
+			cmd.Dir = repoRoot
+			cmd.Env = os.Environ()
+			cmd.Env = append(cmd.Env, fmt.Sprintf("TZ=%s", c.String("timezone")))
+			if c.Bool("image-renderer") {
+				cmd.Env = append(cmd.Env, "INSTALL_IMAGE_RENDERER=true")
+			}
+			cmd.Stdout = prefixGrafana(os.Stdout)
+			cmd.Stderr = prefixGrafana(os.Stderr)
+			cmd.Stdin = nil
+
+			if err := cmd.Run(); err != nil {
+				fmt.Println("Error running Grafana:", err)
+			}
+		}()
+
+		//nolint:gosec
+		cmd := exec.CommandContext(ctx, waitForGrafanaPath)
+		cmd.Dir = repoRoot
+		cmd.Env = os.Environ()
+		cmd.Env = append(cmd.Env, fmt.Sprintf("TZ=%s", c.String("timezone")))
+		cmd.Stdout = prefixGrafana(os.Stdout)
+		cmd.Stderr = prefixGrafana(os.Stderr)
+		cmd.Stdin = nil
+		if err := cmd.Run(); err != nil {
+			return fmt.Errorf("failed to wait for Grafana: %w", err)
+		}
+	}
+
+	args := []string{"run", "cypress", c.String("command"),
+		"--env", joinCypressCfg(env),
+		"--config", joinCypressCfg(cypressConfig),
+		"--browser", c.String("browser")}
+	args = append(args, c.StringSlice("parameters")...)
+	//nolint:gosec
+	cmd := exec.CommandContext(ctx, "yarn", args...)
+	cmd.Dir = repoRoot
+	cmd.Env = os.Environ()
+	cmd.Env = append(cmd.Env, fmt.Sprintf("TZ=%s", c.String("timezone")))
+	cmd.Stdout = prefixCypress(os.Stdout)
+	cmd.Stderr = prefixCypress(os.Stderr)
+	cmd.Stdin = os.Stdin
+	return cmd.Run()
+}
+
+func gitRepoRoot(ctx context.Context, dir string) (string, error) {
+	cmd := exec.CommandContext(ctx, "git", "rev-parse", "--show-toplevel")
+	cmd.Dir = dir
+	out, err := cmd.Output()
+	if err != nil {
+		return "", fmt.Errorf("failed to get git repo root: %w", err)
+	}
+	p := strings.TrimSpace(string(out))
+	p, err = normalisePath(p)
+	if err != nil {
+		return "", fmt.Errorf("failed to normalise git repo root path: %w", err)
+	}
+	return p, nil
+}
+
+func normalisePath(p string) (string, error) {
+	absPath, err := filepath.Abs(p)
+	if err != nil {
+		return "", fmt.Errorf("failed to get absolute path: %w", err)
+	}
+	return path.Clean(filepath.ToSlash(absPath)), nil
+}
+
+func joinCypressCfg(cfg map[string]string) string {
+	config := make([]string, 0, len(cfg))
+	for k, v := range cfg {
+		config = append(config, fmt.Sprintf("%s=%s", k, v))
+	}
+	return strings.Join(config, ",")
+}
+
+const (
+	resetColor  = "\033[0m"
+	yellowColor = "\033[0;33m"
+	cyanColor   = "\033[0;36m"
+)
+
+func prefixCypress(w io.Writer) io.Writer {
+	if _, ok := os.LookupEnv("CI"); ok {
+		return w
+	}
+
+	return newWrappingOutput(cyanColor+"Cypress: ", resetColor, w)
+}
+
+func prefixGrafana(w io.Writer) io.Writer {
+	if _, ok := os.LookupEnv("CI"); ok {
+		return w
+	}
+
+	return newWrappingOutput(yellowColor+"Grafana: ", resetColor, w)
+}
+
+var _ io.Writer = (*wrappingOutput)(nil)
+
+type wrappingOutput struct {
+	prefix        string
+	suffix        string
+	mu            *sync.Mutex
+	inner         io.Writer
+	writtenPrefix bool
+}
+
+func newWrappingOutput(prefix, suffix string, inner io.Writer) *wrappingOutput {
+	return &wrappingOutput{
+		prefix: prefix,
+		suffix: suffix,
+		mu:     &sync.Mutex{},
+		inner:  inner,
+	}
+}
+
+func (p *wrappingOutput) Write(b []byte) (int, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	for line := range bytes.Lines(b) {
+		if !p.writtenPrefix {
+			if _, err := p.inner.Write([]byte(p.prefix)); err != nil {
+				return 0, err
+			}
+			p.writtenPrefix = true
+		}
+		if _, err := p.inner.Write(line); err != nil {
+			return 0, err
+		}
+		if bytes.HasSuffix(line, []byte("\n")) {
+			p.writtenPrefix = false
+			if _, err := p.inner.Write([]byte(p.suffix)); err != nil {
+				return 0, err
+			}
+		}
+	}
+	return len(b), nil
+}
@@ -26,7 +26,7 @@ declare -A env=(
 )

 testFilesForSingleSuite="*.spec.ts"
-rootForEnterpriseSuite="./e2e/extensions-suite"
+rootForEnterpriseSuite="./e2e/extensions"
 rootForOldArch="./e2e/old-arch"
 rootForKubernetesDashboards="./e2e/dashboards-suite"
 rootForSearchDashboards="./e2e/dashboards-search-suite"
@@ -45,6 +45,7 @@ declare -A cypressConfig=(
  [trashAssetsBeforeRuns]=false
  [reporter]=./e2e/log-reporter.js
  [baseUrl]=${BASE_URL:-"http://$HOST:$PORT"}
+  [video]=${CYPRESS_VIDEO:-false}
 )

 case "$1" in
@@ -70,8 +71,6 @@ case "$1" in
    "enterprise")
        echo "Enterprise"
        env[SMTP_PLUGIN_ENABLED]=true
-        CLEANUP="rm -rf ./e2e/extensions-suite"
-        SETUP="cp -Lr ./e2e/extensions ./e2e/extensions-suite"
        enterpriseSuite=$(basename "${args[1]}")
        case "$2" in
            "debug")
@@ -87,7 +86,6 @@ case "$1" in
                ;;
        esac
        cypressConfig[specPattern]=$rootForEnterpriseSuite/$enterpriseSuite/*-suite/*.spec.ts
-        $CLEANUP && $SETUP
        ;;
    "")
        ;;
@@ -156,6 +156,7 @@ require (
 	github.com/ua-parser/uap-go v0.0.0-20250213224047-9c035f085b90 // @grafana/grafana-backend-group
 	github.com/urfave/cli v1.22.16 // indirect; @grafana/grafana-backend-group
 	github.com/urfave/cli/v2 v2.27.6 // @grafana/grafana-backend-group
+	github.com/urfave/cli/v3 v3.3.3 // @grafana/grafana-backend-group
 	github.com/wk8/go-ordered-map v1.0.0 // @grafana/grafana-backend-group
 	github.com/xlab/treeprint v1.2.0 // @grafana/observability-traces-and-profiling
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // @grafana/grafana-operator-experience-squad
@@ -2442,6 +2442,8 @@ github.com/urfave/cli v1.22.16 h1:MH0k6uJxdwdeWQTwhSO42Pwr4YLrNLwBtg1MRgTqPdQ=
 github.com/urfave/cli v1.22.16/go.mod h1:EeJR6BKodywf4zciqrdw6hpCPk68JO9z5LazXZMn5Po=
 github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g=
 github.com/urfave/cli/v2 v2.27.6/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
+github.com/urfave/cli/v3 v3.3.3 h1:byCBaVdIXuLPIDm5CYZRVG6NvT7tv1ECqdU4YzlEa3I=
+github.com/urfave/cli/v3 v3.3.3/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZtpYpo=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs=
@@ -217,12 +217,12 @@ func enterpriseDirectory(ctx context.Context, opts *pipeline.ArgumentOpts) (any,
 		return nil, fmt.Errorf("error initializing grafana directory: %w", err)
 	}

-	src, err := cloneOrMount(ctx, opts.Client, o.EnterpriseDir, o.EnterpriseRepo, o.EnterpriseRef, o.GitHubToken)
+	clone, err := cloneOrMount(ctx, opts.Client, o.EnterpriseDir, o.EnterpriseRepo, o.EnterpriseRef, o.GitHubToken)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("error cloning or mounting Grafana Enterprise directory: %w", err)
 	}

-	return InitializeEnterprise(opts.Client, grafanaDir.(*dagger.Directory), src), nil
+	return InitializeEnterprise(opts.Client, grafanaDir.(*dagger.Directory), clone), nil
 }

 var GrafanaDirectoryFlags = []cli.Flag{
@@ -2,71 +2,173 @@ package main

 import (
 	"context"
-	"flag"
 	"fmt"
 	"log"
 	"os"
+	"os/signal"
+	"path"

 	"dagger.io/dagger"
+	"github.com/urfave/cli/v3"
 )

 func main() {
-	var (
-		ctx         = context.Background()
-		grafanaPath = flag.String("grafana-dir", ".", "Path to cloned grafana repo")
-		targzPath   = flag.String("package", "grafana.tar.gz", "Path to grafana tar.gz package")
-		suite       = flag.String("suite", "", "e2e suite name (used in arg to run-suite script)")
-	)
-	flag.Parse()
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer cancel()
+
+	if err := NewApp().Run(ctx, os.Args); err != nil {
+		cancel()
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func NewApp() *cli.Command {
+	return &cli.Command{
+		Name:  "e2e",
+		Usage: "Run the E2E tests for Grafana",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:      "suite",
+				Usage:     "E2E test suite path (e.g. e2e/various-suite)",
+				Validator: mustBeDir("suite"),
+				TakesFile: true,
+				Required:  true,
+			},
+
+			&cli.StringFlag{
+				Name:      "grafana-dir",
+				Usage:     "Path to the grafana/grafana clone directory",
+				Value:     ".",
+				Validator: mustBeDir("grafana-dir"),
+				TakesFile: true,
+			},
+			&cli.StringFlag{
+				Name:      "package",
+				Usage:     "Path to the grafana tar.gz package",
+				Value:     "grafana.tar.gz",
+				Validator: mustBeFile("package", false),
+				TakesFile: true,
+			},
+			&cli.StringFlag{
+				Name:      "license",
+				Usage:     "Path to the Grafana Enterprise license file (optional)",
+				Validator: mustBeFile("license", true),
+				TakesFile: true,
+			},
+			&cli.StringFlag{
+				Name:  "flags",
+				Usage: "Flags to pass through to the e2e runner",
+			},
+			&cli.BoolFlag{
+				Name:  "image-renderer",
+				Usage: "Install the image renderer plugin",
+				Value: false,
+			},
+		},
+		Action: run,
+	}
+}
+
+func run(ctx context.Context, cmd *cli.Command) error {
+	grafanaDir := cmd.String("grafana-dir")
+	suite := cmd.String("suite")
+	targzPath := cmd.String("package")
+	licensePath := cmd.String("license")
+	imageRenderer := cmd.Bool("image-renderer")
+	runnerFlags := cmd.String("flags")

 	d, err := dagger.Connect(ctx)
 	if err != nil {
-		panic(err)
+		return fmt.Errorf("failed to connect to Dagger: %w", err)
 	}

 	yarnCache := d.CacheVolume("yarn")

-	log.Println("grafana dir:", *grafanaPath)
-	log.Println("targz:", *targzPath)
+	log.Println("grafana dir:", grafanaDir)
+	log.Println("targz:", targzPath)
+	log.Println("license path:", licensePath)

 	grafana := d.Host().Directory(".", dagger.HostDirectoryOpts{
-		Exclude: []string{".git", "node_modules", "*.tar.gz"},
+		Exclude: []string{"node_modules", "*.tar.gz"},
 	})
+	targz := d.Host().File(targzPath)

-	targz := d.Host().File("grafana.tar.gz")
-
-	svc, err := GrafanaService(ctx, d, GrafanaServiceOpts{
-		GrafanaDir:   grafana,
-		GrafanaTarGz: targz,
-		YarnCache:    yarnCache,
-	})
-	if err != nil {
-		panic(err)
+	var license *dagger.File
+	if licensePath != "" {
+		license = d.Host().File(licensePath)
 	}

-	videosDir := fmt.Sprintf("/src/e2e/%s/videos", *suite)
+	svc, err := GrafanaService(ctx, d, GrafanaServiceOpts{
+		GrafanaDir:           grafana,
+		GrafanaTarGz:         targz,
+		YarnCache:            yarnCache,
+		License:              license,
+		InstallImageRenderer: imageRenderer,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to create Grafana service: %w", err)
+	}
+
+	videosDir := path.Join("/src", suite, "videos")
 	// *spec.ts.mp4
-	c := RunSuite(d, svc, grafana, yarnCache, *suite)
+	c := RunSuite(d, svc, grafana, yarnCache, suite, runnerFlags)
 	c, err = c.Sync(ctx)
 	if err != nil {
-		log.Fatalf("error running dagger: %s", err)
+		return fmt.Errorf("failed to run e2e test suite: %w", err)
 	}

 	code, err := c.ExitCode(ctx)
 	if err != nil {
-		log.Fatalf("error getting exit code: %s", err)
+		return fmt.Errorf("failed to get exit code of e2e test suite: %w", err)
 	}

 	log.Println("exit code:", code)

 	// No sync error; export the videos dir
 	if _, err := c.Directory(videosDir).Export(ctx, "videos"); err != nil {
-		log.Fatalf("error getting videos: %s", err)
+		return fmt.Errorf("failed to export videos directory: %w", err)
 	}

 	if code != 0 {
-		log.Printf("tests failed: exit code %d", code)
+		return fmt.Errorf("e2e tests failed with exit code %d", code)
 	}

-	os.Exit(code)
+	log.Println("e2e tests completed successfully")
+	return nil
+}
+
+func mustBeFile(arg string, emptyOk bool) func(string) error {
+	return func(s string) error {
+		if s == "" {
+			if emptyOk {
+				return nil
+			}
+			return cli.Exit(arg+" cannot be empty", 1)
+		}
+		stat, err := os.Stat(s)
+		if err != nil {
+			return cli.Exit(arg+" does not exist or cannot be read: "+s, 1)
+		}
+		if stat.IsDir() {
+			return cli.Exit(arg+" must be a file, not a directory: "+s, 1)
+		}
+		return nil
+	}
+}
+
+func mustBeDir(arg string) func(string) error {
+	return func(s string) error {
+		if s == "" {
+			return cli.Exit(arg+" cannot be empty", 1)
+		}
+		stat, err := os.Stat(s)
+		if err != nil {
+			return cli.Exit(arg+" does not exist or cannot be read: "+s, 1)
+		}
+		if !stat.IsDir() {
+			return cli.Exit(arg+" must be a directory: "+s, 1)
+		}
+		return nil
+	}
 }
@@ -6,17 +6,14 @@ import (
 	"dagger.io/dagger"
 )

-func RunSuite(d *dagger.Client, svc *dagger.Service, src *dagger.Directory, cache *dagger.CacheVolume, suite string) *dagger.Container {
+func RunSuite(d *dagger.Client, svc *dagger.Service, src *dagger.Directory, cache *dagger.CacheVolume, suite, runnerFlags string) *dagger.Container {
+	command := fmt.Sprintf(
+		"./e2e-runner --start-grafana=false --cypress-video"+
+			" --grafana-base-url http://grafana:3001 --suite %s %s", suite, runnerFlags)
+
 	return WithYarnCache(WithGrafanaFrontend(d.Container().From("cypress/included:13.1.0"), src), cache).
 		WithWorkdir("/src").
-		WithEnvVariable("HOST", "grafana").
-		WithEnvVariable("PORT", "3001").
 		WithServiceBinding("grafana", svc).
 		WithExec([]string{"yarn", "install", "--immutable"}).
-		WithExec([]string{
-			"/bin/bash", "-c",
-			fmt.Sprintf("./e2e/run-suite %s true", suite),
-		}, dagger.ContainerWithExecOpts{
-			Expect: dagger.ReturnTypeAny,
-		})
+		WithExec([]string{"/bin/bash", "-c", command}, dagger.ContainerWithExecOpts{Expect: dagger.ReturnTypeAny})
 }
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"os"
 	"strings"

 	"dagger.io/dagger"
@@ -22,9 +23,11 @@ func NodeImage(version string) string {
 }

 type GrafanaServiceOpts struct {
-	GrafanaDir   *dagger.Directory
-	GrafanaTarGz *dagger.File
-	YarnCache    *dagger.CacheVolume
+	GrafanaDir           *dagger.Directory
+	GrafanaTarGz         *dagger.File
+	YarnCache            *dagger.CacheVolume
+	License              *dagger.File
+	InstallImageRenderer bool
 }

 func Frontend(src *dagger.Directory) *dagger.Directory {
@@ -71,8 +74,9 @@ func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOp
 		WithExec([]string{"yarn", "install", "--immutable"}).
 		WithExec([]string{"yarn", "e2e:plugin:build"})

-	svc := d.Container().From("alpine").
-		WithExec([]string{"apk", "add", "bash"}).
+	// _/ubuntu:latest sticks to latest LTS.
+	// We need ubuntu to support the image renderer plugin, which assumes glibc.
+	container := d.Container().From("ubuntu:latest").
 		WithMountedFile("/src/grafana.tar.gz", opts.GrafanaTarGz).
 		WithExec([]string{"mkdir", "-p", "/src/grafana"}).
 		WithExec([]string{"tar", "--strip-components=1", "-xzf", "/src/grafana.tar.gz", "-C", "/src/grafana"}).
@@ -84,8 +88,30 @@ func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOp
 		WithEnvVariable("GF_APP_MODE", "development").
 		WithEnvVariable("GF_SERVER_HTTP_PORT", "3001").
 		WithEnvVariable("GF_SERVER_ROUTER_LOGGING", "1").
-		WithExposedPort(3001).
-		AsService(dagger.ContainerAsServiceOpts{Args: []string{"bash", "-x", "scripts/grafana-server/start-server"}})
+		WithExposedPort(3001)
+
+	var licenseArg string
+	if opts.License != nil {
+		container = container.WithMountedFile("/src/license.jwt", opts.License)
+		licenseArg = "/src/license.jwt"
+	}
+
+	if opts.InstallImageRenderer {
+		container = container.WithEnvVariable("INSTALL_IMAGE_RENDERER", "true").
+			WithExec([]string{"apt-get", "update"}).
+			WithExec([]string{"apt-get", "install", "-y", "ca-certificates"})
+	}
+
+	// We add all GF_ environment variables to allow for overriding Grafana configuration.
+	// It is unlikely the runner has any such otherwise.
+	for _, env := range os.Environ() {
+		if strings.HasPrefix(env, "GF_") {
+			parts := strings.SplitN(env, "=", 2)
+			container = container.WithEnvVariable(parts[0], parts[1])
+		}
+	}
+
+	svc := container.AsService(dagger.ContainerAsServiceOpts{Args: []string{"bash", "-x", "scripts/grafana-server/start-server", licenseArg}})

 	return svc, nil
 }
@@ -82,6 +82,7 @@ require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/google/go-github/v70 v70.0.0
 	github.com/quasilyte/go-ruleguard/dsl v0.3.22
+	github.com/urfave/cli/v3 v3.3.3
 )

 require (
@@ -251,6 +251,8 @@ github.com/urfave/cli v1.22.16 h1:MH0k6uJxdwdeWQTwhSO42Pwr4YLrNLwBtg1MRgTqPdQ=
 github.com/urfave/cli v1.22.16/go.mod h1:EeJR6BKodywf4zciqrdw6hpCPk68JO9z5LazXZMn5Po=
 github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g=
 github.com/urfave/cli/v2 v2.27.6/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
+github.com/urfave/cli/v3 v3.3.3 h1:byCBaVdIXuLPIDm5CYZRVG6NvT7tv1ECqdU4YzlEa3I=
+github.com/urfave/cli/v3 v3.3.3/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZtpYpo=
 github.com/vektah/gqlparser/v2 v2.5.27 h1:RHPD3JOplpk5mP5JGX8RKZkt2/Vwj/PZv0HxTdwFp0s=
 github.com/vektah/gqlparser/v2 v2.5.27/go.mod h1:D1/VCZtV3LPnQrcPBeR/q5jkSQIPti0uYCP/RI0gIeo=
 github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=