Infrastructure: Revert to pre-Playwright E2E system for release branch compatibility

Strategic combination of modern daggerbuild with stable E2E infrastructure:

**Modern Daggerbuild (from main):**
- Latest CLI framework (github.com/urfave/cli/v3)
- External infrastructure fixes (musl.cc → dl.grafana.com/ci)
- Complete pkg/build/ system with all enhancements
- Updated dependencies and architecture improvements

**Stable E2E Infrastructure (pre-Playwright):**
- pkg/build/e2e/service.go: Reverted to version before commit b6580ccb10
- pkg/build/a11y/: Reverted to stable version compatible with existing infrastructure
- Removed experimental pkg/build/e2e-playwright/ system
- Removed experimental e2e-playwright/ test infrastructure

**Why This Approach:**
- GitHub Actions workflows call 'go run ./pkg/build/e2e' (modern system)
- Modern system now expects only standard e2e/ directory (compatible)
- Release branches have standard e2e/ infrastructure (dashboards-suite, various-suite, etc.)
- Eliminates dependency on experimental Playwright infrastructure

**Key Dependencies Met:**
- e2e/pa11yci.conf.js: Added for a11y test compatibility
- /src/e2e directory: Standard directory structure (no e2e-playwright needed)
- Existing test plugins: Uses stable e2e/test-plugins/

**Result:**
Best of both worlds - modern, reliable daggerbuild infrastructure with
proven E2E testing that works across all release branches without
requiring experimental dependencies.
This commit is contained in:
jev forsberg
2025-07-16 16:47:27 -06:00
parent 2d98040d07
commit 38eb3039af
9 changed files with 142 additions and 511 deletions
+140
View File
@@ -0,0 +1,140 @@
const config = {
defaults: {
concurrency: 1,
runners: ['axe'],
useIncognitoBrowserContext: false,
standard: 'WCAG2AA',
chromeLaunchConfig: {
executablePath: '/usr/bin/google-chrome',
args: ['--no-sandbox'],
},
// see https://github.com/grafana/grafana/pull/41693#issuecomment-979921463 for context
// on why we're ignoring singleValue/react-select-*-placeholder elements
hideElements: '#updateVersion, [class*="-singleValue"], [id^="react-select-"][id$="-placeholder"]',
reporters: ['cli', ['json', { fileName: './pa11y-ci-results.json' }]],
},
urls: [
{
url: '${HOST}/login',
threshold: 0,
},
{
url: '${HOST}/login',
actions: [
"wait for element input[name='user'] to be added",
"set field input[name='user'] to admin",
"set field input[name='password'] to admin",
"click element button[data-testid='data-testid Login button']",
"wait for element button[data-testid='data-testid Skip change password button'] to be visible",
],
threshold: 2,
},
{
url: '${HOST}/?orgId=1',
threshold: 0,
},
{
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge',
threshold: 0,
},
// Dashboard settings
{
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=settings',
threshold: 0,
},
{
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=annotations',
threshold: 0,
},
{
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=variables',
threshold: 0,
},
{
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=links',
threshold: 0,
},
{
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=versions',
threshold: 0,
},
{
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=permissions',
// TODO: improve the accessibility of the permission tab https://github.com/grafana/grafana/issues/77203
threshold: 5,
},
{
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=dashboard_json',
threshold: 2,
},
// Misc
{
url: '${HOST}/?orgId=1&search=open',
threshold: 0,
},
{
url: '${HOST}/alerting/list',
// the unified alerting promotion alert's content contrast is too low
// see https://github.com/grafana/grafana/pull/41829
threshold: 7,
},
{
url: '${HOST}/datasources',
threshold: 0,
},
{
url: '${HOST}/org/users',
threshold: 2,
},
{
url: '${HOST}/org/teams',
threshold: 1,
},
{
url: '${HOST}/plugins',
threshold: 0,
},
{
url: '${HOST}/org',
threshold: 2,
},
{
url: '${HOST}/org/apikeys',
threshold: 4,
},
{
url: '${HOST}/dashboards',
threshold: 2,
},
],
};
function myPa11yCiConfiguration(urls, defaults) {
const HOST_SERVER = process.env.HOST || 'localhost';
const PORT_SERVER = process.env.PORT || '3001';
const noThresholds = process.env.NO_THRESHOLDS === 'true';
urls = urls.map((test, index) => {
return {
...test,
url: test.url.replace('${HOST}', `${HOST_SERVER}:${PORT_SERVER}`),
screenCapture: `./screenshots/screenshot-${index}.png`,
rootElement: '.main-view',
wait: 500,
// Depending on NO_THRESHOLDS (--no-threshold-fail in the dagger command), clear the thresholds
// to allow pa11y to fail the check and include error details in the results file
threshold: noThresholds ? undefined : test.threshold,
};
});
return {
defaults: defaults,
urls: urls,
};
}
module.exports = myPa11yCiConfiguration(config.urls, config.defaults);
+1 -1
View File
@@ -91,7 +91,7 @@ func run(ctx context.Context, cmd *cli.Command) error {
hostSrc := d.Host().Directory(grafanaDir, dagger.HostDirectoryOpts{
Include: []string{
"./devenv",
"./e2e-playwright/test-plugins", // Directory is included so provisioning works, but they're not actually build
"./e2e/test-plugins", // Directory is included so provisioning works, but they're not actually build
"./scripts/grafana-server/custom.ini",
"./scripts/grafana-server/start-server",
"./scripts/grafana-server/kill-server",
+1 -1
View File
@@ -22,7 +22,7 @@ func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOp
WithExec([]string{"mkdir", "-p", "/src/grafana"}).
WithExec([]string{"tar", "--strip-components=1", "-xzf", "/src/grafana.tar.gz", "-C", "/src/grafana"}).
WithDirectory("/src/grafana/devenv", opts.HostSrc.Directory("./devenv")).
WithDirectory("/src/grafana/e2e-playwright/test-plugins", opts.HostSrc.Directory("./e2e-playwright/test-plugins")).
WithDirectory("/src/grafana/e2e/test-plugins", opts.HostSrc.Directory("./e2e/test-plugins")).
WithDirectory("/src/grafana/scripts", opts.HostSrc.Directory("./scripts")).
WithWorkdir("/src/grafana").
WithEnvVariable("GF_APP_MODE", "development").
-22
View File
@@ -1,22 +0,0 @@
# Pa11y accessability tests
We use pa11y to run some automated simple accessability tests. They're ran with dagger to help orchestrate starting server + tests in a reproducable manner.
To run the tests locally:
1. Install dagger locally https://docs.dagger.io/install/
2. Grab the grafana.tar.gz artifact by either
1. Downloading it from the Github Action artifact from your PR
1. Build it locally with:
```sh
dagger run go run ./pkg/build/cmd artifacts -a targz:grafana:linux/amd64 --grafana-dir="$PWD" > dist/files.txt
cat dist/files.txt # Will output the path to the grafana.tar.gz
```
3. Run the dagger pipeline with:
```sh
dagger -v run go run ./pkg/build/a11y --package=(full path to .tar.gz) --results=./pa11y-ci-results.json
```
The JSON results file will be saved to the file from the `--results` arg
4. If they fail and you want to see the full output
1. Run the dagger command with `dagger -vE [...]`
2. At the end, arrow up to the exec pa11y-ci segment and hit Enter
-101
View File
@@ -1,101 +0,0 @@
package main
import (
"context"
"fmt"
"strings"
"dagger.io/dagger"
)
var (
// Locations in the container to write results to
testResultsDir = "/playwright-test-results"
htmlResultsDir = "/playwright-html-report"
blobResultsDir = "/playwright-blob-report"
)
type RunTestOpts struct {
GrafanaService *dagger.Service
FrontendContainer *dagger.Container
HostSrc *dagger.Directory
Shard string
HTMLReportExportDir string
BlobReportExportDir string
TestResultsExportDir string
}
func RunTest(
ctx context.Context,
d *dagger.Client,
opts RunTestOpts,
) (*dagger.Container, error) {
playwrightCommand := buildPlaywrightCommand(opts)
e2eContainer := opts.FrontendContainer.
WithWorkdir("/src").
WithDirectory("/src", opts.HostSrc).
WithMountedCache(".nx", d.CacheVolume("nx-cache")).
WithEnvVariable("HOST", grafanaHost).
WithEnvVariable("PORT", fmt.Sprint(grafanaPort)).
WithServiceBinding(grafanaHost, opts.GrafanaService).
WithEnvVariable("bustcache", "1").
WithEnvVariable("PLAYWRIGHT_HTML_OPEN", "never").
WithEnvVariable("PLAYWRIGHT_HTML_OUTPUT_DIR", htmlResultsDir).
WithEnvVariable("PLAYWRIGHT_BLOB_OUTPUT_DIR", blobResultsDir).
WithExec(playwrightCommand, dagger.ContainerWithExecOpts{
Expect: dagger.ReturnTypeAny,
})
if opts.TestResultsExportDir != "" {
_, err := e2eContainer.Directory(testResultsDir).Export(ctx, opts.TestResultsExportDir)
if err != nil {
return nil, err
}
}
if opts.HTMLReportExportDir != "" {
_, err := e2eContainer.Directory(htmlResultsDir).Export(ctx, opts.HTMLReportExportDir)
if err != nil {
return nil, err
}
}
if opts.BlobReportExportDir != "" {
_, err := e2eContainer.Directory(blobResultsDir).Export(ctx, opts.BlobReportExportDir)
if err != nil {
return nil, err
}
}
return e2eContainer, nil
}
func buildPlaywrightCommand(opts RunTestOpts) []string {
playwrightReporters := []string{
"dot", // minimal output in shards
}
if opts.HTMLReportExportDir != "" {
playwrightReporters = append(playwrightReporters, "html")
}
if opts.BlobReportExportDir != "" {
playwrightReporters = append(playwrightReporters, "blob")
}
playwrightCommand := []string{
"yarn",
"e2e:playwright",
"--reporter",
strings.Join(playwrightReporters, ","),
"--output",
testResultsDir,
}
if opts.Shard != "" {
playwrightCommand = append(playwrightCommand, "--shard", opts.Shard)
}
return playwrightCommand
}
-84
View File
@@ -1,84 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
"dagger.io/dagger"
)
type Deps struct {
NodeVersion string
PlaywrightVersion string
}
// Create a container with frontend dependencies installed, ready to build plugins
// or run e2e tests.
// Theoretically we would setup Playwright in e2e.go, but to optimise layer caching
// we want it to happen before yarn install.
func WithFrontendContainer(ctx context.Context, d *dagger.Client, yarnHostSrc *dagger.Directory) (*dagger.Container, error) {
deps, err := GetVersions(ctx, yarnHostSrc)
if err != nil {
return nil, err
}
nodeBase := WithNode(d, deps.NodeVersion)
playwrightBase := WithPlaywright(d, nodeBase, deps.PlaywrightVersion)
return WithYarnInstall(d, playwrightBase, yarnHostSrc), nil
}
func GetVersions(ctx context.Context, src *dagger.Directory) (Deps, error) {
nvmrc, err := src.File(".nvmrc").Contents(ctx)
if err != nil {
return Deps{}, err
}
pkgJSON, err := src.File("package.json").Contents(ctx)
if err != nil {
return Deps{}, err
}
// parse package.json
var pkgJson struct {
DevDependencies map[string]string `json:"devDependencies"`
}
if err := json.Unmarshal([]byte(pkgJSON), &pkgJson); err != nil {
return Deps{}, err
}
return Deps{
NodeVersion: strings.TrimSpace(strings.TrimPrefix(nvmrc, "v")),
PlaywrightVersion: strings.TrimSpace(pkgJson.DevDependencies["@playwright/test"]),
}, nil
}
func WithNode(d *dagger.Client, version string) *dagger.Container {
nodeImage := fmt.Sprintf("node:%s-slim", strings.TrimPrefix(version, "v"))
return d.Container().From(nodeImage)
}
func WithPlaywright(d *dagger.Client, base *dagger.Container, version string) *dagger.Container {
brCache := d.CacheVolume("playwright-browsers")
return base.
WithEnvVariable("PLAYWRIGHT_BROWSERS_PATH", "/playwright-cache").
WithMountedCache("/playwright-cache", brCache).
WithExec([]string{"npx", "-y", "playwright@" + version, "install", "--with-deps"})
}
func WithYarnInstall(d *dagger.Client, base *dagger.Container, yarnHostSrc *dagger.Directory) *dagger.Container {
yarnCache := d.CacheVolume("yarn-cache")
return base.
WithWorkdir("/src").
WithMountedCache("/.yarn", yarnCache).
WithEnvVariable("YARN_CACHE_FOLDER", "/.yarn").
WithEnvVariable("CYPRESS_INSTALL_BINARY", "0"). // Don't download Cypress binaries
// It's important to copy all files here because the whole src directory is then copied into the test runner container
WithDirectory("/src", yarnHostSrc).
WithExec([]string{"corepack", "enable"}).
WithExec([]string{"corepack", "install"}).
WithExec([]string{"yarn", "install", "--immutable"})
}
-243
View File
@@ -1,243 +0,0 @@
package main
import (
"context"
"fmt"
"log"
"os"
"os/signal"
"dagger.io/dagger"
"github.com/urfave/cli/v3"
)
var (
grafanaHost = "grafana"
grafanaPort = 3001
)
func main() {
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
defer cancel()
if err := NewApp().Run(ctx, os.Args); err != nil {
cancel()
fmt.Println(err)
os.Exit(1)
}
}
func NewApp() *cli.Command {
return &cli.Command{
Name: "a11y",
Usage: "Run Grafana playwright e2e tests",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "grafana-dir",
Usage: "Path to the grafana/grafana clone directory",
Value: ".",
Validator: mustBeDir("grafana-dir", false, false),
TakesFile: true,
},
&cli.StringFlag{
Name: "package",
Usage: "Path to the grafana tar.gz package",
Value: "grafana.tar.gz",
Validator: mustBeFile("package", false),
TakesFile: true,
},
&cli.StringFlag{
Name: "license",
Usage: "Path to the Grafana Enterprise license file (optional)",
Validator: mustBeFile("license", true),
TakesFile: true,
},
&cli.StringFlag{
Name: "shard",
Usage: "Test shard to run. See Playwright docs",
},
&cli.StringFlag{
Name: "results-dir",
Usage: "Path to the directory to export the playwright test results to (optional)",
Validator: mustBeDir("results-dir", true, true),
},
&cli.StringFlag{
Name: "html-dir",
Usage: "Enables the HTML reporter, exported to this directory (optional)",
Validator: mustBeDir("html-dir", true, true),
},
&cli.StringFlag{
Name: "blob-dir",
Usage: "Enables the blob reporter, exported to this directory. Useful with --shard (optional)",
Validator: mustBeDir("blob-dir", true, true),
},
},
Action: run,
}
}
func run(ctx context.Context, cmd *cli.Command) error {
grafanaDir := cmd.String("grafana-dir")
targzPath := cmd.String("package")
licensePath := cmd.String("license")
pwShard := cmd.String("shard")
resultsDir := cmd.String("results-dir")
htmlDir := cmd.String("html-dir")
blobDir := cmd.String("blob-dir")
// pa11yConfigPath := cmd.String("config")
// pa11yResultsPath := cmd.String("results")
// noThresholdFail := cmd.Bool("no-threshold-fail")
d, err := dagger.Connect(ctx)
if err != nil {
return fmt.Errorf("failed to connect to Dagger: %w", err)
}
// Explicitly only the files used by the grafana-server service
grafanaHostSrc := d.Host().Directory(grafanaDir, dagger.HostDirectoryOpts{
Include: []string{
"./devenv",
// Must build test plugins to run e2e tests
"./e2e-playwright/test-plugins",
"./packages/grafana-plugin-configs",
"./scripts/grafana-server/custom.ini",
"./scripts/grafana-server/start-server",
"./scripts/grafana-server/kill-server",
"./scripts/grafana-server/variables",
},
})
// Minimal files needed to run yarn install
yarnHostSrc := d.Host().Directory(grafanaDir, dagger.HostDirectoryOpts{
Include: []string{
"package.json",
"yarn.lock",
".yarnrc.yml",
".yarn",
"packages/*/package.json",
"packages/grafana-plugin-configs",
"public/app/plugins/*/*/package.json",
"e2e-playwright/test-plugins/*/package.json",
".nvmrc",
},
})
// Files needed to run e2e tests. yarnHostSrc will be copied into the test runner container as well.
e2eHostSrc := d.Host().Directory(".", dagger.HostDirectoryOpts{
Include: []string{
"public/app/types/*.d.ts",
"public/app/core/icons/cached.json",
// packages we use in playwright tests
"packages", // TODO: do we need all of this?
"public/app/plugins", // TODO: do we need all of this?
// e2e files
"e2e-playwright",
"e2e-playwright/test-plugins",
"playwright.config.ts",
},
Exclude: []string{
"**/dist",
},
})
frontendContainer, err := WithFrontendContainer(ctx, d, yarnHostSrc)
if err != nil {
return fmt.Errorf("failed to create frontend container: %w", err)
}
targz := d.Host().File(targzPath)
var license *dagger.File
if licensePath != "" {
license = d.Host().File(licensePath)
}
svc, err := GrafanaService(ctx, d, GrafanaServiceOpts{
HostSrc: grafanaHostSrc,
FrontendContainer: frontendContainer,
GrafanaTarGz: targz,
License: license,
})
if err != nil {
return fmt.Errorf("failed to create Grafana service: %w", err)
}
runOpts := RunTestOpts{
GrafanaService: svc,
FrontendContainer: frontendContainer,
HostSrc: e2eHostSrc,
Shard: pwShard,
TestResultsExportDir: resultsDir,
HTMLReportExportDir: htmlDir,
BlobReportExportDir: blobDir,
}
c, runErr := RunTest(ctx, d, runOpts)
if runErr != nil {
return fmt.Errorf("failed to run e2e test suite: %w", runErr)
}
c, syncErr := c.Sync(ctx)
if syncErr != nil {
return fmt.Errorf("failed to sync e2e test suite: %w", syncErr)
}
code, codeErr := c.ExitCode(ctx)
if codeErr != nil {
return fmt.Errorf("failed to get exit code of e2e test suite: %w", codeErr)
}
if code == 0 {
log.Printf("e2e tests passed with exit code %d", code)
} else {
return fmt.Errorf("e2e tests failed with exit code %d", code)
}
log.Println("e2e tests completed successfully")
return nil
}
func mustBeFile(arg string, emptyOk bool) func(string) error {
return func(s string) error {
if s == "" {
if emptyOk {
return nil
}
return cli.Exit(arg+" cannot be empty", 1)
}
stat, err := os.Stat(s)
if err != nil {
return cli.Exit(arg+" does not exist or cannot be read: "+s, 1)
}
if stat.IsDir() {
return cli.Exit(arg+" must be a file, not a directory: "+s, 1)
}
return nil
}
}
func mustBeDir(arg string, emptyOk bool, notExistOk bool) func(string) error {
return func(s string) error {
if s == "" {
if emptyOk {
return nil
}
return cli.Exit(arg+" cannot be empty", 1)
}
stat, err := os.Stat(s)
if err != nil {
if notExistOk {
return nil
}
return cli.Exit(arg+" does not exist or cannot be read: "+s, 1)
}
if !stat.IsDir() {
return cli.Exit(arg+" must be a directory: "+s, 1)
}
return nil
}
}
-58
View File
@@ -1,58 +0,0 @@
package main
import (
"context"
"fmt"
"os"
"strings"
"dagger.io/dagger"
)
type GrafanaServiceOpts struct {
HostSrc *dagger.Directory
FrontendContainer *dagger.Container
GrafanaTarGz *dagger.File
License *dagger.File
}
func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOpts) (*dagger.Service, error) {
testPlugins := opts.FrontendContainer.
WithDirectory("e2e-playwright/test-plugins", opts.HostSrc.Directory("./e2e-playwright/test-plugins")).
WithDirectory("packages/grafana-plugin-configs", opts.HostSrc.Directory("./packages/grafana-plugin-configs")).
WithExec([]string{"yarn", "e2e:plugin:build"})
container := d.Container().From("alpine:3").
WithExec([]string{"apk", "add", "--no-cache", "bash", "tar", "netcat-openbsd", "util-linux"}).
WithMountedFile("/src/grafana.tar.gz", opts.GrafanaTarGz).
WithExec([]string{"mkdir", "-p", "/src/grafana"}).
WithExec([]string{"tar", "--strip-components=1", "-xzf", "/src/grafana.tar.gz", "-C", "/src/grafana"}).
WithDirectory("/src/grafana/devenv", opts.HostSrc.Directory("./devenv")).
WithDirectory("/src/grafana/e2e-playwright/test-plugins", testPlugins.Directory("./e2e-playwright/test-plugins")).
WithDirectory("/src/grafana/scripts", opts.HostSrc.Directory("./scripts")).
WithWorkdir("/src/grafana").
// Only set config variables here that are specific to the dagger/GHA runner.
// Prefer to configure scripts/grafana-server/custom.ini instead so they're also used
// when running locally.
WithEnvVariable("GF_SERVER_HTTP_PORT", fmt.Sprint(grafanaPort)).
WithExposedPort(grafanaPort)
var licenseArg string
if opts.License != nil {
licenseArg = "/src/license.jwt"
container = container.WithMountedFile(licenseArg, opts.License)
}
// We add all GF_ environment variables to allow for overriding Grafana configuration.
// It is unlikely the runner has any such otherwise.
for _, env := range os.Environ() {
if strings.HasPrefix(env, "GF_") {
parts := strings.SplitN(env, "=", 2)
container = container.WithEnvVariable(parts[0], parts[1])
}
}
svc := container.AsService(dagger.ContainerAsServiceOpts{Args: []string{"bash", "scripts/grafana-server/start-server", licenseArg}})
return svc, nil
}
-1
View File
@@ -82,7 +82,6 @@ func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOp
WithExec([]string{"tar", "--strip-components=1", "-xzf", "/src/grafana.tar.gz", "-C", "/src/grafana"}).
WithDirectory("/src/grafana/devenv", src.Directory("/src/devenv")).
WithDirectory("/src/grafana/e2e", src.Directory("/src/e2e")).
WithDirectory("/src/grafana/e2e-playwright/test-plugins", src.Directory("/src/e2e-playwright/test-plugins")).
WithDirectory("/src/grafana/scripts", src.Directory("/src/scripts")).
WithDirectory("/src/grafana/tools", src.Directory("/src/tools")).
WithWorkdir("/src/grafana").