diff --git a/e2e/pa11yci.conf.js b/e2e/pa11yci.conf.js new file mode 100644 index 00000000000..b4ed68a5412 --- /dev/null +++ b/e2e/pa11yci.conf.js @@ -0,0 +1,140 @@ +const config = { + defaults: { + concurrency: 1, + runners: ['axe'], + useIncognitoBrowserContext: false, + standard: 'WCAG2AA', + chromeLaunchConfig: { + executablePath: '/usr/bin/google-chrome', + args: ['--no-sandbox'], + }, + // see https://github.com/grafana/grafana/pull/41693#issuecomment-979921463 for context + // on why we're ignoring singleValue/react-select-*-placeholder elements + hideElements: '#updateVersion, [class*="-singleValue"], [id^="react-select-"][id$="-placeholder"]', + reporters: ['cli', ['json', { fileName: './pa11y-ci-results.json' }]], + }, + + urls: [ + { + url: '${HOST}/login', + threshold: 0, + }, + { + url: '${HOST}/login', + actions: [ + "wait for element input[name='user'] to be added", + "set field input[name='user'] to admin", + "set field input[name='password'] to admin", + "click element button[data-testid='data-testid Login button']", + "wait for element button[data-testid='data-testid Skip change password button'] to be visible", + ], + threshold: 2, + }, + { + url: '${HOST}/?orgId=1', + threshold: 0, + }, + { + url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge', + threshold: 0, + }, + + // Dashboard settings + { + url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=settings', + threshold: 0, + }, + { + url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=annotations', + threshold: 0, + }, + { + url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=variables', + threshold: 0, + }, + { + url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=links', + threshold: 0, + }, + { + url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=versions', + threshold: 0, + }, + { + url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=permissions', + // TODO: improve the accessibility of the permission tab https://github.com/grafana/grafana/issues/77203 + threshold: 5, + }, + { + url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=dashboard_json', + threshold: 2, + }, + + // Misc + { + url: '${HOST}/?orgId=1&search=open', + threshold: 0, + }, + { + url: '${HOST}/alerting/list', + // the unified alerting promotion alert's content contrast is too low + // see https://github.com/grafana/grafana/pull/41829 + threshold: 7, + }, + { + url: '${HOST}/datasources', + threshold: 0, + }, + { + url: '${HOST}/org/users', + threshold: 2, + }, + { + url: '${HOST}/org/teams', + threshold: 1, + }, + { + url: '${HOST}/plugins', + threshold: 0, + }, + { + url: '${HOST}/org', + threshold: 2, + }, + { + url: '${HOST}/org/apikeys', + threshold: 4, + }, + { + url: '${HOST}/dashboards', + threshold: 2, + }, + ], +}; + +function myPa11yCiConfiguration(urls, defaults) { + const HOST_SERVER = process.env.HOST || 'localhost'; + const PORT_SERVER = process.env.PORT || '3001'; + const noThresholds = process.env.NO_THRESHOLDS === 'true'; + + urls = urls.map((test, index) => { + return { + ...test, + url: test.url.replace('${HOST}', `${HOST_SERVER}:${PORT_SERVER}`), + screenCapture: `./screenshots/screenshot-${index}.png`, + rootElement: '.main-view', + wait: 500, + + // Depending on NO_THRESHOLDS (--no-threshold-fail in the dagger command), clear the thresholds + // to allow pa11y to fail the check and include error details in the results file + threshold: noThresholds ? undefined : test.threshold, + }; + }); + + return { + defaults: defaults, + urls: urls, + }; +} + +module.exports = myPa11yCiConfiguration(config.urls, config.defaults); diff --git a/pkg/build/a11y/main.go b/pkg/build/a11y/main.go index dd85c2c3328..444e59213d2 100644 --- a/pkg/build/a11y/main.go +++ b/pkg/build/a11y/main.go @@ -91,7 +91,7 @@ func run(ctx context.Context, cmd *cli.Command) error { hostSrc := d.Host().Directory(grafanaDir, dagger.HostDirectoryOpts{ Include: []string{ "./devenv", - "./e2e-playwright/test-plugins", // Directory is included so provisioning works, but they're not actually build + "./e2e/test-plugins", // Directory is included so provisioning works, but they're not actually build "./scripts/grafana-server/custom.ini", "./scripts/grafana-server/start-server", "./scripts/grafana-server/kill-server", diff --git a/pkg/build/a11y/service.go b/pkg/build/a11y/service.go index 98bcb5cf20b..c866bc60520 100644 --- a/pkg/build/a11y/service.go +++ b/pkg/build/a11y/service.go @@ -22,7 +22,7 @@ func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOp WithExec([]string{"mkdir", "-p", "/src/grafana"}). WithExec([]string{"tar", "--strip-components=1", "-xzf", "/src/grafana.tar.gz", "-C", "/src/grafana"}). WithDirectory("/src/grafana/devenv", opts.HostSrc.Directory("./devenv")). - WithDirectory("/src/grafana/e2e-playwright/test-plugins", opts.HostSrc.Directory("./e2e-playwright/test-plugins")). + WithDirectory("/src/grafana/e2e/test-plugins", opts.HostSrc.Directory("./e2e/test-plugins")). WithDirectory("/src/grafana/scripts", opts.HostSrc.Directory("./scripts")). WithWorkdir("/src/grafana"). WithEnvVariable("GF_APP_MODE", "development"). diff --git a/pkg/build/e2e-playwright/README.md b/pkg/build/e2e-playwright/README.md deleted file mode 100644 index d5fc311d696..00000000000 --- a/pkg/build/e2e-playwright/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Pa11y accessability tests - -We use pa11y to run some automated simple accessability tests. They're ran with dagger to help orchestrate starting server + tests in a reproducable manner. - -To run the tests locally: - -1. Install dagger locally https://docs.dagger.io/install/ -2. Grab the grafana.tar.gz artifact by either - 1. Downloading it from the Github Action artifact from your PR - 1. Build it locally with: - ```sh - dagger run go run ./pkg/build/cmd artifacts -a targz:grafana:linux/amd64 --grafana-dir="$PWD" > dist/files.txt - cat dist/files.txt # Will output the path to the grafana.tar.gz - ``` -3. Run the dagger pipeline with: - ```sh - dagger -v run go run ./pkg/build/a11y --package=(full path to .tar.gz) --results=./pa11y-ci-results.json - ``` - The JSON results file will be saved to the file from the `--results` arg -4. If they fail and you want to see the full output - 1. Run the dagger command with `dagger -vE [...]` - 2. At the end, arrow up to the exec pa11y-ci segment and hit Enter \ No newline at end of file diff --git a/pkg/build/e2e-playwright/e2e.go b/pkg/build/e2e-playwright/e2e.go deleted file mode 100644 index 35579e8dd2f..00000000000 --- a/pkg/build/e2e-playwright/e2e.go +++ /dev/null @@ -1,101 +0,0 @@ -package main - -import ( - "context" - "fmt" - "strings" - - "dagger.io/dagger" -) - -var ( - // Locations in the container to write results to - testResultsDir = "/playwright-test-results" - htmlResultsDir = "/playwright-html-report" - blobResultsDir = "/playwright-blob-report" -) - -type RunTestOpts struct { - GrafanaService *dagger.Service - FrontendContainer *dagger.Container - HostSrc *dagger.Directory - Shard string - HTMLReportExportDir string - BlobReportExportDir string - TestResultsExportDir string -} - -func RunTest( - ctx context.Context, - d *dagger.Client, - opts RunTestOpts, -) (*dagger.Container, error) { - playwrightCommand := buildPlaywrightCommand(opts) - - e2eContainer := opts.FrontendContainer. - WithWorkdir("/src"). - WithDirectory("/src", opts.HostSrc). - WithMountedCache(".nx", d.CacheVolume("nx-cache")). - WithEnvVariable("HOST", grafanaHost). - WithEnvVariable("PORT", fmt.Sprint(grafanaPort)). - WithServiceBinding(grafanaHost, opts.GrafanaService). - WithEnvVariable("bustcache", "1"). - WithEnvVariable("PLAYWRIGHT_HTML_OPEN", "never"). - WithEnvVariable("PLAYWRIGHT_HTML_OUTPUT_DIR", htmlResultsDir). - WithEnvVariable("PLAYWRIGHT_BLOB_OUTPUT_DIR", blobResultsDir). - WithExec(playwrightCommand, dagger.ContainerWithExecOpts{ - Expect: dagger.ReturnTypeAny, - }) - - if opts.TestResultsExportDir != "" { - _, err := e2eContainer.Directory(testResultsDir).Export(ctx, opts.TestResultsExportDir) - if err != nil { - return nil, err - } - } - - if opts.HTMLReportExportDir != "" { - _, err := e2eContainer.Directory(htmlResultsDir).Export(ctx, opts.HTMLReportExportDir) - if err != nil { - return nil, err - } - } - - if opts.BlobReportExportDir != "" { - _, err := e2eContainer.Directory(blobResultsDir).Export(ctx, opts.BlobReportExportDir) - if err != nil { - return nil, err - } - } - - return e2eContainer, nil -} - -func buildPlaywrightCommand(opts RunTestOpts) []string { - playwrightReporters := []string{ - "dot", // minimal output in shards - } - - if opts.HTMLReportExportDir != "" { - playwrightReporters = append(playwrightReporters, "html") - } - - if opts.BlobReportExportDir != "" { - playwrightReporters = append(playwrightReporters, "blob") - } - - playwrightCommand := []string{ - "yarn", - "e2e:playwright", - "--reporter", - strings.Join(playwrightReporters, ","), - "--output", - testResultsDir, - } - - if opts.Shard != "" { - playwrightCommand = append(playwrightCommand, "--shard", opts.Shard) - } - - return playwrightCommand -} diff --git a/pkg/build/e2e-playwright/frontend.go b/pkg/build/e2e-playwright/frontend.go deleted file mode 100644 index 625ea155ae4..00000000000 --- a/pkg/build/e2e-playwright/frontend.go +++ /dev/null @@ -1,84 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "strings" - - "dagger.io/dagger" -) - -type Deps struct { - NodeVersion string - PlaywrightVersion string -} - -// Create a container with frontend dependencies installed, ready to build plugins -// or run e2e tests. -// Theoretically we would setup Playwright in e2e.go, but to optimise layer caching -// we want it to happen before yarn install. -func WithFrontendContainer(ctx context.Context, d *dagger.Client, yarnHostSrc *dagger.Directory) (*dagger.Container, error) { - deps, err := GetVersions(ctx, yarnHostSrc) - if err != nil { - return nil, err - } - - nodeBase := WithNode(d, deps.NodeVersion) - playwrightBase := WithPlaywright(d, nodeBase, deps.PlaywrightVersion) - - return WithYarnInstall(d, playwrightBase, yarnHostSrc), nil -} - -func GetVersions(ctx context.Context, src *dagger.Directory) (Deps, error) { - nvmrc, err := src.File(".nvmrc").Contents(ctx) - if err != nil { - return Deps{}, err - } - pkgJSON, err := src.File("package.json").Contents(ctx) - if err != nil { - return Deps{}, err - } - - // parse package.json - var pkgJson struct { - DevDependencies map[string]string `json:"devDependencies"` - } - if err := json.Unmarshal([]byte(pkgJSON), &pkgJson); err != nil { - return Deps{}, err - } - - return Deps{ - NodeVersion: strings.TrimSpace(strings.TrimPrefix(nvmrc, "v")), - PlaywrightVersion: strings.TrimSpace(pkgJson.DevDependencies["@playwright/test"]), - }, nil -} - -func WithNode(d *dagger.Client, version string) *dagger.Container { - nodeImage := fmt.Sprintf("node:%s-slim", strings.TrimPrefix(version, "v")) - return d.Container().From(nodeImage) -} - -func WithPlaywright(d *dagger.Client, base *dagger.Container, version string) *dagger.Container { - brCache := d.CacheVolume("playwright-browsers") - return base. - WithEnvVariable("PLAYWRIGHT_BROWSERS_PATH", "/playwright-cache"). - WithMountedCache("/playwright-cache", brCache). - WithExec([]string{"npx", "-y", "playwright@" + version, "install", "--with-deps"}) -} - -func WithYarnInstall(d *dagger.Client, base *dagger.Container, yarnHostSrc *dagger.Directory) *dagger.Container { - yarnCache := d.CacheVolume("yarn-cache") - - return base. - WithWorkdir("/src"). - WithMountedCache("/.yarn", yarnCache). - WithEnvVariable("YARN_CACHE_FOLDER", "/.yarn"). - WithEnvVariable("CYPRESS_INSTALL_BINARY", "0"). // Don't download Cypress binaries - - // It's important to copy all files here because the whole src directory is then copied into the test runner container - WithDirectory("/src", yarnHostSrc). - WithExec([]string{"corepack", "enable"}). - WithExec([]string{"corepack", "install"}). - WithExec([]string{"yarn", "install", "--immutable"}) -} diff --git a/pkg/build/e2e-playwright/main.go b/pkg/build/e2e-playwright/main.go deleted file mode 100644 index 09384c8614b..00000000000 --- a/pkg/build/e2e-playwright/main.go +++ /dev/null @@ -1,243 +0,0 @@ -package main - -import ( - "context" - "fmt" - "log" - "os" - "os/signal" - - "dagger.io/dagger" - "github.com/urfave/cli/v3" -) - -var ( - grafanaHost = "grafana" - grafanaPort = 3001 -) - -func main() { - ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) - defer cancel() - - if err := NewApp().Run(ctx, os.Args); err != nil { - cancel() - fmt.Println(err) - os.Exit(1) - } -} - -func NewApp() *cli.Command { - return &cli.Command{ - Name: "a11y", - Usage: "Run Grafana playwright e2e tests", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "grafana-dir", - Usage: "Path to the grafana/grafana clone directory", - Value: ".", - Validator: mustBeDir("grafana-dir", false, false), - TakesFile: true, - }, - &cli.StringFlag{ - Name: "package", - Usage: "Path to the grafana tar.gz package", - Value: "grafana.tar.gz", - Validator: mustBeFile("package", false), - TakesFile: true, - }, - &cli.StringFlag{ - Name: "license", - Usage: "Path to the Grafana Enterprise license file (optional)", - Validator: mustBeFile("license", true), - TakesFile: true, - }, - &cli.StringFlag{ - Name: "shard", - Usage: "Test shard to run. See Playwright docs", - }, - &cli.StringFlag{ - Name: "results-dir", - Usage: "Path to the directory to export the playwright test results to (optional)", - Validator: mustBeDir("results-dir", true, true), - }, - &cli.StringFlag{ - Name: "html-dir", - Usage: "Enables the HTML reporter, exported to this directory (optional)", - Validator: mustBeDir("html-dir", true, true), - }, - &cli.StringFlag{ - Name: "blob-dir", - Usage: "Enables the blob reporter, exported to this directory. Useful with --shard (optional)", - Validator: mustBeDir("blob-dir", true, true), - }, - }, - Action: run, - } -} - -func run(ctx context.Context, cmd *cli.Command) error { - grafanaDir := cmd.String("grafana-dir") - targzPath := cmd.String("package") - licensePath := cmd.String("license") - pwShard := cmd.String("shard") - resultsDir := cmd.String("results-dir") - htmlDir := cmd.String("html-dir") - blobDir := cmd.String("blob-dir") - // pa11yConfigPath := cmd.String("config") - // pa11yResultsPath := cmd.String("results") - // noThresholdFail := cmd.Bool("no-threshold-fail") - - d, err := dagger.Connect(ctx) - if err != nil { - return fmt.Errorf("failed to connect to Dagger: %w", err) - } - - // Explicitly only the files used by the grafana-server service - grafanaHostSrc := d.Host().Directory(grafanaDir, dagger.HostDirectoryOpts{ - Include: []string{ - "./devenv", - - // Must build test plugins to run e2e tests - "./e2e-playwright/test-plugins", - "./packages/grafana-plugin-configs", - - "./scripts/grafana-server/custom.ini", - "./scripts/grafana-server/start-server", - "./scripts/grafana-server/kill-server", - "./scripts/grafana-server/variables", - }, - }) - - // Minimal files needed to run yarn install - yarnHostSrc := d.Host().Directory(grafanaDir, dagger.HostDirectoryOpts{ - Include: []string{ - "package.json", - "yarn.lock", - ".yarnrc.yml", - ".yarn", - "packages/*/package.json", - "packages/grafana-plugin-configs", - "public/app/plugins/*/*/package.json", - "e2e-playwright/test-plugins/*/package.json", - ".nvmrc", - }, - }) - - // Files needed to run e2e tests. yarnHostSrc will be copied into the test runner container as well. - e2eHostSrc := d.Host().Directory(".", dagger.HostDirectoryOpts{ - Include: []string{ - "public/app/types/*.d.ts", - "public/app/core/icons/cached.json", - - // packages we use in playwright tests - "packages", // TODO: do we need all of this? - "public/app/plugins", // TODO: do we need all of this? - - // e2e files - "e2e-playwright", - "e2e-playwright/test-plugins", - "playwright.config.ts", - }, - Exclude: []string{ - "**/dist", - }, - }) - - frontendContainer, err := WithFrontendContainer(ctx, d, yarnHostSrc) - if err != nil { - return fmt.Errorf("failed to create frontend container: %w", err) - } - - targz := d.Host().File(targzPath) - - var license *dagger.File - if licensePath != "" { - license = d.Host().File(licensePath) - } - - svc, err := GrafanaService(ctx, d, GrafanaServiceOpts{ - HostSrc: grafanaHostSrc, - FrontendContainer: frontendContainer, - GrafanaTarGz: targz, - License: license, - }) - if err != nil { - return fmt.Errorf("failed to create Grafana service: %w", err) - } - - runOpts := RunTestOpts{ - GrafanaService: svc, - FrontendContainer: frontendContainer, - HostSrc: e2eHostSrc, - Shard: pwShard, - TestResultsExportDir: resultsDir, - HTMLReportExportDir: htmlDir, - BlobReportExportDir: blobDir, - } - - c, runErr := RunTest(ctx, d, runOpts) - if runErr != nil { - return fmt.Errorf("failed to run e2e test suite: %w", runErr) - } - - c, syncErr := c.Sync(ctx) - if syncErr != nil { - return fmt.Errorf("failed to sync e2e test suite: %w", syncErr) - } - - code, codeErr := c.ExitCode(ctx) - if codeErr != nil { - return fmt.Errorf("failed to get exit code of e2e test suite: %w", codeErr) - } - - if code == 0 { - log.Printf("e2e tests passed with exit code %d", code) - } else { - return fmt.Errorf("e2e tests failed with exit code %d", code) - } - - log.Println("e2e tests completed successfully") - return nil -} - -func mustBeFile(arg string, emptyOk bool) func(string) error { - return func(s string) error { - if s == "" { - if emptyOk { - return nil - } - return cli.Exit(arg+" cannot be empty", 1) - } - stat, err := os.Stat(s) - if err != nil { - return cli.Exit(arg+" does not exist or cannot be read: "+s, 1) - } - if stat.IsDir() { - return cli.Exit(arg+" must be a file, not a directory: "+s, 1) - } - return nil - } -} - -func mustBeDir(arg string, emptyOk bool, notExistOk bool) func(string) error { - return func(s string) error { - if s == "" { - if emptyOk { - return nil - } - return cli.Exit(arg+" cannot be empty", 1) - } - stat, err := os.Stat(s) - if err != nil { - if notExistOk { - return nil - } - return cli.Exit(arg+" does not exist or cannot be read: "+s, 1) - } - if !stat.IsDir() { - return cli.Exit(arg+" must be a directory: "+s, 1) - } - return nil - } -} diff --git a/pkg/build/e2e-playwright/service.go b/pkg/build/e2e-playwright/service.go deleted file mode 100644 index d162451ef70..00000000000 --- a/pkg/build/e2e-playwright/service.go +++ /dev/null @@ -1,58 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "strings" - - "dagger.io/dagger" -) - -type GrafanaServiceOpts struct { - HostSrc *dagger.Directory - FrontendContainer *dagger.Container - GrafanaTarGz *dagger.File - License *dagger.File -} - -func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOpts) (*dagger.Service, error) { - testPlugins := opts.FrontendContainer. - WithDirectory("e2e-playwright/test-plugins", opts.HostSrc.Directory("./e2e-playwright/test-plugins")). - WithDirectory("packages/grafana-plugin-configs", opts.HostSrc.Directory("./packages/grafana-plugin-configs")). - WithExec([]string{"yarn", "e2e:plugin:build"}) - - container := d.Container().From("alpine:3"). - WithExec([]string{"apk", "add", "--no-cache", "bash", "tar", "netcat-openbsd", "util-linux"}). - WithMountedFile("/src/grafana.tar.gz", opts.GrafanaTarGz). - WithExec([]string{"mkdir", "-p", "/src/grafana"}). - WithExec([]string{"tar", "--strip-components=1", "-xzf", "/src/grafana.tar.gz", "-C", "/src/grafana"}). - WithDirectory("/src/grafana/devenv", opts.HostSrc.Directory("./devenv")). - WithDirectory("/src/grafana/e2e-playwright/test-plugins", testPlugins.Directory("./e2e-playwright/test-plugins")). - WithDirectory("/src/grafana/scripts", opts.HostSrc.Directory("./scripts")). - WithWorkdir("/src/grafana"). - // Only set config variables here that are specific to the dagger/GHA runner. - // Prefer to configure scripts/grafana-server/custom.ini instead so they're also used - // when running locally. - WithEnvVariable("GF_SERVER_HTTP_PORT", fmt.Sprint(grafanaPort)). - WithExposedPort(grafanaPort) - - var licenseArg string - if opts.License != nil { - licenseArg = "/src/license.jwt" - container = container.WithMountedFile(licenseArg, opts.License) - } - - // We add all GF_ environment variables to allow for overriding Grafana configuration. - // It is unlikely the runner has any such otherwise. - for _, env := range os.Environ() { - if strings.HasPrefix(env, "GF_") { - parts := strings.SplitN(env, "=", 2) - container = container.WithEnvVariable(parts[0], parts[1]) - } - } - - svc := container.AsService(dagger.ContainerAsServiceOpts{Args: []string{"bash", "scripts/grafana-server/start-server", licenseArg}}) - - return svc, nil -} diff --git a/pkg/build/e2e/service.go b/pkg/build/e2e/service.go index 5c6eb0c4e78..95de093ec18 100644 --- a/pkg/build/e2e/service.go +++ b/pkg/build/e2e/service.go @@ -82,7 +82,6 @@ func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOp WithExec([]string{"tar", "--strip-components=1", "-xzf", "/src/grafana.tar.gz", "-C", "/src/grafana"}). WithDirectory("/src/grafana/devenv", src.Directory("/src/devenv")). WithDirectory("/src/grafana/e2e", src.Directory("/src/e2e")). - WithDirectory("/src/grafana/e2e-playwright/test-plugins", src.Directory("/src/e2e-playwright/test-plugins")). WithDirectory("/src/grafana/scripts", src.Directory("/src/scripts")). WithDirectory("/src/grafana/tools", src.Directory("/src/tools")). WithWorkdir("/src/grafana").