From a76ce1f56751c6a08d4b290b387b545fa3179458 Mon Sep 17 00:00:00 2001 From: Kevin Minehart Date: Fri, 2 Sep 2022 10:02:10 -0500 Subject: [PATCH] CI: move `grabpl e2e tests` from grabpl to grafana (#53075) * move grabpl e2e tests --- .drone.yml | 50 +++++++++++------------ pkg/build/cmd/e2etests.go | 49 ++++++++++++++++++++++ pkg/build/cmd/flags.go | 5 +++ pkg/build/cmd/main.go | 22 ++++++++++ pkg/build/e2eutil/grafana_server.go | 63 +++++++++++++++++++++++++++++ scripts/drone/steps/lib.star | 2 +- 6 files changed, 165 insertions(+), 26 deletions(-) create mode 100644 pkg/build/cmd/e2etests.go create mode 100644 pkg/build/e2eutil/grafana_server.go diff --git a/.drone.yml b/.drone.yml index 35a9eab4eb1..3a24127df3b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -348,7 +348,7 @@ steps: name: grafana-server - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite + - ./bin/build e2e-tests --port 3001 --suite dashboards-suite depends_on: - grafana-server environment: @@ -357,7 +357,7 @@ steps: name: end-to-end-tests-dashboards-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite + - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite depends_on: - grafana-server environment: @@ -366,7 +366,7 @@ steps: name: end-to-end-tests-smoke-tests-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite + - ./bin/build e2e-tests --port 3001 --suite panels-suite depends_on: - grafana-server environment: @@ -375,7 +375,7 @@ steps: name: end-to-end-tests-panels-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite various-suite + - ./bin/build e2e-tests --port 3001 --suite various-suite depends_on: - grafana-server environment: @@ -1098,7 +1098,7 @@ steps: name: grafana-server - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite + - ./bin/build e2e-tests --port 3001 --suite dashboards-suite depends_on: - grafana-server environment: @@ -1107,7 +1107,7 @@ steps: name: end-to-end-tests-dashboards-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite + - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite depends_on: - grafana-server environment: @@ -1116,7 +1116,7 @@ steps: name: end-to-end-tests-smoke-tests-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite + - ./bin/build e2e-tests --port 3001 --suite panels-suite depends_on: - grafana-server environment: @@ -1125,7 +1125,7 @@ steps: name: end-to-end-tests-panels-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite various-suite + - ./bin/build e2e-tests --port 3001 --suite various-suite depends_on: - grafana-server environment: @@ -1825,7 +1825,7 @@ steps: name: grafana-server - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 + - ./bin/build e2e-tests --port 3001 --suite dashboards-suite --tries 3 depends_on: - grafana-server environment: @@ -1834,7 +1834,7 @@ steps: name: end-to-end-tests-dashboards-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 + - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 depends_on: - grafana-server environment: @@ -1843,7 +1843,7 @@ steps: name: end-to-end-tests-smoke-tests-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite --tries 3 + - ./bin/build e2e-tests --port 3001 --suite panels-suite --tries 3 depends_on: - grafana-server environment: @@ -1852,7 +1852,7 @@ steps: name: end-to-end-tests-panels-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite various-suite --tries 3 + - ./bin/build e2e-tests --port 3001 --suite various-suite --tries 3 depends_on: - grafana-server environment: @@ -2458,7 +2458,7 @@ steps: name: grafana-server - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 + - ./bin/build e2e-tests --port 3001 --suite dashboards-suite --tries 3 depends_on: - grafana-server environment: @@ -2467,7 +2467,7 @@ steps: name: end-to-end-tests-dashboards-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 + - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 depends_on: - grafana-server environment: @@ -2476,7 +2476,7 @@ steps: name: end-to-end-tests-smoke-tests-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite --tries 3 + - ./bin/build e2e-tests --port 3001 --suite panels-suite --tries 3 depends_on: - grafana-server environment: @@ -2485,7 +2485,7 @@ steps: name: end-to-end-tests-panels-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite various-suite --tries 3 + - ./bin/build e2e-tests --port 3001 --suite various-suite --tries 3 depends_on: - grafana-server environment: @@ -3794,7 +3794,7 @@ steps: name: grafana-server - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 + - ./bin/build e2e-tests --port 3001 --suite dashboards-suite --tries 3 depends_on: - grafana-server environment: @@ -3803,7 +3803,7 @@ steps: name: end-to-end-tests-dashboards-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 + - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 depends_on: - grafana-server environment: @@ -3812,7 +3812,7 @@ steps: name: end-to-end-tests-smoke-tests-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite --tries 3 + - ./bin/build e2e-tests --port 3001 --suite panels-suite --tries 3 depends_on: - grafana-server environment: @@ -3821,7 +3821,7 @@ steps: name: end-to-end-tests-panels-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite various-suite --tries 3 + - ./bin/build e2e-tests --port 3001 --suite various-suite --tries 3 depends_on: - grafana-server environment: @@ -4376,7 +4376,7 @@ steps: name: grafana-server - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 + - ./bin/build e2e-tests --port 3001 --suite dashboards-suite --tries 3 depends_on: - grafana-server environment: @@ -4385,7 +4385,7 @@ steps: name: end-to-end-tests-dashboards-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 + - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 depends_on: - grafana-server environment: @@ -4394,7 +4394,7 @@ steps: name: end-to-end-tests-smoke-tests-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite --tries 3 + - ./bin/build e2e-tests --port 3001 --suite panels-suite --tries 3 depends_on: - grafana-server environment: @@ -4403,7 +4403,7 @@ steps: name: end-to-end-tests-panels-suite - commands: - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite various-suite --tries 3 + - ./bin/build e2e-tests --port 3001 --suite various-suite --tries 3 depends_on: - grafana-server environment: @@ -5093,6 +5093,6 @@ kind: secret name: packages_secret_access_key --- kind: signature -hmac: 54f9d4c1b212da26968eb3c35278ad82bc9d204ca0f6f99eb990df1b9950516b +hmac: fb770c3cc5bd204e3aceae01d35997818ee210efb54c8ccbe37a410773f1ce73 ... diff --git a/pkg/build/cmd/e2etests.go b/pkg/build/cmd/e2etests.go new file mode 100644 index 00000000000..286fe8e8073 --- /dev/null +++ b/pkg/build/cmd/e2etests.go @@ -0,0 +1,49 @@ +package main + +import ( + "fmt" + "log" + "os" + "os/exec" + + "github.com/grafana/grafana/pkg/build/e2eutil" + "github.com/urfave/cli/v2" +) + +func EndToEndTests(c *cli.Context) error { + var ( + tries = c.Int("tries") + suite = c.String("suite") + host = c.String("host") + ) + + log.Printf("Running Grafana e2e tests") + + port := c.Int("port") + env := append(os.Environ(), fmt.Sprintf("PORT=%d", port)) + + grafanaServer := e2eutil.Server(host, port) + // TODO implement grafanaServer.Start() + grafanaServer.Wait() + + var err error + for i := 0; i < tries; i++ { + log.Printf("Running e2e test suite attempt #%d", i+1) + //nolint:gosec + cmd := exec.Command("./e2e/run-suite", suite) + cmd.Env = env + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + if err == nil { + break + } + + log.Printf("Running the test suite failed: %s", err) + } + if err != nil { + return cli.Exit(fmt.Sprintf("e2e tests failed: %s", err), 1) + } + + return nil +} diff --git a/pkg/build/cmd/flags.go b/pkg/build/cmd/flags.go index 9d8451dfe4b..3ae24027d76 100644 --- a/pkg/build/cmd/flags.go +++ b/pkg/build/cmd/flags.go @@ -20,6 +20,11 @@ var ( Name: "variants", Usage: "Comma-separated list of variants to build", } + triesFlag = cli.IntFlag{ + Name: "tries", + Usage: "Specify number of tries before failing", + Value: 1, + } noInstallDepsFlag = cli.BoolFlag{ Name: "no-install-deps", Usage: "Don't install dependencies", diff --git a/pkg/build/cmd/main.go b/pkg/build/cmd/main.go index c9bc7d87aec..9beb0b8d3ed 100644 --- a/pkg/build/cmd/main.go +++ b/pkg/build/cmd/main.go @@ -25,6 +25,28 @@ func main() { &buildIDFlag, }, }, + { + Name: "e2e-tests", + Usage: "Run Grafana e2e tests", + Action: EndToEndTests, + Flags: []cli.Flag{ + &triesFlag, + &cli.IntFlag{ + Name: "port", + Value: 3001, + Usage: "Specify the server port", + }, + &cli.StringFlag{ + Name: "suite", + Usage: "Specify the end-to-end tests suite to be used", + }, + &cli.StringFlag{ + Name: "host", + Value: "grafana-server", + Usage: "Specify the server host", + }, + }, + }, { Name: "build-frontend", Usage: "Build front-end artifacts", diff --git a/pkg/build/e2eutil/grafana_server.go b/pkg/build/e2eutil/grafana_server.go new file mode 100644 index 00000000000..b232d4a042c --- /dev/null +++ b/pkg/build/e2eutil/grafana_server.go @@ -0,0 +1,63 @@ +package e2eutil + +import ( + "fmt" + "io" + "log" + "net/http" + "os" + "time" +) + +const timeout = 60 + +type GrafanaServer struct { + Port int + Host string +} + +func Server(host string, port int) *GrafanaServer { + return &GrafanaServer{ + Host: host, + Port: port, + } +} + +func (g *GrafanaServer) Wait() { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + timeoutExceeded := time.After(timeout * time.Second) + + for { + select { + case <-timeoutExceeded: + log.Printf("grafana server failed to start after %d second(s) timeout", timeout) + os.Exit(1) + + case <-ticker.C: + url := fmt.Sprintf("http://%s:%d", g.Host, g.Port) + //nolint:gosec + resp, err := http.Get(url) + if err == nil { + body, err := io.ReadAll(resp.Body) + if err != nil { + log.Printf("failed to read response body: %q", err) + return + } + log.Println("connected to grafana-server!") + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + log.Printf("status code: %d, body: %s, exiting...", resp.StatusCode, string(body)) + os.Exit(1) + } + err = resp.Body.Close() + if err != nil { + log.Printf("error closing response body, body: %s", string(body)) + return + } + return + } + log.Printf("failed attempt to connect to grafana-server on url %s, retrying...", url) + } + } +} diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 5ef0d314def..f06e5e02802 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -714,7 +714,7 @@ def grafana_server_step(edition, port=3001): def e2e_tests_step(suite, edition, port=3001, tries=None): - cmd = './bin/grabpl e2e-tests --port {} --suite {}'.format(port, suite) + cmd = './bin/build e2e-tests --port {} --suite {}'.format(port, suite) if tries: cmd += ' --tries {}'.format(tries) return {