From 75d7f9072c874091862f67a5cc7151d7679d3a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 16 Mar 2020 14:35:55 +0100 Subject: [PATCH] E2E: Add new run modes for e2e tests (#22795) * WIP: started e2e tests for bar gauge * Updated * Updated * Updated cypress from 3.7 -> 4.1 * reverted cypress upgrade * Updated test * Update e2e docs * Updated docs --- .circleci/config.yml | 7 +- .gitignore | 3 +- contribute/style-guides/e2e.md | 65 ++++++++++++++---- e2e/run-suite | 19 ++++- .../bar-gauge-gradient (1).png | Bin 0 -> 96138 bytes .../bar-gauge.spec.ts/bar-gauge-gradient.png | Bin 0 -> 108492 bytes .../expected/bar-gauge-gradient.png | Bin 0 -> 108492 bytes e2e/suite1/specs/bar-gauge.spec.ts | 18 +++++ jest.config.e2e.js | 15 ---- package.json | 3 +- .../grafana-e2e/cypress/fixtures/example.json | 5 ++ yarn.lock | 6 ++ 12 files changed, 103 insertions(+), 38 deletions(-) create mode 100644 e2e/suite1/screenshots/bar-gauge.spec.ts/bar-gauge-gradient (1).png create mode 100644 e2e/suite1/screenshots/bar-gauge.spec.ts/bar-gauge-gradient.png create mode 100644 e2e/suite1/screenshots/expected/bar-gauge-gradient.png create mode 100644 e2e/suite1/specs/bar-gauge.spec.ts delete mode 100644 jest.config.e2e.js create mode 100644 packages/grafana-e2e/cypress/fixtures/example.json diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e42a7c7c8c..03a79831ce8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -524,11 +524,8 @@ jobs: command: ./e2e/run-suite no_output_timeout: 5m - store_artifacts: - path: e2e/suite1/screenshots/expected - destination: expected-screenshots - - store_artifacts: - path: e2e/suite1/screenshots/received - destination: received-screenshots + path: e2e/suite1/screenshots + destination: screenshots - store_artifacts: path: e2e/suite1/videos destination: output-videos diff --git a/.gitignore b/.gitignore index cc6bb1ee8ff..a808efad312 100644 --- a/.gitignore +++ b/.gitignore @@ -105,5 +105,6 @@ compilation-stats.json /packages/grafana-e2e/cypress/videos /packages/grafana-e2e/cypress/logs /e2e/server.log -/e2e/suite1/screenshots/received +/e2e/suite1/screenshots +!/e2e/suite1/screenshots/expeced/* /e2e/suite1/videos/* diff --git a/contribute/style-guides/e2e.md b/contribute/style-guides/e2e.md index 434bf394eac..005097c477f 100644 --- a/contribute/style-guides/e2e.md +++ b/contribute/style-guides/e2e.md @@ -1,19 +1,43 @@ # End to end test framework + Grafana Labs uses a minimal home grown solution built on top of Cypress for our end to end (e2e) tests. +## Commands + +- `yarn e2e` Creates an isolated grafana-server home under `/e2e/tmp` with provisioned data sources and dashboards. This + copies locally build binary and frontned assets from your repo root so you need to have a built backend and frontend + for this to run locally. The server starts on port 3001 so it does not conflict with your normal dev server. +- `yarn e2e:debug` Same as above but runs the tests in chrome and does not shutdown after completion. +- `yarn e2e:dev` Same as above but does not run any tests on startup. It lets you pick a test first. + +The above commands use some utils scripts under `/e2e` that can also be used for more control. + +- `./e2e/start-server` This creates a fresh new grafana server working dir, setup's config and starts the server. It + will also kill any previously started server that is still running using pid file at `/e2e/tmp/pid`. +- `./e2e/run-suite ` Starts cypress in different modes. + +## Test Suites + +All the integration tests are located at `e2e/suite/specs`. The page objects and reusable flows are in the +`packages/grafana-e2e` package. + ## Basic concepts + Here is a good introduction to e2e best practices: https://martinfowler.com/bliki/PageObject.html. + - `Selector`: A unique identifier that is used from the e2e framework to retrieve an element from the Browser - `Page`: An abstraction for an object that contains one or more `Selectors` - `Flow`: An abstraction that contains a sequence of actions on one or more `Pages` that can be reused and shared between tests ## Basic example + Let's start with a simple example with a single selector. For simplicity, all examples are in JSX. In our example app, we have an input that we want to type some text into during our e2e test. + ```jsx harmony
- +
``` @@ -21,25 +45,28 @@ We could define a selector using `JQuery` [type selectors](https://api.jquery.co At Grafana, we use `aria-label` as our preferred way of defining selectors instead of `data-*` attributes. This also aids in accessibility. Let's add a descriptive `aria-label` to our simple example. + ```jsx harmony
- +
``` Now that we added the `aria-label` we suddenly get more information about this particular field. It's an input field that represents a username, but there it's still not really signaling that it's part of an e2e test. The next step is to create a `Page` representation in our e2e test framework to glue the test with the real implementation using the `pageFactory` function. For that function we can supply a `url` and `selectors` like in the example below: + ```typescript export const Login = pageFactory({ - url: '/login', // used when called from Login.visit() + url: "/login", // used when called from Login.visit() selectors: { - username: 'Username input field', // used when called from Login.username().type('Hello World') + username: "Username input field", // used when called from Login.username().type('Hello World') }, }); ``` The next step is to add the `Login` page to the exported const `Pages` in `packages/grafana-e2e/src/pages/index.ts` so that it appears when we type `e2e.pages` in our IDE. + ```ecmascript 6 export const Pages = { Login, @@ -49,17 +76,21 @@ export const Pages = { }; ``` + Now that we have a `Page` called `Login` in our `Pages` const we can use that to add a selector in our html like shown below and now this really signals to future developers that it is part of an e2e test. + ```jsx harmony
- +
``` The last step in our example is to use our `Login` page as part of a test. The `pageFactory` function we used before gives us two things: + - The `url` property is used whenever we call the `visit` function and is equivalent to the Cypress function [cy.visit()](https://docs.cypress.io/api/commands/visit.html#Syntax). -> Best practice after calling `visit` is to always call `should` on a selector to prevent flaky tests when you try to access an element that isn't ready. For more information, refer to [Commands vs. assertions](https://docs.cypress.io/guides/core-concepts/retry-ability.html#Commands-vs-assertions). + > Best practice after calling `visit` is to always call `should` on a selector to prevent flaky tests when you try to access an element that isn't ready. For more information, refer to [Commands vs. assertions](https://docs.cypress.io/guides/core-concepts/retry-ability.html#Commands-vs-assertions). - Any defined selector in the `selectors` property can be accessed from the `Login` page by invoking it. This is equivalent to the result of the Cypress function [cy.get(...)](https://docs.cypress.io/api/commands/get.html#Syntax). + ```ecmascript 6 describe('Login test', () => { it('Should pass', () => { @@ -73,6 +104,7 @@ describe('Login test', () => { ``` ## Advanced example + Let's take a look at an example that uses the same `selector` for multiple items in a list for instance. In this example app we have a list of data sources that we want to click on during an e2e test. ```jsx harmony @@ -80,15 +112,14 @@ Let's take a look at an example that uses the same `selector` for multiple items {dataSources.map(dataSource => (
  • -
    - {dataSource.name} -
    +
    {dataSource.name}
    • ))} ``` -``` + +```` Just as before in the basic example we'll start by creating a page abstraction using the `pageFactory` function: ```typescript @@ -98,12 +129,14 @@ export const DataSources = pageFactory({ dataSources: (dataSourceName: string) => `Data source list item ${dataSourceName}`, }, }); -``` +```` + You might have noticed that instead of a simple `string` as the `selector`, we're using a `function` that takes a string parameter as an argument and returns a formatted string using the argument. Just as before we need to add the `DataSources` page to the exported const `Pages` in `packages/grafana-e2e/src/pages/index.ts`. The next step is to use the `dataSources` selector function as in our example below: + ```jsx harmony