diff --git a/.circleci/config.yml b/.circleci/config.yml index 42a4d1bb2b9..1d74e6e6431 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -91,6 +91,12 @@ jobs: name: run end-to-end tests command: 'env BASE_URL=http://127.0.0.1:3000 yarn e2e-tests' no_output_timeout: 5m + - store_artifacts: + path: public/e2e-test/screenShots/theTruth + destination: expected-screenshots + - store_artifacts: + path: public/e2e-test/screenShots/theOutput + destination: output-screenshots codespell: docker: @@ -662,6 +668,10 @@ workflows: - mysql-integration-test - postgres-integration-test filters: *filter-only-master + - end-to-end-test: + requires: + - grafana-docker-master + filters: *filter-only-master release: jobs: - build-all: diff --git a/public/e2e-test/core/images.ts b/public/e2e-test/core/images.ts index eb4ca3538d2..2897ba8aa2d 100644 --- a/public/e2e-test/core/images.ts +++ b/public/e2e-test/core/images.ts @@ -23,8 +23,21 @@ export const compareScreenShots = async (fileName: string) => return; } - expect(screenShotFromTest.width).toEqual(screenShotFromTruth.width); - expect(screenShotFromTest.height).toEqual(screenShotFromTruth.height); + if (screenShotFromTest.width !== screenShotFromTruth.width) { + throw new Error( + `The screenshot:[${fileName}] taken during the test has a width:[${ + screenShotFromTest.width + }] that differs from the expected: [${screenShotFromTruth.width}].` + ); + } + + if (screenShotFromTest.height !== screenShotFromTruth.height) { + throw new Error( + `The screenshot:[${fileName}] taken during the test has a width:[${ + screenShotFromTest.height + }] that differs from the expected: [${screenShotFromTruth.height}].` + ); + } const diff = new PNG({ width: screenShotFromTest.width, height: screenShotFromTruth.height }); const numDiffPixels = pixelmatch( @@ -36,7 +49,27 @@ export const compareScreenShots = async (fileName: string) => { threshold: 0.1 } ); - expect(numDiffPixels).toBe(0); + if (numDiffPixels !== 0) { + const localMessage = `\nCompare the output from expected:[${constants.screenShotsTruthDir}] with outcome:[${ + constants.screenShotsOutputDir + }]`; + const circleCIMessage = '\nCheck the Artifacts tab in the CircleCi build output for the actual screenshots.'; + const checkMessage = process.env.CIRCLE_SHA1 ? circleCIMessage : localMessage; + let msg = `\nThe screenshot:[${ + constants.screenShotsOutputDir + }/${fileName}.png] taken during the test differs by:[${numDiffPixels}] pixels from the expected.`; + msg += '\n'; + msg += checkMessage; + msg += '\n'; + msg += '\n If the difference between expected and outcome is NOT acceptable then do the following:'; + msg += '\n - Check the code for changes that causes this difference, fix that and retry.'; + msg += '\n'; + msg += '\n If the difference between expected and outcome is acceptable then do the following:'; + msg += '\n - Replace the expected image with the outcome and retry.'; + msg += '\n'; + throw new Error(msg); + } + resolve(); }; diff --git a/public/e2e-test/install/install.ts b/public/e2e-test/install/install.ts index fa71acfb785..61bfca0bfca 100644 --- a/public/e2e-test/install/install.ts +++ b/public/e2e-test/install/install.ts @@ -11,7 +11,9 @@ export const downloadBrowserIfNeeded = async (): Promise => { console.log('Did not find any local revisions for browser, downloading latest this might take a while.'); await browserFetcher.download(constants.chromiumRevision, (downloaded, total) => { - console.log(`Downloaded ${downloaded}bytes of ${total}bytes.`); + if (downloaded === total) { + console.log('Chromium successfully downloaded'); + } }); }; diff --git a/public/e2e-test/screenShots/theTruth/smoke-test-scenario.png b/public/e2e-test/screenShots/theTruth/smoke-test-scenario.png index 8ea1294d4d6..83216350230 100644 Binary files a/public/e2e-test/screenShots/theTruth/smoke-test-scenario.png and b/public/e2e-test/screenShots/theTruth/smoke-test-scenario.png differ