@grafana/e2e: screenshots and panel flow (#25203)

* Cleanup

* addPanel now supports (optional) custom dashboardUid

* addPanel now supports (optional) visualization name

* Added CLI option for updating screenshot fixtures

* Added support for console.* functions within tests

* Refactored screenshot command for greater simplicity

* addPanel now sets a unique title

* Updated lockfile

packages/grafana-e2e/cypress/plugins/compareScreenshots.js

@@ -0,0 +1,49 @@
+'use strict';
+const BlinkDiff = require('blink-diff');
+const { resolve } = require('path');
+
+// @todo use npmjs.com/pixelmatch or an available cypress plugin
+const compareSceenshots = async ({ config, screenshotsFolder, specName }) => {
+  const name = config.name || config; // @todo use `??`
+  const threshold = config.threshold || 0.001; // @todo use `??`
+
+  const imageAPath = `${screenshotsFolder}/${specName}/${name}.png`;
+  const imageBPath = resolve(`${screenshotsFolder}/../expected/${specName}/${name}.png`);
+
+  const imageOutputPath = screenshotsFolder.endsWith('actual') ? imageAPath.replace('.png', '.diff.png') : undefined;
+
+  const { code } = await new Promise((resolve, reject) => {
+    new BlinkDiff({
+      imageAPath,
+      imageBPath,
+      imageOutputPath,
+      threshold,
+      thresholdType: BlinkDiff.THRESHOLD_PERCENT,
+    }).run((error, result) => {
+      if (error) {
+        reject(error);
+      } else {
+        resolve(result);
+      }
+    });
+  });
+
+  if (code <= 1) {
+    let msg = `\nThe screenshot [${imageAPath}] differs from [${imageBPath}]`;
+    msg += '\n';
+    msg += '\nCheck the Artifacts tab in the CircleCi build output for the actual screenshots.';
+    msg += '\n';
+    msg += '\n  If the difference between expected and outcome is NOT acceptable then do the following:';
+    msg += '\n    - Check the code for changes that causes this difference, fix that and retry.';
+    msg += '\n';
+    msg += '\n  If the difference between expected and outcome is acceptable then do the following:';
+    msg += '\n    - Replace the expected image with the outcome and retry.';
+    msg += '\n';
+    throw new Error(msg);
+  } else {
+    // Must return a value
+    return true;
+  }
+};
+
+module.exports = compareSceenshots;

packages/grafana-e2e/cypress/plugins/compareSnapshots.js

@@ -1,25 +0,0 @@
-const BlinkDiff = require('blink-diff');
-
-function compareSnapshotsPlugin(args) {
-  args.threshold = args.threshold || 0.001;
-
-  return new Promise((resolve, reject) => {
-    const diff = new BlinkDiff({
-      imageAPath: args.pathToFileA,
-      imageBPath: args.pathToFileB,
-      thresholdType: BlinkDiff.THRESHOLD_PERCENT,
-      threshold: args.threshold,
-      imageOutputPath: args.pathToFileA.replace('.png', '.diff.png'),
-    });
-
-    diff.run((error, result) => {
-      if (error) {
-        reject(error);
-      } else {
-        resolve(result);
-      }
-    });
-  });
-}
-
-module.exports = compareSnapshotsPlugin;

packages/grafana-e2e/cypress/plugins/extendConfig.js

@@ -4,10 +4,11 @@ const {
 } = require('fs');
 const { resolve } = require('path');
 
+// @todo use https://github.com/bahmutov/cypress-extends when possible
 module.exports = async baseConfig => {
   // From CLI
   const {
-    env: { CWD },
+    env: { CWD, UPDATE_SCREENSHOTS },
   } = baseConfig;
 
   if (CWD) {
@@ -21,7 +22,7 @@ module.exports = async baseConfig => {
       reporterOptions: {
         output: `${CWD}/cypress/report.json`,
       },
-      screenshotsFolder: `${CWD}/cypress/screenshots`,
+      screenshotsFolder: `${CWD}/cypress/screenshots/${UPDATE_SCREENSHOTS ? 'expected' : 'actual'}`,
       videosFolder: `${CWD}/cypress/videos`,
     };
 

packages/grafana-e2e/cypress/plugins/index.js

@@ -1,24 +1,22 @@
-const compareSnapshotsPlugin = require('./compareSnapshots');
+const compareScreenshots = require('./compareScreenshots');
 const extendConfig = require('./extendConfig');
 const readProvisions = require('./readProvisions');
 const typescriptPreprocessor = require('./typescriptPreprocessor');
+const { install: installConsoleLogger } = require('cypress-log-to-output');
 
 module.exports = (on, config) => {
-  // yarn build fails with:
-  // >> /Users/hugo/go/src/github.com/grafana/grafana/node_modules/stringmap/stringmap.js:99
-  // >>             throw new Error("StringMap expected string key");
-  // on('task', {
-  //   failed: require('cypress-failed-log/src/failed')(),
-  // });
   on('file:preprocessor', typescriptPreprocessor);
-  on('task', { compareSnapshotsPlugin, readProvisions });
+  on('task', { compareScreenshots, readProvisions });
   on('task', {
+    // @todo remove
     log({ message, optional }) {
       optional ? console.log(message, optional) : console.log(message);
       return null;
     },
   });
 
+  installConsoleLogger(on);
+
   // Always extend with this library's config and return for diffing
   // @todo remove this when possible: https://github.com/cypress-io/cypress/issues/5674
   return extendConfig(config);