Live: performance tests e2e part (#43915)

* #41993: live perf tests e2e part

* #41993: added bash upgrade instructions

* #41993: remove custom feature toggle

* #41993: fix typo in 'integrationFolder'

packages/grafana-e2e/src/flows/importDashboard.ts

@@ -13,8 +13,9 @@ export type Dashboard = { title: string; panels: Panel[]; uid: string; [key: str
  * Smoke test a particular dashboard by quickly importing a json file and validate that all the panels finish loading
  * @param dashboardToImport a sample dashboard
  * @param queryTimeout a number of ms to wait for the imported dashboard to finish loading
+ * @param skipPanelValidation skip panel validation
  */
-export const importDashboard = (dashboardToImport: Dashboard, queryTimeout?: number) => {
+export const importDashboard = (dashboardToImport: Dashboard, queryTimeout?: number, skipPanelValidation?: boolean) => {
   e2e().visit(fromBaseUrl('/dashboard/import'));
 
   // Note: normally we'd use 'click' and then 'type' here, but the json object is so big that using 'val' is much faster
@@ -45,21 +46,25 @@ export const importDashboard = (dashboardToImport: Dashboard, queryTimeout?: num
       expect(dashboardToImport.uid).to.equal(uid);
     });
 
-  dashboardToImport.panels.forEach((panel) => {
-    // Look at the json data
-    e2e.components.Panels.Panel.title(panel.title).should('be.visible').click();
-    e2e.components.Panels.Panel.headerItems('Inspect').should('be.visible').click();
-    e2e.components.Tab.title('JSON').should('be.visible').click();
-    e2e.components.PanelInspector.Json.content().should('be.visible').contains('Panel JSON').click({ force: true });
-    e2e.components.Select.option().should('be.visible').contains('Data').click();
+  if (!skipPanelValidation) {
+    dashboardToImport.panels.forEach((panel) => {
+      // Look at the json data
+      e2e.components.Panels.Panel.title(panel.title).should('be.visible').click();
+      e2e.components.Panels.Panel.headerItems('Inspect').should('be.visible').click();
+      e2e.components.Tab.title('JSON').should('be.visible').click();
+      e2e.components.PanelInspector.Json.content().should('be.visible').contains('Panel JSON').click({ force: true });
+      e2e.components.Select.option().should('be.visible').contains('Data').click();
 
-    // ensures that panel has loaded without knowingly hitting an error
-    // note: this does not prove that data came back as we expected it,
-    // it could get `state: Done` for no data for example
-    // but it ensures we didn't hit a 401 or 500 or something like that
-    e2e.components.CodeEditor.container().should('be.visible').contains('"state": "Done"');
+      // ensures that panel has loaded without knowingly hitting an error
+      // note: this does not prove that data came back as we expected it,
+      // it could get `state: Done` for no data for example
+      // but it ensures we didn't hit a 401 or 500 or something like that
+      e2e.components.CodeEditor.container()
+        .should('be.visible')
+        .contains(/"state": "(Done|Streaming)"/);
 
-    // need to close panel
-    e2e.components.Drawer.General.close().click();
-  });
+      // need to close panel
+      e2e.components.Drawer.General.close().click();
+    });
+  }
 };

packages/grafana-e2e/src/flows/importDashboards.ts

@@ -7,13 +7,14 @@ import { e2e } from '../index';
  * @param dirPath the relative path to a directory which contains json files representing dashboards,
  * for example if your dashboards live in `cypress/testDashboards` you can pass `/testDashboards`
  * @param queryTimeout a number of ms to wait for the imported dashboard to finish loading
+ * @param skipPanelValidation skips panel validation
  */
-export const importDashboards = async (dirPath: string, queryTimeout?: number) => {
+export const importDashboards = async (dirPath: string, queryTimeout?: number, skipPanelValidation?: boolean) => {
   e2e()
     .getJSONFilesFromDir(dirPath)
     .then((jsonFiles: Dashboard[]) => {
       jsonFiles.forEach((file) => {
-        importDashboard(file, queryTimeout || 6000);
+        importDashboard(file, queryTimeout || 6000, skipPanelValidation);
       });
     });
 };

packages/grafana-e2e/src/index.ts

@@ -4,6 +4,7 @@
  * @packageDocumentation
  */
 import { e2eScenario, ScenarioArguments } from './support/scenario';
+import { benchmark } from './support/benchmark';
 import { getScenarioContext, setScenarioContext } from './support/scenarioContext';
 import { e2eFactory } from './support';
 import { E2ESelectors, Selectors, selectors } from '@grafana/e2e-selectors';
@@ -16,6 +17,7 @@ const e2eObject = {
   blobToBase64String: (blob: any) => Cypress.Blob.blobToBase64String(blob),
   imgSrcToBlob: (url: string) => Cypress.Blob.imgSrcToBlob(url),
   scenario: (args: ScenarioArguments) => e2eScenario(args),
+  benchmark,
   pages: e2eFactory({ selectors: selectors.pages }),
   typings,
   components: e2eFactory({ selectors: selectors.components }),

packages/grafana-e2e/src/support/benchmark.ts

@@ -0,0 +1,81 @@
+import { e2e } from '../';
+
+export interface BenchmarkArguments {
+  name: string;
+  dashboard: {
+    folder: string;
+    delayAfterOpening: number;
+    skipPanelValidation: boolean;
+  };
+  repeat: number;
+  duration: number;
+  appStats?: {
+    startCollecting?: (window: Window) => void;
+    collect: (window: Window) => Record<string, unknown>;
+  };
+  skipScenario?: boolean;
+}
+
+export const benchmark = ({
+  name,
+  skipScenario = false,
+  repeat,
+  duration,
+  appStats,
+  dashboard,
+}: BenchmarkArguments) => {
+  if (skipScenario) {
+    describe(name, () => {
+      it.skip(name, () => {});
+    });
+  }
+
+  describe(name, () => {
+    before(() => {
+      e2e.flows.login(e2e.env('USERNAME'), e2e.env('PASSWORD'));
+    });
+
+    beforeEach(() => {
+      e2e.flows.importDashboards(dashboard.folder, 1000, dashboard.skipPanelValidation);
+      Cypress.Cookies.preserveOnce('grafana_session');
+    });
+
+    afterEach(() => e2e.flows.revertAllChanges());
+    after(() => {
+      e2e().clearCookies();
+    });
+
+    Array(repeat)
+      .fill(0)
+      .map((_, i) => {
+        const testName = `${name}-${i}`;
+        return it(testName, () => {
+          e2e.flows.openDashboard();
+
+          e2e().wait(dashboard.delayAfterOpening);
+
+          if (appStats) {
+            const startCollecting = appStats.startCollecting;
+            if (startCollecting) {
+              e2e()
+                .window()
+                .then((win) => startCollecting(win));
+            }
+
+            e2e().startBenchmarking(testName);
+            e2e().wait(duration);
+
+            e2e()
+              .window()
+              .then((win) => {
+                e2e().stopBenchmarking(testName, appStats.collect(win));
+              });
+          } else {
+            e2e().startBenchmarking(testName);
+            e2e().wait(duration);
+            e2e().stopBenchmarking(testName, {});
+          }
+        });
+      });
+  });
+};