Live: performance tests e2e part (#43915)

* #41993: live perf tests e2e part

* #41993: added bash upgrade instructions

* #41993: remove custom feature toggle

* #41993: fix typo in 'integrationFolder'

packages/grafana-e2e/cypress/plugins/benchmark/CDPDataCollector.ts

@@ -0,0 +1,136 @@
+import CDP from 'chrome-remote-interface';
+import Tracelib, { TraceEvent } from 'tracelib';
+
+import { countBy, mean } from 'lodash';
+import ProtocolProxyApi from 'devtools-protocol/types/protocol-proxy-api';
+import { CollectedData, DataCollector, DataCollectorName } from './DataCollector';
+
+type CDPDataCollectorDeps = {
+  port: number;
+};
+
+export class CDPDataCollector implements DataCollector {
+  private tracingCategories: string[];
+
+  private state: {
+    client?: CDP.Client;
+    tracingPromise?: Promise<CollectedData>;
+    traceEvents: TraceEvent[];
+  };
+
+  constructor(private deps: CDPDataCollectorDeps) {
+    this.state = this.getDefaultState();
+    this.tracingCategories = [
+      'disabled-by-default-v8.cpu_profile',
+      'disabled-by-default-v8.cpu_profiler',
+      'disabled-by-default-v8.cpu_profiler.hires',
+      'disabled-by-default-devtools.timeline.frame',
+      'disabled-by-default-devtools.timeline',
+      'disabled-by-default-devtools.timeline.inputs',
+      'disabled-by-default-devtools.timeline.stack',
+      'disabled-by-default-devtools.timeline.invalidationTracking',
+      'disabled-by-default-layout_shift.debug',
+      'disabled-by-default-cc.debug.scheduler.frames',
+      'disabled-by-default-blink.debug.display_lock',
+    ];
+  }
+
+  getName = () => DataCollectorName.CDP;
+
+  private resetState = async () => {
+    if (this.state.client) {
+      await this.state.client.close();
+    }
+    this.state = this.getDefaultState();
+  };
+
+  private getDefaultState = () => ({
+    traceEvents: [],
+  });
+
+  // workaround for type declaration issues in cdp lib
+  private asApis = (
+    client: CDP.Client
+  ): {
+    Profiler: ProtocolProxyApi.ProfilerApi;
+    Page: ProtocolProxyApi.PageApi;
+    Tracing: ProtocolProxyApi.TracingApi;
+  } => client;
+
+  private getClientApis = async () => this.asApis(await this.getClient());
+
+  private getClient = async () => {
+    if (this.state.client) {
+      return this.state.client;
+    }
+
+    const client = await CDP({ port: this.deps.port });
+
+    const { Profiler, Page } = this.asApis(client);
+    await Promise.all([Page.enable(), Profiler.enable(), Profiler.setSamplingInterval({ interval: 100 })]);
+
+    this.state.client = client;
+
+    return client;
+  };
+
+  start: DataCollector['start'] = async ({ id }) => {
+    if (this.state.tracingPromise) {
+      throw new Error(`collection in progress - can't start another one! ${id}`);
+    }
+
+    const { Tracing, Profiler } = await this.getClientApis();
+
+    await Promise.all([
+      Tracing.start({
+        bufferUsageReportingInterval: 1000,
+        traceConfig: {
+          includedCategories: this.tracingCategories,
+        },
+      }),
+      Profiler.start(),
+    ]);
+
+    Tracing.on('dataCollected', ({ value: events }) => {
+      this.state.traceEvents.push(...events);
+    });
+
+    let resolveFn: (data: CollectedData) => void;
+    this.state.tracingPromise = new Promise<CollectedData>((resolve) => {
+      resolveFn = resolve;
+    });
+    Tracing.on('tracingComplete', ({ dataLossOccurred }) => {
+      const t = new Tracelib(this.state.traceEvents);
+
+      const eventCounts = countBy(this.state.traceEvents, (ev) => ev.name);
+
+      const fps = t.getFPS();
+
+      resolveFn({
+        eventCounts,
+        fps: mean(fps.values),
+        tracingDataLoss: dataLossOccurred ? 1 : 0,
+        warnings: t.getWarningCounts(),
+      });
+    });
+  };
+
+  stop: DataCollector['stop'] = async (req) => {
+    if (!this.state.tracingPromise) {
+      throw new Error(`collection was never started - there is nothing to stop!`);
+    }
+
+    const { Tracing, Profiler } = await this.getClientApis();
+
+    // TODO: capture profiler data
+    const [, , traceData] = await Promise.all([Profiler.stop(), Tracing.end(), this.state.tracingPromise]);
+
+    await this.resetState();
+
+    return traceData;
+  };
+
+  close: DataCollector['close'] = async () => {
+    await this.resetState();
+  };
+}

packages/grafana-e2e/cypress/plugins/benchmark/DataCollector.ts

@@ -0,0 +1,14 @@
+export type CollectedData = Record<string, unknown>;
+
+export enum DataCollectorName {
+  CDP = 'CDP',
+}
+
+type DataCollectorRequest = { id: string };
+
+export type DataCollector<T extends CollectedData = CollectedData> = {
+  start: (input: DataCollectorRequest) => Promise<void>;
+  stop: (input: DataCollectorRequest) => Promise<T>;
+  getName: () => DataCollectorName;
+  close: () => Promise<void>;
+};

packages/grafana-e2e/cypress/plugins/benchmark/formatting.ts

@@ -0,0 +1,138 @@
+import { CollectedData, DataCollectorName } from './DataCollector';
+import { fromPairs } from 'lodash';
+
+type Stats = {
+  sum: number;
+  min: number;
+  max: number;
+  count: number;
+  avg: number;
+  time: number;
+};
+
+export enum MeasurementName {
+  DataRenderDelay = 'DataRenderDelay',
+}
+
+type LivePerformanceAppStats = Record<MeasurementName, Stats[]>;
+
+const isLivePerformanceAppStats = (data: CollectedData[]): data is LivePerformanceAppStats[] =>
+  data.some((st) => {
+    const stat = st?.[MeasurementName.DataRenderDelay];
+    return Array.isArray(stat) && Boolean(stat?.length);
+  });
+
+type FormattedStats = {
+  total: {
+    count: number[];
+    avg: number[];
+  };
+  lastInterval: {
+    avg: number[];
+    min: number[];
+    max: number[];
+    count: number[];
+  };
+};
+
+export const formatAppStats = (allStats: CollectedData[]) => {
+  if (!isLivePerformanceAppStats(allStats)) {
+    return {};
+  }
+
+  const names = Object.keys(MeasurementName) as MeasurementName[];
+
+  return fromPairs(
+    names.map((name) => {
+      const statsForMeasurement = allStats.map((s) => s[name]);
+      const res: FormattedStats = {
+        total: {
+          count: [],
+          avg: [],
+        },
+        lastInterval: {
+          avg: [],
+          min: [],
+          max: [],
+          count: [],
+        },
+      };
+
+      statsForMeasurement.forEach((s) => {
+        const total = s.reduce(
+          (prev, next) => {
+            prev.count += next.count;
+            prev.avg += next.avg;
+            return prev;
+          },
+          { count: 0, avg: 0 }
+        );
+        res.total.count.push(Math.round(total.count));
+        res.total.avg.push(Math.round(total.avg / s.length));
+
+        const lastInterval = s[s.length - 1];
+
+        res.lastInterval.avg.push(Math.round(lastInterval?.avg));
+        res.lastInterval.min.push(Math.round(lastInterval?.min));
+        res.lastInterval.max.push(Math.round(lastInterval?.max));
+        res.lastInterval.count.push(Math.round(lastInterval?.count));
+      });
+
+      return [name, res];
+    })
+  );
+};
+
+type CDPData = {
+  eventCounts: Record<string, unknown>;
+  fps: number;
+  tracingDataLoss: number;
+  warnings: Record<string, unknown>;
+};
+
+const isCDPData = (data: any[]): data is CDPData[] => data.every((d) => typeof d.eventCounts === 'object');
+
+type FormattedCDPData = {
+  minorGC: number[];
+  majorGC: number[];
+  droppedFrames: number[];
+  fps: number[];
+  tracingDataLossOccurred: boolean;
+  longTaskWarnings: number[];
+};
+
+const emptyFormattedCDPData = (): FormattedCDPData => ({
+  minorGC: [],
+  majorGC: [],
+  droppedFrames: [],
+  fps: [],
+  tracingDataLossOccurred: false,
+  longTaskWarnings: [],
+});
+
+const formatCDPData = (data: any): FormattedCDPData => {
+  if (!isCDPData(data)) {
+    return emptyFormattedCDPData();
+  }
+
+  return data.reduce((acc, next) => {
+    acc.majorGC.push((next.eventCounts.MajorGC as number) ?? 0);
+    acc.minorGC.push((next.eventCounts.MinorGC as number) ?? 0);
+    acc.fps.push(Math.round(next.fps) ?? 0);
+    acc.tracingDataLossOccurred = acc.tracingDataLossOccurred || Boolean(next.tracingDataLoss);
+    acc.droppedFrames.push((next.eventCounts.DroppedFrame as number) ?? 0);
+    acc.longTaskWarnings.push((next.warnings.LongTask as number) ?? 0);
+    return acc;
+  }, emptyFormattedCDPData());
+};
+
+export const formatResults = (
+  results: Array<{ appStats: CollectedData; collectorsData: CollectedData }>
+): CollectedData => {
+  return {
+    ...formatAppStats(results.map(({ appStats }) => appStats)),
+    ...formatCDPData(results.map(({ collectorsData }) => collectorsData[DataCollectorName.CDP])),
+
+    __raw: results,
+  };
+};

packages/grafana-e2e/cypress/plugins/benchmark/index.ts

@@ -0,0 +1,87 @@
+import { CollectedData, DataCollector } from './DataCollector';
+import { CDPDataCollector } from './CDPDataCollector';
+import { fromPairs } from 'lodash';
+import fs from 'fs';
+import { formatResults } from './formatting';
+const remoteDebuggingPortOptionPrefix = '--remote-debugging-port=';
+
+const getOrAddRemoteDebuggingPort = (args: string[]) => {
+  const existing = args.find((arg) => arg.startsWith(remoteDebuggingPortOptionPrefix));
+
+  if (existing) {
+    return Number(existing.substring(remoteDebuggingPortOptionPrefix.length));
+  }
+
+  const port = 40000 + Math.round(Math.random() * 25000);
+  args.push(`${remoteDebuggingPortOptionPrefix}${port}`);
+  return port;
+};
+
+let collectors: DataCollector[] = [];
+let results: Array<{ appStats: CollectedData; collectorsData: CollectedData }> = [];
+
+const startBenchmarking = async ({ testName }: { testName: string }) => {
+  await Promise.all(collectors.map((coll) => coll.start({ id: testName })));
+
+  return true;
+};
+
+const stopBenchmarking = async ({ testName, appStats }: { testName: string; appStats: CollectedData }) => {
+  const data = await Promise.all(collectors.map(async (coll) => [coll.getName(), await coll.stop({ id: testName })]));
+
+  results.push({
+    collectorsData: fromPairs(data),
+    appStats: appStats,
+  });
+
+  return true;
+};
+const afterRun = async () => {
+  await Promise.all(collectors.map((coll) => coll.close()));
+  collectors = [];
+  results = [];
+};
+
+const afterSpec = (resultsFolder: string) => async (spec: { name: string }) => {
+  fs.writeFileSync(`${resultsFolder}/${spec.name}-${Date.now()}.json`, JSON.stringify(formatResults(results), null, 2));
+
+  results = [];
+};
+
+export const initialize: Cypress.PluginConfig = (on, config) => {
+  const resultsFolder = config.env['BENCHMARK_PLUGIN_RESULTS_FOLDER'];
+
+  if (!fs.existsSync(resultsFolder)) {
+    fs.mkdirSync(resultsFolder, { recursive: true });
+    console.log(`Created folder for benchmark results ${resultsFolder}`);
+  }
+
+  on('before:browser:launch', async (browser, options) => {
+    if (browser.family !== 'chromium' || browser.name === 'electron') {
+      throw new Error('benchmarking plugin requires chrome');
+    }
+
+    const { args } = options;
+
+    const port = getOrAddRemoteDebuggingPort(args);
+    collectors.push(new CDPDataCollector({ port }));
+
+    args.push('--start-fullscreen');
+
+    console.log(
+      `initialized benchmarking plugin with ${collectors.length} collectors: ${collectors
+        .map((col) => col.getName())
+        .join(', ')}`
+    );
+
+    return options;
+  });
+
+  on('task', {
+    startBenchmarking,
+    stopBenchmarking,
+  });
+
+  on('after:run', afterRun);
+  on('after:spec', afterSpec(resultsFolder));
+};

packages/grafana-e2e/cypress/plugins/benchmark/tracelib.d.ts

@@ -0,0 +1,15 @@
+type TraceEvent = {
+  name: string;
+};
+
+declare class Tracelib {
+  constructor(private events: TraceEvent[]) {}
+
+  getFPS: () => { times: number[]; values: number[] };
+  getWarningCounts: () => Record<string, number>;
+}
+declare module 'tracelib' {
+  export = Tracelib;
+
+  export { TraceEvent };
+}

packages/grafana-e2e/cypress/plugins/index.js

@@ -5,8 +5,13 @@ const compareScreenshots = require('./compareScreenshots');
 const extendConfig = require('./extendConfig');
 const readProvisions = require('./readProvisions');
 const typescriptPreprocessor = require('./typescriptPreprocessor');
+const benchmarkPlugin = require('./benchmark');
 
 module.exports = (on, config) => {
+  if (config.env['BENCHMARK_PLUGIN_ENABLED'] === true) {
+    benchmarkPlugin.initialize(on, config);
+  }
+
   on('file:preprocessor', typescriptPreprocessor);
   on('task', { compareScreenshots, readProvisions });
   on('task', {

packages/grafana-e2e/cypress/support/commands.ts

@@ -31,3 +31,11 @@ Cypress.Commands.add('getJSONFilesFromDir', (dirPath: string) => {
     relativePath: dirPath,
   });
 });
+
+Cypress.Commands.add('startBenchmarking', (testName: string) => {
+  return cy.task('startBenchmarking', { testName });
+});
+
+Cypress.Commands.add('stopBenchmarking', (testName: string, appStats: Record<string, unknown>) => {
+  return cy.task('stopBenchmarking', { testName, appStats });
+});

packages/grafana-e2e/cypress/support/index.d.ts

@@ -6,5 +6,7 @@ declare namespace Cypress {
     logToConsole(message: string, optional?: any): void;
     readProvisions(filePaths: string[]): Chainable;
     getJSONFilesFromDir(dirPath: string): Chainable;
+    startBenchmarking(testName: string): void;
+    stopBenchmarking(testName: string, appStats: Record<string, unknown>): void;
   }
 }

packages/grafana-e2e/cypress/tsconfig.json

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "declaration": false,
     "module": "commonjs",
-    "types": ["cypress", "cypress-file-upload"]
+    "types": ["cypress", "cypress-file-upload", "node"]
   },
   "extends": "@grafana/tsconfig",
   "include": ["**/*.ts"]

packages/grafana-e2e/package.json

@@ -26,12 +26,15 @@
     "docsExtract": "mkdir -p ../../reports/docs && api-extractor run 2>&1 | tee ../../reports/docs/$(basename $(pwd)).log",
     "open": "cypress open",
     "start": "cypress run --browser=chrome",
+    "start-benchmark": "CYPRESS_NO_COMMAND_LOG=1 yarn start",
     "test": "pushd test && node ../dist/bin/grafana-e2e.js run",
     "typecheck": "tsc --noEmit"
   },
   "devDependencies": {
     "@rollup/plugin-commonjs": "21.0.1",
     "@rollup/plugin-node-resolve": "13.1.3",
+    "@types/chrome-remote-interface": "0.31.4",
+    "@types/lodash": "4.14.149",
     "@types/node": "16.11.19",
     "@types/uuid": "8.3.4",
     "rollup": "2.63.0",
@@ -50,13 +53,17 @@
     "@mochajs/json-file-reporter": "^1.2.0",
     "babel-loader": "8.2.3",
     "blink-diff": "1.0.13",
+    "chrome-remote-interface": "0.31.1",
     "commander": "8.3.0",
     "cypress": "9.2.0",
     "cypress-file-upload": "5.0.8",
+    "devtools-protocol": "0.0.927104",
     "execa": "5.1.1",
+    "lodash": "4.17.21",
     "mocha": "9.1.3",
     "resolve-as-bin": "2.1.0",
     "rimraf": "3.0.2",
+    "tracelib": "1.0.1",
     "ts-loader": "6.2.1",
     "tslib": "2.3.1",
     "typescript": "4.5.4",

packages/grafana-e2e/src/flows/importDashboard.ts

@@ -13,8 +13,9 @@ export type Dashboard = { title: string; panels: Panel[]; uid: string; [key: str
  * Smoke test a particular dashboard by quickly importing a json file and validate that all the panels finish loading
  * @param dashboardToImport a sample dashboard
  * @param queryTimeout a number of ms to wait for the imported dashboard to finish loading
+ * @param skipPanelValidation skip panel validation
  */
-export const importDashboard = (dashboardToImport: Dashboard, queryTimeout?: number) => {
+export const importDashboard = (dashboardToImport: Dashboard, queryTimeout?: number, skipPanelValidation?: boolean) => {
   e2e().visit(fromBaseUrl('/dashboard/import'));
 
   // Note: normally we'd use 'click' and then 'type' here, but the json object is so big that using 'val' is much faster
@@ -45,21 +46,25 @@ export const importDashboard = (dashboardToImport: Dashboard, queryTimeout?: num
       expect(dashboardToImport.uid).to.equal(uid);
     });
 
-  dashboardToImport.panels.forEach((panel) => {
-    // Look at the json data
-    e2e.components.Panels.Panel.title(panel.title).should('be.visible').click();
-    e2e.components.Panels.Panel.headerItems('Inspect').should('be.visible').click();
-    e2e.components.Tab.title('JSON').should('be.visible').click();
-    e2e.components.PanelInspector.Json.content().should('be.visible').contains('Panel JSON').click({ force: true });
-    e2e.components.Select.option().should('be.visible').contains('Data').click();
+  if (!skipPanelValidation) {
+    dashboardToImport.panels.forEach((panel) => {
+      // Look at the json data
+      e2e.components.Panels.Panel.title(panel.title).should('be.visible').click();
+      e2e.components.Panels.Panel.headerItems('Inspect').should('be.visible').click();
+      e2e.components.Tab.title('JSON').should('be.visible').click();
+      e2e.components.PanelInspector.Json.content().should('be.visible').contains('Panel JSON').click({ force: true });
+      e2e.components.Select.option().should('be.visible').contains('Data').click();
 
-    // ensures that panel has loaded without knowingly hitting an error
-    // note: this does not prove that data came back as we expected it,
-    // it could get `state: Done` for no data for example
-    // but it ensures we didn't hit a 401 or 500 or something like that
-    e2e.components.CodeEditor.container().should('be.visible').contains('"state": "Done"');
+      // ensures that panel has loaded without knowingly hitting an error
+      // note: this does not prove that data came back as we expected it,
+      // it could get `state: Done` for no data for example
+      // but it ensures we didn't hit a 401 or 500 or something like that
+      e2e.components.CodeEditor.container()
+        .should('be.visible')
+        .contains(/"state": "(Done|Streaming)"/);
 
-    // need to close panel
-    e2e.components.Drawer.General.close().click();
-  });
+      // need to close panel
+      e2e.components.Drawer.General.close().click();
+    });
+  }
 };

packages/grafana-e2e/src/flows/importDashboards.ts

@@ -7,13 +7,14 @@ import { e2e } from '../index';
  * @param dirPath the relative path to a directory which contains json files representing dashboards,
  * for example if your dashboards live in `cypress/testDashboards` you can pass `/testDashboards`
  * @param queryTimeout a number of ms to wait for the imported dashboard to finish loading
+ * @param skipPanelValidation skips panel validation
  */
-export const importDashboards = async (dirPath: string, queryTimeout?: number) => {
+export const importDashboards = async (dirPath: string, queryTimeout?: number, skipPanelValidation?: boolean) => {
   e2e()
     .getJSONFilesFromDir(dirPath)
     .then((jsonFiles: Dashboard[]) => {
       jsonFiles.forEach((file) => {
-        importDashboard(file, queryTimeout || 6000);
+        importDashboard(file, queryTimeout || 6000, skipPanelValidation);
       });
     });
 };

packages/grafana-e2e/src/index.ts

@@ -4,6 +4,7 @@
  * @packageDocumentation
  */
 import { e2eScenario, ScenarioArguments } from './support/scenario';
+import { benchmark } from './support/benchmark';
 import { getScenarioContext, setScenarioContext } from './support/scenarioContext';
 import { e2eFactory } from './support';
 import { E2ESelectors, Selectors, selectors } from '@grafana/e2e-selectors';
@@ -16,6 +17,7 @@ const e2eObject = {
   blobToBase64String: (blob: any) => Cypress.Blob.blobToBase64String(blob),
   imgSrcToBlob: (url: string) => Cypress.Blob.imgSrcToBlob(url),
   scenario: (args: ScenarioArguments) => e2eScenario(args),
+  benchmark,
   pages: e2eFactory({ selectors: selectors.pages }),
   typings,
   components: e2eFactory({ selectors: selectors.components }),

packages/grafana-e2e/src/support/benchmark.ts

@@ -0,0 +1,81 @@
+import { e2e } from '../';
+
+export interface BenchmarkArguments {
+  name: string;
+  dashboard: {
+    folder: string;
+    delayAfterOpening: number;
+    skipPanelValidation: boolean;
+  };
+  repeat: number;
+  duration: number;
+  appStats?: {
+    startCollecting?: (window: Window) => void;
+    collect: (window: Window) => Record<string, unknown>;
+  };
+  skipScenario?: boolean;
+}
+
+export const benchmark = ({
+  name,
+  skipScenario = false,
+  repeat,
+  duration,
+  appStats,
+  dashboard,
+}: BenchmarkArguments) => {
+  if (skipScenario) {
+    describe(name, () => {
+      it.skip(name, () => {});
+    });
+  }
+
+  describe(name, () => {
+    before(() => {
+      e2e.flows.login(e2e.env('USERNAME'), e2e.env('PASSWORD'));
+    });
+
+    beforeEach(() => {
+      e2e.flows.importDashboards(dashboard.folder, 1000, dashboard.skipPanelValidation);
+      Cypress.Cookies.preserveOnce('grafana_session');
+    });
+
+    afterEach(() => e2e.flows.revertAllChanges());
+    after(() => {
+      e2e().clearCookies();
+    });
+
+    Array(repeat)
+      .fill(0)
+      .map((_, i) => {
+        const testName = `${name}-${i}`;
+        return it(testName, () => {
+          e2e.flows.openDashboard();
+
+          e2e().wait(dashboard.delayAfterOpening);
+
+          if (appStats) {
+            const startCollecting = appStats.startCollecting;
+            if (startCollecting) {
+              e2e()
+                .window()
+                .then((win) => startCollecting(win));
+            }
+
+            e2e().startBenchmarking(testName);
+            e2e().wait(duration);
+
+            e2e()
+              .window()
+              .then((win) => {
+                e2e().stopBenchmarking(testName, appStats.collect(win));
+              });
+          } else {
+            e2e().startBenchmarking(testName);
+            e2e().wait(duration);
+            e2e().stopBenchmarking(testName, {});
+          }
+        });
+      });
+  });
+};