Live: performance tests e2e part (#43915)
* #41993: live perf tests e2e part * #41993: added bash upgrade instructions * #41993: remove custom feature toggle * #41993: fix typo in 'integrationFolder'
This commit is contained in:
Artur Wierzbicki
@@ -136,6 +136,8 @@ compilation-stats.json
|
|||||||
/e2e/**/screenshots
|
/e2e/**/screenshots
|
||||||
!/e2e/**/screenshots/expected/*
|
!/e2e/**/screenshots/expected/*
|
||||||
/e2e/**/videos/*
|
/e2e/**/videos/*
|
||||||
|
/e2e/benchmarks/**/results/*
|
||||||
|
/e2e/benchmarks/**/results
|
||||||
|
|
||||||
# a11y tests
|
# a11y tests
|
||||||
/pa11y-ci-results.json
|
/pa11y-ci-results.json
|
||||||
|
|||||||
@@ -0,0 +1,399 @@
|
|||||||
|
{
|
||||||
|
"annotations": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"builtIn": 1,
|
||||||
|
"datasource": "-- Grafana --",
|
||||||
|
"enable": true,
|
||||||
|
"hide": true,
|
||||||
|
"iconColor": "rgba(0, 211, 255, 1)",
|
||||||
|
"name": "Annotations & Alerts",
|
||||||
|
"target": {
|
||||||
|
"limit": 100,
|
||||||
|
"matchAny": false,
|
||||||
|
"tags": [],
|
||||||
|
"type": "dashboard"
|
||||||
|
},
|
||||||
|
"type": "dashboard"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"editable": true,
|
||||||
|
"fiscalYearStartMonth": 0,
|
||||||
|
"graphTooltip": 0,
|
||||||
|
"id": 82,
|
||||||
|
"links": [],
|
||||||
|
"liveNow": false,
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "grafana",
|
||||||
|
"uid": "grafana"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"custom": {
|
||||||
|
"axisLabel": "",
|
||||||
|
"axisPlacement": "auto",
|
||||||
|
"barAlignment": 0,
|
||||||
|
"drawStyle": "line",
|
||||||
|
"fillOpacity": 0,
|
||||||
|
"gradientMode": "none",
|
||||||
|
"hideFrom": {
|
||||||
|
"legend": false,
|
||||||
|
"tooltip": false,
|
||||||
|
"viz": false
|
||||||
|
},
|
||||||
|
"lineInterpolation": "linear",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"pointSize": 5,
|
||||||
|
"scaleDistribution": {
|
||||||
|
"type": "linear"
|
||||||
|
},
|
||||||
|
"showPoints": "auto",
|
||||||
|
"spanNulls": false,
|
||||||
|
"stacking": {
|
||||||
|
"group": "A",
|
||||||
|
"mode": "none"
|
||||||
|
},
|
||||||
|
"thresholdsStyle": {
|
||||||
|
"mode": "off"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "red",
|
||||||
|
"value": 80
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 0
|
||||||
|
},
|
||||||
|
"id": 10,
|
||||||
|
"maxDataPoints": 1500,
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"calcs": [],
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "single"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"channel": "plugin/testdata/random-20Hz-stream-5",
|
||||||
|
"datasource": {
|
||||||
|
"type": "datasource",
|
||||||
|
"uid": "grafana"
|
||||||
|
},
|
||||||
|
"queryType": "measurements",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "5",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "grafana",
|
||||||
|
"uid": "grafana"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"custom": {
|
||||||
|
"axisLabel": "",
|
||||||
|
"axisPlacement": "auto",
|
||||||
|
"barAlignment": 0,
|
||||||
|
"drawStyle": "line",
|
||||||
|
"fillOpacity": 0,
|
||||||
|
"gradientMode": "none",
|
||||||
|
"hideFrom": {
|
||||||
|
"legend": false,
|
||||||
|
"tooltip": false,
|
||||||
|
"viz": false
|
||||||
|
},
|
||||||
|
"lineInterpolation": "linear",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"pointSize": 5,
|
||||||
|
"scaleDistribution": {
|
||||||
|
"type": "linear"
|
||||||
|
},
|
||||||
|
"showPoints": "auto",
|
||||||
|
"spanNulls": false,
|
||||||
|
"stacking": {
|
||||||
|
"group": "A",
|
||||||
|
"mode": "none"
|
||||||
|
},
|
||||||
|
"thresholdsStyle": {
|
||||||
|
"mode": "off"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "red",
|
||||||
|
"value": 80
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 0
|
||||||
|
},
|
||||||
|
"id": 6,
|
||||||
|
"maxDataPoints": 1500,
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"calcs": [],
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "single"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"channel": "plugin/testdata/random-20Hz-stream-2",
|
||||||
|
"datasource": {
|
||||||
|
"type": "datasource",
|
||||||
|
"uid": "grafana"
|
||||||
|
},
|
||||||
|
"queryType": "measurements",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "2",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "grafana",
|
||||||
|
"uid": "grafana"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"custom": {
|
||||||
|
"axisLabel": "",
|
||||||
|
"axisPlacement": "auto",
|
||||||
|
"barAlignment": 0,
|
||||||
|
"drawStyle": "line",
|
||||||
|
"fillOpacity": 0,
|
||||||
|
"gradientMode": "none",
|
||||||
|
"hideFrom": {
|
||||||
|
"legend": false,
|
||||||
|
"tooltip": false,
|
||||||
|
"viz": false
|
||||||
|
},
|
||||||
|
"lineInterpolation": "linear",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"pointSize": 5,
|
||||||
|
"scaleDistribution": {
|
||||||
|
"type": "linear"
|
||||||
|
},
|
||||||
|
"showPoints": "auto",
|
||||||
|
"spanNulls": false,
|
||||||
|
"stacking": {
|
||||||
|
"group": "A",
|
||||||
|
"mode": "none"
|
||||||
|
},
|
||||||
|
"thresholdsStyle": {
|
||||||
|
"mode": "off"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "red",
|
||||||
|
"value": 80
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 8
|
||||||
|
},
|
||||||
|
"id": 8,
|
||||||
|
"maxDataPoints": 1500,
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"calcs": [],
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "single"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"channel": "plugin/testdata/random-20Hz-stream-4",
|
||||||
|
"datasource": {
|
||||||
|
"type": "datasource",
|
||||||
|
"uid": "grafana"
|
||||||
|
},
|
||||||
|
"queryType": "measurements",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "4",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "grafana",
|
||||||
|
"uid": "grafana"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"custom": {
|
||||||
|
"axisLabel": "",
|
||||||
|
"axisPlacement": "auto",
|
||||||
|
"barAlignment": 0,
|
||||||
|
"drawStyle": "line",
|
||||||
|
"fillOpacity": 0,
|
||||||
|
"gradientMode": "none",
|
||||||
|
"hideFrom": {
|
||||||
|
"legend": false,
|
||||||
|
"tooltip": false,
|
||||||
|
"viz": false
|
||||||
|
},
|
||||||
|
"lineInterpolation": "linear",
|
||||||
|
"lineWidth": 3,
|
||||||
|
"pointSize": 5,
|
||||||
|
"scaleDistribution": {
|
||||||
|
"type": "linear"
|
||||||
|
},
|
||||||
|
"showPoints": "auto",
|
||||||
|
"spanNulls": false,
|
||||||
|
"stacking": {
|
||||||
|
"group": "A",
|
||||||
|
"mode": "none"
|
||||||
|
},
|
||||||
|
"thresholdsStyle": {
|
||||||
|
"mode": "off"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "red",
|
||||||
|
"value": 80
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 8
|
||||||
|
},
|
||||||
|
"id": 2,
|
||||||
|
"maxDataPoints": 1500,
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"calcs": [],
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "none"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"buffer": 60000,
|
||||||
|
"channel": "plugin/testdata/random-20Hz-stream",
|
||||||
|
"datasource": {
|
||||||
|
"type": "datasource",
|
||||||
|
"uid": "grafana"
|
||||||
|
},
|
||||||
|
"filter": {
|
||||||
|
"fields": []
|
||||||
|
},
|
||||||
|
"queryType": "measurements",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "1",
|
||||||
|
"transparent": true,
|
||||||
|
"type": "timeseries"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "",
|
||||||
|
"schemaVersion": 33,
|
||||||
|
"style": "dark",
|
||||||
|
"tags": [],
|
||||||
|
"templating": {
|
||||||
|
"list": []
|
||||||
|
},
|
||||||
|
"time": {
|
||||||
|
"from": "now-60s",
|
||||||
|
"to": "now"
|
||||||
|
},
|
||||||
|
"timepicker": {},
|
||||||
|
"timezone": "",
|
||||||
|
"title": "Grafana Live performance benchmarking",
|
||||||
|
"uid": "S4M5r9cnk",
|
||||||
|
"version": 9,
|
||||||
|
"weekStart": ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import { e2e } from '@grafana/e2e';
|
||||||
|
|
||||||
|
type WithGrafanaRuntime<T> = T & {
|
||||||
|
grafanaRuntime: {
|
||||||
|
livePerformance: {
|
||||||
|
start: () => void;
|
||||||
|
getStats: () => Record<string, unknown>;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const hasGrafanaRuntime = <T>(obj: T): obj is WithGrafanaRuntime<T> => {
|
||||||
|
return typeof (obj as any)?.grafanaRuntime === 'object';
|
||||||
|
};
|
||||||
|
|
||||||
|
e2e.benchmark({
|
||||||
|
name: 'Live performance benchmarking - 4x20hz panels',
|
||||||
|
dashboard: {
|
||||||
|
folder: '/dashboards/live',
|
||||||
|
delayAfterOpening: 1000,
|
||||||
|
skipPanelValidation: true,
|
||||||
|
},
|
||||||
|
repeat: 10,
|
||||||
|
duration: 120000,
|
||||||
|
appStats: {
|
||||||
|
startCollecting: (window) => {
|
||||||
|
if (!hasGrafanaRuntime(window)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
return window.grafanaRuntime.livePerformance.start();
|
||||||
|
},
|
||||||
|
collect: (window) => {
|
||||||
|
if (!hasGrafanaRuntime(window)) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
return window.grafanaRuntime.livePerformance.getStats() ?? {};
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
{
|
||||||
|
"extends": "../../tsconfig.json",
|
||||||
|
"include": ["**/*.ts", "../../packages/grafana-e2e/cypress/support/index.d.ts"],
|
||||||
|
"resolveJsonModule": true
|
||||||
|
}
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
[feature_toggles]
|
||||||
|
enable =
|
||||||
+65
-11
@@ -1,8 +1,14 @@
|
|||||||
#!/bin/bash
|
#!/usr/bin/env bash
|
||||||
set -xeo pipefail
|
set -xeo pipefail
|
||||||
|
|
||||||
. e2e/variables
|
. e2e/variables
|
||||||
|
|
||||||
|
if ((BASH_VERSINFO[0] < 4)); then
|
||||||
|
echo "Bash ver >= 4 is needed to run this script"
|
||||||
|
echo "Please upgrade your bash - run 'brew install bash' if you use Homebrew on MacOS"
|
||||||
|
exit 1;
|
||||||
|
fi
|
||||||
|
|
||||||
HOST=${HOST:-$DEFAULT_HOST}
|
HOST=${HOST:-$DEFAULT_HOST}
|
||||||
PORT=${PORT:-$DEFAULT_PORT}
|
PORT=${PORT:-$DEFAULT_PORT}
|
||||||
|
|
||||||
@@ -12,31 +18,79 @@ args=("$@")
|
|||||||
|
|
||||||
CMD="start"
|
CMD="start"
|
||||||
PARAMS=""
|
PARAMS=""
|
||||||
SLOWMO=0
|
|
||||||
URL=${BASE_URL:-"http://$HOST:$PORT"}
|
declare -A env=(
|
||||||
integrationFolder=../../e2e
|
[BASE_URL]=${BASE_URL:-"http://$HOST:$PORT"}
|
||||||
testFiles=*-suite/*spec.ts
|
[SLOWMO]=0
|
||||||
|
)
|
||||||
|
|
||||||
|
testFilesForSingleSuite="*.spec.ts"
|
||||||
|
|
||||||
|
declare -A cypressConfig=(
|
||||||
|
[integrationFolder]=../../e2e
|
||||||
|
[screenshotsFolder]=../../e2e/"${args[0]}"/screenshots
|
||||||
|
[videosFolder]=../../e2e/"${args[0]}"/videos
|
||||||
|
[fileServerFolder]=./cypress
|
||||||
|
[testFiles]=*-suite/*spec.ts
|
||||||
|
[defaultCommandTimeout]=30000
|
||||||
|
[viewportWidth]=1920
|
||||||
|
[viewportHeight]=1080
|
||||||
|
[trashAssetsBeforeRuns]=false
|
||||||
|
[videoUploadOnPasses]=false
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
cd packages/grafana-e2e
|
cd packages/grafana-e2e
|
||||||
|
|
||||||
case "$1" in
|
case "$1" in
|
||||||
"debug")
|
"debug")
|
||||||
echo -e "Debug mode"
|
echo -e "Debug mode"
|
||||||
SLOWMO=1
|
env[SLOWMO]=1
|
||||||
PARAMS="--no-exit"
|
PARAMS="--no-exit"
|
||||||
;;
|
;;
|
||||||
"dev")
|
"dev")
|
||||||
echo "Dev mode"
|
echo "Dev mode"
|
||||||
CMD="open"
|
CMD="open"
|
||||||
;;
|
;;
|
||||||
|
"benchmark")
|
||||||
|
echo "Benchmark"
|
||||||
|
PARAMS="--headed"
|
||||||
|
CMD="start-benchmark"
|
||||||
|
env[BENCHMARK_PLUGIN_ENABLED]=true
|
||||||
|
env[BENCHMARK_PLUGIN_RESULTS_FOLDER]=../../e2e/benchmarks/"${args[1]}"/results
|
||||||
|
cypressConfig[video]=false
|
||||||
|
cypressConfig[integrationFolder]=../../e2e/benchmarks/"${args[1]}"
|
||||||
|
cypressConfig[screenshotsFolder]=../../e2e/benchmarks/"${args[1]}"/screenshots
|
||||||
|
cypressConfig[testFiles]=$testFilesForSingleSuite
|
||||||
|
;;
|
||||||
"")
|
"")
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
integrationFolder=../../e2e/"${args[0]}"
|
cypressConfig[integrationFolder]=../../e2e/"${args[0]}"
|
||||||
testFiles="*.spec.ts"
|
cypressConfig[testFiles]=$testFilesForSingleSuite
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
yarn $CMD --env BASE_URL=$URL,SLOWMO=$SLOWMO \
|
function join () {
|
||||||
--config defaultCommandTimeout=30000,testFiles=$testFiles,integrationFolder=$integrationFolder,screenshotsFolder=../../e2e/"${args[0]}"/screenshots,videosFolder=../../e2e/"${args[0]}"/videos,fileServerFolder=./cypress,viewportWidth=1920,viewportHeight=1080,trashAssetsBeforeRuns=false,videoUploadOnPasses=false \
|
local -n map=$1
|
||||||
|
local delimiter=","
|
||||||
|
|
||||||
|
local res=""
|
||||||
|
|
||||||
|
for key in "${!map[@]}"
|
||||||
|
do
|
||||||
|
value=${map[$key]}
|
||||||
|
if [ -z "${res}" ]; then
|
||||||
|
res=$key=$value
|
||||||
|
else
|
||||||
|
res=$res$delimiter$key=$value
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "$res"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
yarn $CMD --env "$(join env)" \
|
||||||
|
--config "$(join cypressConfig)" \
|
||||||
$PARAMS
|
$PARAMS
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ else
|
|||||||
mkdir $PROV_DIR/datasources
|
mkdir $PROV_DIR/datasources
|
||||||
mkdir $PROV_DIR/dashboards
|
mkdir $PROV_DIR/dashboards
|
||||||
|
|
||||||
|
cp ./e2e/custom.ini $RUNDIR/conf/custom.ini
|
||||||
cp ./conf/defaults.ini $RUNDIR/conf/defaults.ini
|
cp ./conf/defaults.ini $RUNDIR/conf/defaults.ini
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
"e2e": "./e2e/start-and-run-suite",
|
"e2e": "./e2e/start-and-run-suite",
|
||||||
"e2e:debug": "./e2e/start-and-run-suite debug",
|
"e2e:debug": "./e2e/start-and-run-suite debug",
|
||||||
"e2e:dev": "./e2e/start-and-run-suite dev",
|
"e2e:dev": "./e2e/start-and-run-suite dev",
|
||||||
|
"e2e:benchmark:live": "./e2e/start-and-run-suite benchmark live",
|
||||||
"test": "jest --notify --watch",
|
"test": "jest --notify --watch",
|
||||||
"test:accessibility-report": "./scripts/generate-a11y-report.sh",
|
"test:accessibility-report": "./scripts/generate-a11y-report.sh",
|
||||||
"lint": "yarn run lint:ts && yarn run lint:sass",
|
"lint": "yarn run lint:ts && yarn run lint:sass",
|
||||||
|
|||||||
@@ -0,0 +1,136 @@
|
|||||||
|
import CDP from 'chrome-remote-interface';
|
||||||
|
import Tracelib, { TraceEvent } from 'tracelib';
|
||||||
|
|
||||||
|
import { countBy, mean } from 'lodash';
|
||||||
|
import ProtocolProxyApi from 'devtools-protocol/types/protocol-proxy-api';
|
||||||
|
import { CollectedData, DataCollector, DataCollectorName } from './DataCollector';
|
||||||
|
|
||||||
|
type CDPDataCollectorDeps = {
|
||||||
|
port: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export class CDPDataCollector implements DataCollector {
|
||||||
|
private tracingCategories: string[];
|
||||||
|
|
||||||
|
private state: {
|
||||||
|
client?: CDP.Client;
|
||||||
|
tracingPromise?: Promise<CollectedData>;
|
||||||
|
traceEvents: TraceEvent[];
|
||||||
|
};
|
||||||
|
|
||||||
|
constructor(private deps: CDPDataCollectorDeps) {
|
||||||
|
this.state = this.getDefaultState();
|
||||||
|
this.tracingCategories = [
|
||||||
|
'disabled-by-default-v8.cpu_profile',
|
||||||
|
'disabled-by-default-v8.cpu_profiler',
|
||||||
|
'disabled-by-default-v8.cpu_profiler.hires',
|
||||||
|
'disabled-by-default-devtools.timeline.frame',
|
||||||
|
'disabled-by-default-devtools.timeline',
|
||||||
|
'disabled-by-default-devtools.timeline.inputs',
|
||||||
|
'disabled-by-default-devtools.timeline.stack',
|
||||||
|
'disabled-by-default-devtools.timeline.invalidationTracking',
|
||||||
|
'disabled-by-default-layout_shift.debug',
|
||||||
|
'disabled-by-default-cc.debug.scheduler.frames',
|
||||||
|
'disabled-by-default-blink.debug.display_lock',
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
getName = () => DataCollectorName.CDP;
|
||||||
|
|
||||||
|
private resetState = async () => {
|
||||||
|
if (this.state.client) {
|
||||||
|
await this.state.client.close();
|
||||||
|
}
|
||||||
|
this.state = this.getDefaultState();
|
||||||
|
};
|
||||||
|
|
||||||
|
private getDefaultState = () => ({
|
||||||
|
traceEvents: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
// workaround for type declaration issues in cdp lib
|
||||||
|
private asApis = (
|
||||||
|
client: CDP.Client
|
||||||
|
): {
|
||||||
|
Profiler: ProtocolProxyApi.ProfilerApi;
|
||||||
|
Page: ProtocolProxyApi.PageApi;
|
||||||
|
Tracing: ProtocolProxyApi.TracingApi;
|
||||||
|
} => client;
|
||||||
|
|
||||||
|
private getClientApis = async () => this.asApis(await this.getClient());
|
||||||
|
|
||||||
|
private getClient = async () => {
|
||||||
|
if (this.state.client) {
|
||||||
|
return this.state.client;
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = await CDP({ port: this.deps.port });
|
||||||
|
|
||||||
|
const { Profiler, Page } = this.asApis(client);
|
||||||
|
await Promise.all([Page.enable(), Profiler.enable(), Profiler.setSamplingInterval({ interval: 100 })]);
|
||||||
|
|
||||||
|
this.state.client = client;
|
||||||
|
|
||||||
|
return client;
|
||||||
|
};
|
||||||
|
|
||||||
|
start: DataCollector['start'] = async ({ id }) => {
|
||||||
|
if (this.state.tracingPromise) {
|
||||||
|
throw new Error(`collection in progress - can't start another one! ${id}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const { Tracing, Profiler } = await this.getClientApis();
|
||||||
|
|
||||||
|
await Promise.all([
|
||||||
|
Tracing.start({
|
||||||
|
bufferUsageReportingInterval: 1000,
|
||||||
|
traceConfig: {
|
||||||
|
includedCategories: this.tracingCategories,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
Profiler.start(),
|
||||||
|
]);
|
||||||
|
|
||||||
|
Tracing.on('dataCollected', ({ value: events }) => {
|
||||||
|
this.state.traceEvents.push(...events);
|
||||||
|
});
|
||||||
|
|
||||||
|
let resolveFn: (data: CollectedData) => void;
|
||||||
|
this.state.tracingPromise = new Promise<CollectedData>((resolve) => {
|
||||||
|
resolveFn = resolve;
|
||||||
|
});
|
||||||
|
Tracing.on('tracingComplete', ({ dataLossOccurred }) => {
|
||||||
|
const t = new Tracelib(this.state.traceEvents);
|
||||||
|
|
||||||
|
const eventCounts = countBy(this.state.traceEvents, (ev) => ev.name);
|
||||||
|
|
||||||
|
const fps = t.getFPS();
|
||||||
|
|
||||||
|
resolveFn({
|
||||||
|
eventCounts,
|
||||||
|
fps: mean(fps.values),
|
||||||
|
tracingDataLoss: dataLossOccurred ? 1 : 0,
|
||||||
|
warnings: t.getWarningCounts(),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
stop: DataCollector['stop'] = async (req) => {
|
||||||
|
if (!this.state.tracingPromise) {
|
||||||
|
throw new Error(`collection was never started - there is nothing to stop!`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const { Tracing, Profiler } = await this.getClientApis();
|
||||||
|
|
||||||
|
// TODO: capture profiler data
|
||||||
|
const [, , traceData] = await Promise.all([Profiler.stop(), Tracing.end(), this.state.tracingPromise]);
|
||||||
|
|
||||||
|
await this.resetState();
|
||||||
|
|
||||||
|
return traceData;
|
||||||
|
};
|
||||||
|
|
||||||
|
close: DataCollector['close'] = async () => {
|
||||||
|
await this.resetState();
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
export type CollectedData = Record<string, unknown>;
|
||||||
|
|
||||||
|
export enum DataCollectorName {
|
||||||
|
CDP = 'CDP',
|
||||||
|
}
|
||||||
|
|
||||||
|
type DataCollectorRequest = { id: string };
|
||||||
|
|
||||||
|
export type DataCollector<T extends CollectedData = CollectedData> = {
|
||||||
|
start: (input: DataCollectorRequest) => Promise<void>;
|
||||||
|
stop: (input: DataCollectorRequest) => Promise<T>;
|
||||||
|
getName: () => DataCollectorName;
|
||||||
|
close: () => Promise<void>;
|
||||||
|
};
|
||||||
@@ -0,0 +1,138 @@
|
|||||||
|
import { CollectedData, DataCollectorName } from './DataCollector';
|
||||||
|
import { fromPairs } from 'lodash';
|
||||||
|
|
||||||
|
type Stats = {
|
||||||
|
sum: number;
|
||||||
|
min: number;
|
||||||
|
max: number;
|
||||||
|
count: number;
|
||||||
|
avg: number;
|
||||||
|
time: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export enum MeasurementName {
|
||||||
|
DataRenderDelay = 'DataRenderDelay',
|
||||||
|
}
|
||||||
|
|
||||||
|
type LivePerformanceAppStats = Record<MeasurementName, Stats[]>;
|
||||||
|
|
||||||
|
const isLivePerformanceAppStats = (data: CollectedData[]): data is LivePerformanceAppStats[] =>
|
||||||
|
data.some((st) => {
|
||||||
|
const stat = st?.[MeasurementName.DataRenderDelay];
|
||||||
|
return Array.isArray(stat) && Boolean(stat?.length);
|
||||||
|
});
|
||||||
|
|
||||||
|
type FormattedStats = {
|
||||||
|
total: {
|
||||||
|
count: number[];
|
||||||
|
avg: number[];
|
||||||
|
};
|
||||||
|
lastInterval: {
|
||||||
|
avg: number[];
|
||||||
|
min: number[];
|
||||||
|
max: number[];
|
||||||
|
count: number[];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const formatAppStats = (allStats: CollectedData[]) => {
|
||||||
|
if (!isLivePerformanceAppStats(allStats)) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
const names = Object.keys(MeasurementName) as MeasurementName[];
|
||||||
|
|
||||||
|
return fromPairs(
|
||||||
|
names.map((name) => {
|
||||||
|
const statsForMeasurement = allStats.map((s) => s[name]);
|
||||||
|
const res: FormattedStats = {
|
||||||
|
total: {
|
||||||
|
count: [],
|
||||||
|
avg: [],
|
||||||
|
},
|
||||||
|
lastInterval: {
|
||||||
|
avg: [],
|
||||||
|
min: [],
|
||||||
|
max: [],
|
||||||
|
count: [],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
statsForMeasurement.forEach((s) => {
|
||||||
|
const total = s.reduce(
|
||||||
|
(prev, next) => {
|
||||||
|
prev.count += next.count;
|
||||||
|
prev.avg += next.avg;
|
||||||
|
return prev;
|
||||||
|
},
|
||||||
|
{ count: 0, avg: 0 }
|
||||||
|
);
|
||||||
|
res.total.count.push(Math.round(total.count));
|
||||||
|
res.total.avg.push(Math.round(total.avg / s.length));
|
||||||
|
|
||||||
|
const lastInterval = s[s.length - 1];
|
||||||
|
|
||||||
|
res.lastInterval.avg.push(Math.round(lastInterval?.avg));
|
||||||
|
res.lastInterval.min.push(Math.round(lastInterval?.min));
|
||||||
|
res.lastInterval.max.push(Math.round(lastInterval?.max));
|
||||||
|
res.lastInterval.count.push(Math.round(lastInterval?.count));
|
||||||
|
});
|
||||||
|
|
||||||
|
return [name, res];
|
||||||
|
})
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
type CDPData = {
|
||||||
|
eventCounts: Record<string, unknown>;
|
||||||
|
fps: number;
|
||||||
|
tracingDataLoss: number;
|
||||||
|
warnings: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
const isCDPData = (data: any[]): data is CDPData[] => data.every((d) => typeof d.eventCounts === 'object');
|
||||||
|
|
||||||
|
type FormattedCDPData = {
|
||||||
|
minorGC: number[];
|
||||||
|
majorGC: number[];
|
||||||
|
droppedFrames: number[];
|
||||||
|
fps: number[];
|
||||||
|
tracingDataLossOccurred: boolean;
|
||||||
|
longTaskWarnings: number[];
|
||||||
|
};
|
||||||
|
|
||||||
|
const emptyFormattedCDPData = (): FormattedCDPData => ({
|
||||||
|
minorGC: [],
|
||||||
|
majorGC: [],
|
||||||
|
droppedFrames: [],
|
||||||
|
fps: [],
|
||||||
|
tracingDataLossOccurred: false,
|
||||||
|
longTaskWarnings: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
const formatCDPData = (data: any): FormattedCDPData => {
|
||||||
|
if (!isCDPData(data)) {
|
||||||
|
return emptyFormattedCDPData();
|
||||||
|
}
|
||||||
|
|
||||||
|
return data.reduce((acc, next) => {
|
||||||
|
acc.majorGC.push((next.eventCounts.MajorGC as number) ?? 0);
|
||||||
|
acc.minorGC.push((next.eventCounts.MinorGC as number) ?? 0);
|
||||||
|
acc.fps.push(Math.round(next.fps) ?? 0);
|
||||||
|
acc.tracingDataLossOccurred = acc.tracingDataLossOccurred || Boolean(next.tracingDataLoss);
|
||||||
|
acc.droppedFrames.push((next.eventCounts.DroppedFrame as number) ?? 0);
|
||||||
|
acc.longTaskWarnings.push((next.warnings.LongTask as number) ?? 0);
|
||||||
|
return acc;
|
||||||
|
}, emptyFormattedCDPData());
|
||||||
|
};
|
||||||
|
|
||||||
|
export const formatResults = (
|
||||||
|
results: Array<{ appStats: CollectedData; collectorsData: CollectedData }>
|
||||||
|
): CollectedData => {
|
||||||
|
return {
|
||||||
|
...formatAppStats(results.map(({ appStats }) => appStats)),
|
||||||
|
...formatCDPData(results.map(({ collectorsData }) => collectorsData[DataCollectorName.CDP])),
|
||||||
|
|
||||||
|
__raw: results,
|
||||||
|
};
|
||||||
|
};
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
import { CollectedData, DataCollector } from './DataCollector';
|
||||||
|
import { CDPDataCollector } from './CDPDataCollector';
|
||||||
|
import { fromPairs } from 'lodash';
|
||||||
|
import fs from 'fs';
|
||||||
|
import { formatResults } from './formatting';
|
||||||
|
const remoteDebuggingPortOptionPrefix = '--remote-debugging-port=';
|
||||||
|
|
||||||
|
const getOrAddRemoteDebuggingPort = (args: string[]) => {
|
||||||
|
const existing = args.find((arg) => arg.startsWith(remoteDebuggingPortOptionPrefix));
|
||||||
|
|
||||||
|
if (existing) {
|
||||||
|
return Number(existing.substring(remoteDebuggingPortOptionPrefix.length));
|
||||||
|
}
|
||||||
|
|
||||||
|
const port = 40000 + Math.round(Math.random() * 25000);
|
||||||
|
args.push(`${remoteDebuggingPortOptionPrefix}${port}`);
|
||||||
|
return port;
|
||||||
|
};
|
||||||
|
|
||||||
|
let collectors: DataCollector[] = [];
|
||||||
|
let results: Array<{ appStats: CollectedData; collectorsData: CollectedData }> = [];
|
||||||
|
|
||||||
|
const startBenchmarking = async ({ testName }: { testName: string }) => {
|
||||||
|
await Promise.all(collectors.map((coll) => coll.start({ id: testName })));
|
||||||
|
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
|
const stopBenchmarking = async ({ testName, appStats }: { testName: string; appStats: CollectedData }) => {
|
||||||
|
const data = await Promise.all(collectors.map(async (coll) => [coll.getName(), await coll.stop({ id: testName })]));
|
||||||
|
|
||||||
|
results.push({
|
||||||
|
collectorsData: fromPairs(data),
|
||||||
|
appStats: appStats,
|
||||||
|
});
|
||||||
|
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
const afterRun = async () => {
|
||||||
|
await Promise.all(collectors.map((coll) => coll.close()));
|
||||||
|
collectors = [];
|
||||||
|
results = [];
|
||||||
|
};
|
||||||
|
|
||||||
|
const afterSpec = (resultsFolder: string) => async (spec: { name: string }) => {
|
||||||
|
fs.writeFileSync(`${resultsFolder}/${spec.name}-${Date.now()}.json`, JSON.stringify(formatResults(results), null, 2));
|
||||||
|
|
||||||
|
results = [];
|
||||||
|
};
|
||||||
|
|
||||||
|
export const initialize: Cypress.PluginConfig = (on, config) => {
|
||||||
|
const resultsFolder = config.env['BENCHMARK_PLUGIN_RESULTS_FOLDER'];
|
||||||
|
|
||||||
|
if (!fs.existsSync(resultsFolder)) {
|
||||||
|
fs.mkdirSync(resultsFolder, { recursive: true });
|
||||||
|
console.log(`Created folder for benchmark results ${resultsFolder}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
on('before:browser:launch', async (browser, options) => {
|
||||||
|
if (browser.family !== 'chromium' || browser.name === 'electron') {
|
||||||
|
throw new Error('benchmarking plugin requires chrome');
|
||||||
|
}
|
||||||
|
|
||||||
|
const { args } = options;
|
||||||
|
|
||||||
|
const port = getOrAddRemoteDebuggingPort(args);
|
||||||
|
collectors.push(new CDPDataCollector({ port }));
|
||||||
|
|
||||||
|
args.push('--start-fullscreen');
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`initialized benchmarking plugin with ${collectors.length} collectors: ${collectors
|
||||||
|
.map((col) => col.getName())
|
||||||
|
.join(', ')}`
|
||||||
|
);
|
||||||
|
|
||||||
|
return options;
|
||||||
|
});
|
||||||
|
|
||||||
|
on('task', {
|
||||||
|
startBenchmarking,
|
||||||
|
stopBenchmarking,
|
||||||
|
});
|
||||||
|
|
||||||
|
on('after:run', afterRun);
|
||||||
|
on('after:spec', afterSpec(resultsFolder));
|
||||||
|
};
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
type TraceEvent = {
|
||||||
|
name: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
declare class Tracelib {
|
||||||
|
constructor(private events: TraceEvent[]) {}
|
||||||
|
|
||||||
|
getFPS: () => { times: number[]; values: number[] };
|
||||||
|
getWarningCounts: () => Record<string, number>;
|
||||||
|
}
|
||||||
|
declare module 'tracelib' {
|
||||||
|
export = Tracelib;
|
||||||
|
|
||||||
|
export { TraceEvent };
|
||||||
|
}
|
||||||
@@ -5,8 +5,13 @@ const compareScreenshots = require('./compareScreenshots');
|
|||||||
const extendConfig = require('./extendConfig');
|
const extendConfig = require('./extendConfig');
|
||||||
const readProvisions = require('./readProvisions');
|
const readProvisions = require('./readProvisions');
|
||||||
const typescriptPreprocessor = require('./typescriptPreprocessor');
|
const typescriptPreprocessor = require('./typescriptPreprocessor');
|
||||||
|
const benchmarkPlugin = require('./benchmark');
|
||||||
|
|
||||||
module.exports = (on, config) => {
|
module.exports = (on, config) => {
|
||||||
|
if (config.env['BENCHMARK_PLUGIN_ENABLED'] === true) {
|
||||||
|
benchmarkPlugin.initialize(on, config);
|
||||||
|
}
|
||||||
|
|
||||||
on('file:preprocessor', typescriptPreprocessor);
|
on('file:preprocessor', typescriptPreprocessor);
|
||||||
on('task', { compareScreenshots, readProvisions });
|
on('task', { compareScreenshots, readProvisions });
|
||||||
on('task', {
|
on('task', {
|
||||||
|
|||||||
@@ -31,3 +31,11 @@ Cypress.Commands.add('getJSONFilesFromDir', (dirPath: string) => {
|
|||||||
relativePath: dirPath,
|
relativePath: dirPath,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
Cypress.Commands.add('startBenchmarking', (testName: string) => {
|
||||||
|
return cy.task('startBenchmarking', { testName });
|
||||||
|
});
|
||||||
|
|
||||||
|
Cypress.Commands.add('stopBenchmarking', (testName: string, appStats: Record<string, unknown>) => {
|
||||||
|
return cy.task('stopBenchmarking', { testName, appStats });
|
||||||
|
});
|
||||||
|
|||||||
@@ -6,5 +6,7 @@ declare namespace Cypress {
|
|||||||
logToConsole(message: string, optional?: any): void;
|
logToConsole(message: string, optional?: any): void;
|
||||||
readProvisions(filePaths: string[]): Chainable;
|
readProvisions(filePaths: string[]): Chainable;
|
||||||
getJSONFilesFromDir(dirPath: string): Chainable;
|
getJSONFilesFromDir(dirPath: string): Chainable;
|
||||||
|
startBenchmarking(testName: string): void;
|
||||||
|
stopBenchmarking(testName: string, appStats: Record<string, unknown>): void;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"declaration": false,
|
"declaration": false,
|
||||||
"module": "commonjs",
|
"module": "commonjs",
|
||||||
"types": ["cypress", "cypress-file-upload"]
|
"types": ["cypress", "cypress-file-upload", "node"]
|
||||||
},
|
},
|
||||||
"extends": "@grafana/tsconfig",
|
"extends": "@grafana/tsconfig",
|
||||||
"include": ["**/*.ts"]
|
"include": ["**/*.ts"]
|
||||||
|
|||||||
@@ -26,12 +26,15 @@
|
|||||||
"docsExtract": "mkdir -p ../../reports/docs && api-extractor run 2>&1 | tee ../../reports/docs/$(basename $(pwd)).log",
|
"docsExtract": "mkdir -p ../../reports/docs && api-extractor run 2>&1 | tee ../../reports/docs/$(basename $(pwd)).log",
|
||||||
"open": "cypress open",
|
"open": "cypress open",
|
||||||
"start": "cypress run --browser=chrome",
|
"start": "cypress run --browser=chrome",
|
||||||
|
"start-benchmark": "CYPRESS_NO_COMMAND_LOG=1 yarn start",
|
||||||
"test": "pushd test && node ../dist/bin/grafana-e2e.js run",
|
"test": "pushd test && node ../dist/bin/grafana-e2e.js run",
|
||||||
"typecheck": "tsc --noEmit"
|
"typecheck": "tsc --noEmit"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@rollup/plugin-commonjs": "21.0.1",
|
"@rollup/plugin-commonjs": "21.0.1",
|
||||||
"@rollup/plugin-node-resolve": "13.1.3",
|
"@rollup/plugin-node-resolve": "13.1.3",
|
||||||
|
"@types/chrome-remote-interface": "0.31.4",
|
||||||
|
"@types/lodash": "4.14.149",
|
||||||
"@types/node": "16.11.19",
|
"@types/node": "16.11.19",
|
||||||
"@types/uuid": "8.3.4",
|
"@types/uuid": "8.3.4",
|
||||||
"rollup": "2.63.0",
|
"rollup": "2.63.0",
|
||||||
@@ -50,13 +53,17 @@
|
|||||||
"@mochajs/json-file-reporter": "^1.2.0",
|
"@mochajs/json-file-reporter": "^1.2.0",
|
||||||
"babel-loader": "8.2.3",
|
"babel-loader": "8.2.3",
|
||||||
"blink-diff": "1.0.13",
|
"blink-diff": "1.0.13",
|
||||||
|
"chrome-remote-interface": "0.31.1",
|
||||||
"commander": "8.3.0",
|
"commander": "8.3.0",
|
||||||
"cypress": "9.2.0",
|
"cypress": "9.2.0",
|
||||||
"cypress-file-upload": "5.0.8",
|
"cypress-file-upload": "5.0.8",
|
||||||
|
"devtools-protocol": "0.0.927104",
|
||||||
"execa": "5.1.1",
|
"execa": "5.1.1",
|
||||||
|
"lodash": "4.17.21",
|
||||||
"mocha": "9.1.3",
|
"mocha": "9.1.3",
|
||||||
"resolve-as-bin": "2.1.0",
|
"resolve-as-bin": "2.1.0",
|
||||||
"rimraf": "3.0.2",
|
"rimraf": "3.0.2",
|
||||||
|
"tracelib": "1.0.1",
|
||||||
"ts-loader": "6.2.1",
|
"ts-loader": "6.2.1",
|
||||||
"tslib": "2.3.1",
|
"tslib": "2.3.1",
|
||||||
"typescript": "4.5.4",
|
"typescript": "4.5.4",
|
||||||
|
|||||||
@@ -13,8 +13,9 @@ export type Dashboard = { title: string; panels: Panel[]; uid: string; [key: str
|
|||||||
* Smoke test a particular dashboard by quickly importing a json file and validate that all the panels finish loading
|
* Smoke test a particular dashboard by quickly importing a json file and validate that all the panels finish loading
|
||||||
* @param dashboardToImport a sample dashboard
|
* @param dashboardToImport a sample dashboard
|
||||||
* @param queryTimeout a number of ms to wait for the imported dashboard to finish loading
|
* @param queryTimeout a number of ms to wait for the imported dashboard to finish loading
|
||||||
|
* @param skipPanelValidation skip panel validation
|
||||||
*/
|
*/
|
||||||
export const importDashboard = (dashboardToImport: Dashboard, queryTimeout?: number) => {
|
export const importDashboard = (dashboardToImport: Dashboard, queryTimeout?: number, skipPanelValidation?: boolean) => {
|
||||||
e2e().visit(fromBaseUrl('/dashboard/import'));
|
e2e().visit(fromBaseUrl('/dashboard/import'));
|
||||||
|
|
||||||
// Note: normally we'd use 'click' and then 'type' here, but the json object is so big that using 'val' is much faster
|
// Note: normally we'd use 'click' and then 'type' here, but the json object is so big that using 'val' is much faster
|
||||||
@@ -45,21 +46,25 @@ export const importDashboard = (dashboardToImport: Dashboard, queryTimeout?: num
|
|||||||
expect(dashboardToImport.uid).to.equal(uid);
|
expect(dashboardToImport.uid).to.equal(uid);
|
||||||
});
|
});
|
||||||
|
|
||||||
dashboardToImport.panels.forEach((panel) => {
|
if (!skipPanelValidation) {
|
||||||
// Look at the json data
|
dashboardToImport.panels.forEach((panel) => {
|
||||||
e2e.components.Panels.Panel.title(panel.title).should('be.visible').click();
|
// Look at the json data
|
||||||
e2e.components.Panels.Panel.headerItems('Inspect').should('be.visible').click();
|
e2e.components.Panels.Panel.title(panel.title).should('be.visible').click();
|
||||||
e2e.components.Tab.title('JSON').should('be.visible').click();
|
e2e.components.Panels.Panel.headerItems('Inspect').should('be.visible').click();
|
||||||
e2e.components.PanelInspector.Json.content().should('be.visible').contains('Panel JSON').click({ force: true });
|
e2e.components.Tab.title('JSON').should('be.visible').click();
|
||||||
e2e.components.Select.option().should('be.visible').contains('Data').click();
|
e2e.components.PanelInspector.Json.content().should('be.visible').contains('Panel JSON').click({ force: true });
|
||||||
|
e2e.components.Select.option().should('be.visible').contains('Data').click();
|
||||||
|
|
||||||
// ensures that panel has loaded without knowingly hitting an error
|
// ensures that panel has loaded without knowingly hitting an error
|
||||||
// note: this does not prove that data came back as we expected it,
|
// note: this does not prove that data came back as we expected it,
|
||||||
// it could get `state: Done` for no data for example
|
// it could get `state: Done` for no data for example
|
||||||
// but it ensures we didn't hit a 401 or 500 or something like that
|
// but it ensures we didn't hit a 401 or 500 or something like that
|
||||||
e2e.components.CodeEditor.container().should('be.visible').contains('"state": "Done"');
|
e2e.components.CodeEditor.container()
|
||||||
|
.should('be.visible')
|
||||||
|
.contains(/"state": "(Done|Streaming)"/);
|
||||||
|
|
||||||
// need to close panel
|
// need to close panel
|
||||||
e2e.components.Drawer.General.close().click();
|
e2e.components.Drawer.General.close().click();
|
||||||
});
|
});
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -7,13 +7,14 @@ import { e2e } from '../index';
|
|||||||
* @param dirPath the relative path to a directory which contains json files representing dashboards,
|
* @param dirPath the relative path to a directory which contains json files representing dashboards,
|
||||||
* for example if your dashboards live in `cypress/testDashboards` you can pass `/testDashboards`
|
* for example if your dashboards live in `cypress/testDashboards` you can pass `/testDashboards`
|
||||||
* @param queryTimeout a number of ms to wait for the imported dashboard to finish loading
|
* @param queryTimeout a number of ms to wait for the imported dashboard to finish loading
|
||||||
|
* @param skipPanelValidation skips panel validation
|
||||||
*/
|
*/
|
||||||
export const importDashboards = async (dirPath: string, queryTimeout?: number) => {
|
export const importDashboards = async (dirPath: string, queryTimeout?: number, skipPanelValidation?: boolean) => {
|
||||||
e2e()
|
e2e()
|
||||||
.getJSONFilesFromDir(dirPath)
|
.getJSONFilesFromDir(dirPath)
|
||||||
.then((jsonFiles: Dashboard[]) => {
|
.then((jsonFiles: Dashboard[]) => {
|
||||||
jsonFiles.forEach((file) => {
|
jsonFiles.forEach((file) => {
|
||||||
importDashboard(file, queryTimeout || 6000);
|
importDashboard(file, queryTimeout || 6000, skipPanelValidation);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
* @packageDocumentation
|
* @packageDocumentation
|
||||||
*/
|
*/
|
||||||
import { e2eScenario, ScenarioArguments } from './support/scenario';
|
import { e2eScenario, ScenarioArguments } from './support/scenario';
|
||||||
|
import { benchmark } from './support/benchmark';
|
||||||
import { getScenarioContext, setScenarioContext } from './support/scenarioContext';
|
import { getScenarioContext, setScenarioContext } from './support/scenarioContext';
|
||||||
import { e2eFactory } from './support';
|
import { e2eFactory } from './support';
|
||||||
import { E2ESelectors, Selectors, selectors } from '@grafana/e2e-selectors';
|
import { E2ESelectors, Selectors, selectors } from '@grafana/e2e-selectors';
|
||||||
@@ -16,6 +17,7 @@ const e2eObject = {
|
|||||||
blobToBase64String: (blob: any) => Cypress.Blob.blobToBase64String(blob),
|
blobToBase64String: (blob: any) => Cypress.Blob.blobToBase64String(blob),
|
||||||
imgSrcToBlob: (url: string) => Cypress.Blob.imgSrcToBlob(url),
|
imgSrcToBlob: (url: string) => Cypress.Blob.imgSrcToBlob(url),
|
||||||
scenario: (args: ScenarioArguments) => e2eScenario(args),
|
scenario: (args: ScenarioArguments) => e2eScenario(args),
|
||||||
|
benchmark,
|
||||||
pages: e2eFactory({ selectors: selectors.pages }),
|
pages: e2eFactory({ selectors: selectors.pages }),
|
||||||
typings,
|
typings,
|
||||||
components: e2eFactory({ selectors: selectors.components }),
|
components: e2eFactory({ selectors: selectors.components }),
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
import { e2e } from '../';
|
||||||
|
|
||||||
|
export interface BenchmarkArguments {
|
||||||
|
name: string;
|
||||||
|
dashboard: {
|
||||||
|
folder: string;
|
||||||
|
delayAfterOpening: number;
|
||||||
|
skipPanelValidation: boolean;
|
||||||
|
};
|
||||||
|
repeat: number;
|
||||||
|
duration: number;
|
||||||
|
appStats?: {
|
||||||
|
startCollecting?: (window: Window) => void;
|
||||||
|
collect: (window: Window) => Record<string, unknown>;
|
||||||
|
};
|
||||||
|
skipScenario?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const benchmark = ({
|
||||||
|
name,
|
||||||
|
skipScenario = false,
|
||||||
|
repeat,
|
||||||
|
duration,
|
||||||
|
appStats,
|
||||||
|
dashboard,
|
||||||
|
}: BenchmarkArguments) => {
|
||||||
|
if (skipScenario) {
|
||||||
|
describe(name, () => {
|
||||||
|
it.skip(name, () => {});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe(name, () => {
|
||||||
|
before(() => {
|
||||||
|
e2e.flows.login(e2e.env('USERNAME'), e2e.env('PASSWORD'));
|
||||||
|
});
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
e2e.flows.importDashboards(dashboard.folder, 1000, dashboard.skipPanelValidation);
|
||||||
|
Cypress.Cookies.preserveOnce('grafana_session');
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => e2e.flows.revertAllChanges());
|
||||||
|
after(() => {
|
||||||
|
e2e().clearCookies();
|
||||||
|
});
|
||||||
|
|
||||||
|
Array(repeat)
|
||||||
|
.fill(0)
|
||||||
|
.map((_, i) => {
|
||||||
|
const testName = `${name}-${i}`;
|
||||||
|
return it(testName, () => {
|
||||||
|
e2e.flows.openDashboard();
|
||||||
|
|
||||||
|
e2e().wait(dashboard.delayAfterOpening);
|
||||||
|
|
||||||
|
if (appStats) {
|
||||||
|
const startCollecting = appStats.startCollecting;
|
||||||
|
if (startCollecting) {
|
||||||
|
e2e()
|
||||||
|
.window()
|
||||||
|
.then((win) => startCollecting(win));
|
||||||
|
}
|
||||||
|
|
||||||
|
e2e().startBenchmarking(testName);
|
||||||
|
e2e().wait(duration);
|
||||||
|
|
||||||
|
e2e()
|
||||||
|
.window()
|
||||||
|
.then((win) => {
|
||||||
|
e2e().stopBenchmarking(testName, appStats.collect(win));
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
e2e().startBenchmarking(testName);
|
||||||
|
e2e().wait(duration);
|
||||||
|
e2e().stopBenchmarking(testName, {});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
};
|
||||||
@@ -3782,14 +3782,19 @@ __metadata:
|
|||||||
"@mochajs/json-file-reporter": ^1.2.0
|
"@mochajs/json-file-reporter": ^1.2.0
|
||||||
"@rollup/plugin-commonjs": 21.0.1
|
"@rollup/plugin-commonjs": 21.0.1
|
||||||
"@rollup/plugin-node-resolve": 13.1.3
|
"@rollup/plugin-node-resolve": 13.1.3
|
||||||
|
"@types/chrome-remote-interface": 0.31.4
|
||||||
|
"@types/lodash": 4.14.149
|
||||||
"@types/node": 16.11.19
|
"@types/node": 16.11.19
|
||||||
"@types/uuid": 8.3.4
|
"@types/uuid": 8.3.4
|
||||||
babel-loader: 8.2.3
|
babel-loader: 8.2.3
|
||||||
blink-diff: 1.0.13
|
blink-diff: 1.0.13
|
||||||
|
chrome-remote-interface: 0.31.1
|
||||||
commander: 8.3.0
|
commander: 8.3.0
|
||||||
cypress: 9.2.0
|
cypress: 9.2.0
|
||||||
cypress-file-upload: 5.0.8
|
cypress-file-upload: 5.0.8
|
||||||
|
devtools-protocol: 0.0.927104
|
||||||
execa: 5.1.1
|
execa: 5.1.1
|
||||||
|
lodash: 4.17.21
|
||||||
mocha: 9.1.3
|
mocha: 9.1.3
|
||||||
resolve-as-bin: 2.1.0
|
resolve-as-bin: 2.1.0
|
||||||
rimraf: 3.0.2
|
rimraf: 3.0.2
|
||||||
@@ -3797,6 +3802,7 @@ __metadata:
|
|||||||
rollup-plugin-copy: 3.4.0
|
rollup-plugin-copy: 3.4.0
|
||||||
rollup-plugin-sourcemaps: 0.6.3
|
rollup-plugin-sourcemaps: 0.6.3
|
||||||
rollup-plugin-terser: 7.0.2
|
rollup-plugin-terser: 7.0.2
|
||||||
|
tracelib: 1.0.1
|
||||||
ts-loader: 6.2.1
|
ts-loader: 6.2.1
|
||||||
tslib: 2.3.1
|
tslib: 2.3.1
|
||||||
typescript: 4.5.4
|
typescript: 4.5.4
|
||||||
@@ -8946,6 +8952,15 @@ __metadata:
|
|||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
|
"@types/chrome-remote-interface@npm:0.31.4":
|
||||||
|
version: 0.31.4
|
||||||
|
resolution: "@types/chrome-remote-interface@npm:0.31.4"
|
||||||
|
dependencies:
|
||||||
|
devtools-protocol: 0.0.927104
|
||||||
|
checksum: 91c6cf9c749adedc08458b772db12f4142172f36fe885dc741921d259bdb7ec47d64b9e294d793a6c36f0435c6855d6470754fd143711e0b2ff204878dd4f721
|
||||||
|
languageName: node
|
||||||
|
linkType: hard
|
||||||
|
|
||||||
"@types/classnames@npm:2.3.0, @types/classnames@npm:^2.2.7":
|
"@types/classnames@npm:2.3.0, @types/classnames@npm:^2.2.7":
|
||||||
version: 2.3.0
|
version: 2.3.0
|
||||||
resolution: "@types/classnames@npm:2.3.0"
|
resolution: "@types/classnames@npm:2.3.0"
|
||||||
@@ -14078,6 +14093,18 @@ __metadata:
|
|||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
|
"chrome-remote-interface@npm:0.31.1":
|
||||||
|
version: 0.31.1
|
||||||
|
resolution: "chrome-remote-interface@npm:0.31.1"
|
||||||
|
dependencies:
|
||||||
|
commander: 2.11.x
|
||||||
|
ws: ^7.2.0
|
||||||
|
bin:
|
||||||
|
chrome-remote-interface: bin/client.js
|
||||||
|
checksum: fbc7776abaa58b1d9b517ce93c479c0cb5da287df4a97908675f77cc261c145a772d2ba19d4d23845b7800eae1354008278b0a4e410ff419f641a3f43bbcef15
|
||||||
|
languageName: node
|
||||||
|
linkType: hard
|
||||||
|
|
||||||
"chrome-trace-event@npm:^1.0.2":
|
"chrome-trace-event@npm:^1.0.2":
|
||||||
version: 1.0.3
|
version: 1.0.3
|
||||||
resolution: "chrome-trace-event@npm:1.0.3"
|
resolution: "chrome-trace-event@npm:1.0.3"
|
||||||
@@ -14510,6 +14537,13 @@ __metadata:
|
|||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
|
"commander@npm:2.11.x":
|
||||||
|
version: 2.11.0
|
||||||
|
resolution: "commander@npm:2.11.0"
|
||||||
|
checksum: 0d0c622d129a801699b9bbf6fa518108c7e221e51ae12457119aec52f1142ab759b6cd3348ee253604e934639e200c8f0e1cf8342a2ba4b28b3565a7322ead14
|
||||||
|
languageName: node
|
||||||
|
linkType: hard
|
||||||
|
|
||||||
"commander@npm:2.17.x":
|
"commander@npm:2.17.x":
|
||||||
version: 2.17.1
|
version: 2.17.1
|
||||||
resolution: "commander@npm:2.17.1"
|
resolution: "commander@npm:2.17.1"
|
||||||
@@ -16642,6 +16676,13 @@ __metadata:
|
|||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
|
"devtools-protocol@npm:0.0.927104":
|
||||||
|
version: 0.0.927104
|
||||||
|
resolution: "devtools-protocol@npm:0.0.927104"
|
||||||
|
checksum: 13617e735f326b9822e64480060e59068434e937a6141ffe18353d0c7626be890e94d3118e7c262e6fd17eea3a4a2a18c221d0ff1eb31768ebcf0a0b95475d32
|
||||||
|
languageName: node
|
||||||
|
linkType: hard
|
||||||
|
|
||||||
"dezalgo@npm:^1.0.0":
|
"dezalgo@npm:^1.0.0":
|
||||||
version: 1.0.3
|
version: 1.0.3
|
||||||
resolution: "dezalgo@npm:1.0.3"
|
resolution: "dezalgo@npm:1.0.3"
|
||||||
@@ -34063,6 +34104,13 @@ __metadata:
|
|||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
|
"tracelib@npm:1.0.1":
|
||||||
|
version: 1.0.1
|
||||||
|
resolution: "tracelib@npm:1.0.1"
|
||||||
|
checksum: b4b7899491b6a2279e297d3fffca60ae330992a18af5a9cc85436159b8e313f42bc54794d61f504619b271531feea74bcd5b99950a1bed3a51378c23c3fa2e4a
|
||||||
|
languageName: node
|
||||||
|
linkType: hard
|
||||||
|
|
||||||
"traverse@npm:^0.6.6":
|
"traverse@npm:^0.6.6":
|
||||||
version: 0.6.6
|
version: 0.6.6
|
||||||
resolution: "traverse@npm:0.6.6"
|
resolution: "traverse@npm:0.6.6"
|
||||||
@@ -36231,6 +36279,21 @@ __metadata:
|
|||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
|
"ws@npm:^7.2.0":
|
||||||
|
version: 7.5.6
|
||||||
|
resolution: "ws@npm:7.5.6"
|
||||||
|
peerDependencies:
|
||||||
|
bufferutil: ^4.0.1
|
||||||
|
utf-8-validate: ^5.0.2
|
||||||
|
peerDependenciesMeta:
|
||||||
|
bufferutil:
|
||||||
|
optional: true
|
||||||
|
utf-8-validate:
|
||||||
|
optional: true
|
||||||
|
checksum: 0c2ffc9a539dd61dd2b00ff6cc5c98a3371e2521011fe23da4b3578bb7ac26cbdf7ca8a68e8e08023c122ae247013216dde2a20c908de415a6bcc87bdef68c87
|
||||||
|
languageName: node
|
||||||
|
linkType: hard
|
||||||
|
|
||||||
"ws@npm:^8.1.0":
|
"ws@npm:^8.1.0":
|
||||||
version: 8.2.3
|
version: 8.2.3
|
||||||
resolution: "ws@npm:8.2.3"
|
resolution: "ws@npm:8.2.3"
|
||||||
|
|||||||
Reference in New Issue
Block a user