Chore: Move to Cypress 12 and decouple cypress from @grafana/e2e (#74084)
* update drone to use cypress 12 image * upgrade cypress to 12 in core * cypress config actually valid * update @grafana/e2e imports and add lint rule * ignore grafana-e2e from betterer now it's deprecated * fix remaining type errors * fix failing tests * remove unnecessary tsconfig * remove unnecessary comment * update enterprise suite commands to work * add cypress config to CODEOWNERS * export setTimeRange in utils * remove @grafana/e2e from core deps * try running the command through yarn * move CMD to scripts * Update cloud-data-sources e2e image * Update paths --------- Co-authored-by: Andreas Christou <andreas.christou@grafana.com>
This commit is contained in:
Ashley Harrison
co-authored by
Andreas Christou
Andreas Christou
parent
e7a2c95586
commit
0f2f25c5d9
@@ -0,0 +1,120 @@
|
||||
const CDP = require('chrome-remote-interface');
|
||||
const { countBy, mean } = require('lodash');
|
||||
const Tracelib = require('tracelib');
|
||||
|
||||
class CDPDataCollector {
|
||||
tracingCategories;
|
||||
state;
|
||||
|
||||
constructor(deps) {
|
||||
this.state = this.getDefaultState();
|
||||
this.tracingCategories = [
|
||||
'disabled-by-default-v8.cpu_profile',
|
||||
'disabled-by-default-v8.cpu_profiler',
|
||||
'disabled-by-default-v8.cpu_profiler.hires',
|
||||
'disabled-by-default-devtools.timeline.frame',
|
||||
'disabled-by-default-devtools.timeline',
|
||||
'disabled-by-default-devtools.timeline.inputs',
|
||||
'disabled-by-default-devtools.timeline.stack',
|
||||
'disabled-by-default-devtools.timeline.invalidationTracking',
|
||||
'disabled-by-default-layout_shift.debug',
|
||||
'disabled-by-default-cc.debug.scheduler.frames',
|
||||
'disabled-by-default-blink.debug.display_lock',
|
||||
];
|
||||
}
|
||||
|
||||
getName = () => DataCollectorName.CDP;
|
||||
|
||||
resetState = async () => {
|
||||
if (this.state.client) {
|
||||
await this.state.client.close();
|
||||
}
|
||||
this.state = this.getDefaultState();
|
||||
};
|
||||
|
||||
getDefaultState = () => ({
|
||||
traceEvents: [],
|
||||
});
|
||||
|
||||
// workaround for type declaration issues in cdp lib
|
||||
asApis = (client) => client;
|
||||
|
||||
getClientApis = async () => this.asApis(await this.getClient());
|
||||
|
||||
getClient = async () => {
|
||||
if (this.state.client) {
|
||||
return this.state.client;
|
||||
}
|
||||
|
||||
const client = await CDP({ port: this.deps.port });
|
||||
|
||||
const { Profiler, Page } = this.asApis(client);
|
||||
await Promise.all([Page.enable(), Profiler.enable(), Profiler.setSamplingInterval({ interval: 100 })]);
|
||||
|
||||
this.state.client = client;
|
||||
|
||||
return client;
|
||||
};
|
||||
|
||||
start = async ({ id }) => {
|
||||
if (this.state.tracingPromise) {
|
||||
throw new Error(`collection in progress - can't start another one! ${id}`);
|
||||
}
|
||||
|
||||
const { Tracing, Profiler } = await this.getClientApis();
|
||||
|
||||
await Promise.all([
|
||||
Tracing.start({
|
||||
bufferUsageReportingInterval: 1000,
|
||||
traceConfig: {
|
||||
includedCategories: this.tracingCategories,
|
||||
},
|
||||
}),
|
||||
Profiler.start(),
|
||||
]);
|
||||
|
||||
Tracing.on('dataCollected', ({ value: events }) => {
|
||||
this.state.traceEvents.push(...events);
|
||||
});
|
||||
|
||||
let resolveFn;
|
||||
this.state.tracingPromise = new Promise((resolve) => {
|
||||
resolveFn = resolve;
|
||||
});
|
||||
Tracing.on('tracingComplete', ({ dataLossOccurred }) => {
|
||||
const t = new Tracelib(this.state.traceEvents);
|
||||
|
||||
const eventCounts = countBy(this.state.traceEvents, (ev) => ev.name);
|
||||
|
||||
const fps = t.getFPS();
|
||||
|
||||
resolveFn({
|
||||
eventCounts,
|
||||
fps: mean(fps.values),
|
||||
tracingDataLoss: dataLossOccurred ? 1 : 0,
|
||||
warnings: t.getWarningCounts(),
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
stop = async (req) => {
|
||||
if (!this.state.tracingPromise) {
|
||||
throw new Error(`collection was never started - there is nothing to stop!`);
|
||||
}
|
||||
|
||||
const { Tracing, Profiler } = await this.getClientApis();
|
||||
|
||||
// TODO: capture profiler data
|
||||
const [, , traceData] = await Promise.all([Profiler.stop(), Tracing.end(), this.state.tracingPromise]);
|
||||
|
||||
await this.resetState();
|
||||
|
||||
return traceData;
|
||||
};
|
||||
|
||||
close = async () => {
|
||||
await this.resetState();
|
||||
};
|
||||
}
|
||||
|
||||
exports.CDPDataCollector = CDPDataCollector;
|
||||
@@ -0,0 +1,94 @@
|
||||
const { fromPairs } = require('lodash');
|
||||
|
||||
const isLivePerformanceAppStats = (data) =>
|
||||
data.some((st) => {
|
||||
const stat = st?.[MeasurementName.DataRenderDelay];
|
||||
return Array.isArray(stat) && Boolean(stat?.length);
|
||||
});
|
||||
|
||||
const formatAppStats = (allStats) => {
|
||||
if (!isLivePerformanceAppStats(allStats)) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const names = Object.keys(MeasurementName);
|
||||
|
||||
return fromPairs(
|
||||
names.map((name) => {
|
||||
const statsForMeasurement = allStats.map((s) => s[name]);
|
||||
const res = {
|
||||
total: {
|
||||
count: [],
|
||||
avg: [],
|
||||
},
|
||||
lastInterval: {
|
||||
avg: [],
|
||||
min: [],
|
||||
max: [],
|
||||
count: [],
|
||||
},
|
||||
};
|
||||
|
||||
statsForMeasurement.forEach((s) => {
|
||||
const total = s.reduce(
|
||||
(prev, next) => {
|
||||
prev.count += next.count;
|
||||
prev.avg += next.avg;
|
||||
return prev;
|
||||
},
|
||||
{ count: 0, avg: 0 }
|
||||
);
|
||||
res.total.count.push(Math.round(total.count));
|
||||
res.total.avg.push(Math.round(total.avg / s.length));
|
||||
|
||||
const lastInterval = s[s.length - 1];
|
||||
|
||||
res.lastInterval.avg.push(Math.round(lastInterval?.avg));
|
||||
res.lastInterval.min.push(Math.round(lastInterval?.min));
|
||||
res.lastInterval.max.push(Math.round(lastInterval?.max));
|
||||
res.lastInterval.count.push(Math.round(lastInterval?.count));
|
||||
});
|
||||
|
||||
return [name, res];
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
const emptyFormattedCDPData = () => ({
|
||||
minorGC: [],
|
||||
majorGC: [],
|
||||
droppedFrames: [],
|
||||
fps: [],
|
||||
tracingDataLossOccurred: false,
|
||||
longTaskWarnings: [],
|
||||
});
|
||||
|
||||
const isCDPData = (data) => data.every((d) => typeof d.eventCounts === 'object');
|
||||
|
||||
const formatCDPData = (data) => {
|
||||
if (!isCDPData(data)) {
|
||||
return emptyFormattedCDPData();
|
||||
}
|
||||
|
||||
return data.reduce((acc, next) => {
|
||||
acc.majorGC.push(next.eventCounts.MajorGC ?? 0);
|
||||
acc.minorGC.push(next.eventCounts.MinorGC ?? 0);
|
||||
acc.fps.push(Math.round(next.fps) ?? 0);
|
||||
acc.tracingDataLossOccurred = acc.tracingDataLossOccurred || Boolean(next.tracingDataLoss);
|
||||
acc.droppedFrames.push(next.eventCounts.DroppedFrame ?? 0);
|
||||
acc.longTaskWarnings.push(next.warnings.LongTask ?? 0);
|
||||
return acc;
|
||||
}, emptyFormattedCDPData());
|
||||
};
|
||||
|
||||
const formatResults = (results) => {
|
||||
return {
|
||||
...formatAppStats(results.map(({ appStats }) => appStats)),
|
||||
...formatCDPData(results.map(({ collectorsData }) => collectorsData[DataCollectorName.CDP])),
|
||||
|
||||
__raw: results,
|
||||
};
|
||||
};
|
||||
|
||||
exports.formatResults = formatResults;
|
||||
exports.formatAppStats = formatAppStats;
|
||||
@@ -0,0 +1,90 @@
|
||||
const fs = require('fs');
|
||||
const { fromPairs } = require('lodash');
|
||||
|
||||
const { CDPDataCollector } = require('./CDPDataCollector');
|
||||
const { formatResults } = require('./formatting');
|
||||
|
||||
const remoteDebuggingPortOptionPrefix = '--remote-debugging-port=';
|
||||
|
||||
const getOrAddRemoteDebuggingPort = (args) => {
|
||||
const existing = args.find((arg) => arg.startsWith(remoteDebuggingPortOptionPrefix));
|
||||
|
||||
if (existing) {
|
||||
return Number(existing.substring(remoteDebuggingPortOptionPrefix.length));
|
||||
}
|
||||
|
||||
const port = 40000 + Math.round(Math.random() * 25000);
|
||||
args.push(`${remoteDebuggingPortOptionPrefix}${port}`);
|
||||
return port;
|
||||
};
|
||||
|
||||
let collectors = [];
|
||||
let results = [];
|
||||
|
||||
const startBenchmarking = async ({ testName }) => {
|
||||
await Promise.all(collectors.map((coll) => coll.start({ id: testName })));
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
const stopBenchmarking = async ({ testName, appStats }) => {
|
||||
const data = await Promise.all(collectors.map(async (coll) => [coll.getName(), await coll.stop({ id: testName })]));
|
||||
|
||||
results.push({
|
||||
collectorsData: fromPairs(data),
|
||||
appStats: appStats,
|
||||
});
|
||||
|
||||
return true;
|
||||
};
|
||||
const afterRun = async () => {
|
||||
await Promise.all(collectors.map((coll) => coll.close()));
|
||||
collectors = [];
|
||||
results = [];
|
||||
};
|
||||
|
||||
const afterSpec = (resultsFolder) => async (spec) => {
|
||||
fs.writeFileSync(`${resultsFolder}/${spec.name}-${Date.now()}.json`, JSON.stringify(formatResults(results), null, 2));
|
||||
|
||||
results = [];
|
||||
};
|
||||
|
||||
const initialize = (on, config) => {
|
||||
const resultsFolder = config.env['BENCHMARK_PLUGIN_RESULTS_FOLDER'];
|
||||
|
||||
if (!fs.existsSync(resultsFolder)) {
|
||||
fs.mkdirSync(resultsFolder, { recursive: true });
|
||||
console.log(`Created folder for benchmark results ${resultsFolder}`);
|
||||
}
|
||||
|
||||
on('before:browser:launch', async (browser, options) => {
|
||||
if (browser.family !== 'chromium' || browser.name === 'electron') {
|
||||
throw new Error('benchmarking plugin requires chrome');
|
||||
}
|
||||
|
||||
const { args } = options;
|
||||
|
||||
const port = getOrAddRemoteDebuggingPort(args);
|
||||
collectors.push(new CDPDataCollector({ port }));
|
||||
|
||||
args.push('--start-fullscreen');
|
||||
|
||||
console.log(
|
||||
`initialized benchmarking plugin with ${collectors.length} collectors: ${collectors
|
||||
.map((col) => col.getName())
|
||||
.join(', ')}`
|
||||
);
|
||||
|
||||
return options;
|
||||
});
|
||||
|
||||
on('task', {
|
||||
startBenchmarking,
|
||||
stopBenchmarking,
|
||||
});
|
||||
|
||||
on('after:run', afterRun);
|
||||
on('after:spec', afterSpec(resultsFolder));
|
||||
};
|
||||
|
||||
exports.initialize = initialize;
|
||||
@@ -0,0 +1,49 @@
|
||||
'use strict';
|
||||
const BlinkDiff = require('blink-diff');
|
||||
const { resolve } = require('path');
|
||||
|
||||
// @todo use npmjs.com/pixelmatch or an available cypress plugin
|
||||
const compareScreenshots = async ({ config, screenshotsFolder, specName }) => {
|
||||
const name = config.name || config; // @todo use `??`
|
||||
const threshold = config.threshold || 0.001; // @todo use `??`
|
||||
|
||||
const imageAPath = `${screenshotsFolder}/${specName}/${name}.png`;
|
||||
const imageBPath = resolve(`${screenshotsFolder}/../expected/${specName}/${name}.png`);
|
||||
|
||||
const imageOutputPath = screenshotsFolder.endsWith('actual') ? imageAPath.replace('.png', '.diff.png') : undefined;
|
||||
|
||||
const { code } = await new Promise((resolve, reject) => {
|
||||
new BlinkDiff({
|
||||
imageAPath,
|
||||
imageBPath,
|
||||
imageOutputPath,
|
||||
threshold,
|
||||
thresholdType: BlinkDiff.THRESHOLD_PERCENT,
|
||||
}).run((error, result) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve(result);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
if (code <= 1) {
|
||||
let msg = `\nThe screenshot [${imageAPath}] differs from [${imageBPath}]`;
|
||||
msg += '\n';
|
||||
msg += '\nCheck the Artifacts tab in the CircleCi build output for the actual screenshots.';
|
||||
msg += '\n';
|
||||
msg += '\n If the difference between expected and outcome is NOT acceptable then do the following:';
|
||||
msg += '\n - Check the code for changes that causes this difference, fix that and retry.';
|
||||
msg += '\n';
|
||||
msg += '\n If the difference between expected and outcome is acceptable then do the following:';
|
||||
msg += '\n - Replace the expected image with the outcome and retry.';
|
||||
msg += '\n';
|
||||
throw new Error(msg);
|
||||
} else {
|
||||
// Must return a value
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
module.exports = compareScreenshots;
|
||||
@@ -0,0 +1,79 @@
|
||||
'use strict';
|
||||
const {
|
||||
promises: { readFile },
|
||||
} = require('fs');
|
||||
const { resolve } = require('path');
|
||||
|
||||
// @todo use https://github.com/bahmutov/cypress-extends when possible
|
||||
module.exports = async (baseConfig) => {
|
||||
// From CLI
|
||||
const {
|
||||
env: { CWD, UPDATE_SCREENSHOTS },
|
||||
} = baseConfig;
|
||||
|
||||
if (CWD) {
|
||||
// @todo: https://github.com/cypress-io/cypress/issues/6406
|
||||
const jsonReporter = require.resolve('@mochajs/json-file-reporter');
|
||||
|
||||
// @todo `baseUrl: env.CYPRESS_BASEURL`
|
||||
const projectConfig = {
|
||||
fixturesFolder: `${CWD}/cypress/fixtures`,
|
||||
integrationFolder: `${CWD}/cypress/integration`,
|
||||
reporter: jsonReporter,
|
||||
reporterOptions: {
|
||||
output: `${CWD}/cypress/report.json`,
|
||||
},
|
||||
screenshotsFolder: `${CWD}/cypress/screenshots/${UPDATE_SCREENSHOTS ? 'expected' : 'actual'}`,
|
||||
videosFolder: `${CWD}/cypress/videos`,
|
||||
};
|
||||
|
||||
const customProjectConfig = await readFile(`${CWD}/cypress.json`, 'utf8')
|
||||
.then(JSON.parse)
|
||||
.then((config) => {
|
||||
const pathKeys = [
|
||||
'fileServerFolder',
|
||||
'fixturesFolder',
|
||||
'ignoreTestFiles',
|
||||
'integrationFolder',
|
||||
'pluginsFile',
|
||||
'screenshotsFolder',
|
||||
'supportFile',
|
||||
'testFiles',
|
||||
'videosFolder',
|
||||
];
|
||||
|
||||
return Object.fromEntries(
|
||||
Object.entries(config).map(([key, value]) => {
|
||||
if (pathKeys.includes(key)) {
|
||||
return [key, resolve(CWD, value)];
|
||||
} else {
|
||||
return [key, value];
|
||||
}
|
||||
})
|
||||
);
|
||||
})
|
||||
.catch((error) => {
|
||||
if (error.code === 'ENOENT') {
|
||||
// File is optional
|
||||
return {};
|
||||
} else {
|
||||
// Unexpected error
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
...baseConfig,
|
||||
...projectConfig,
|
||||
...customProjectConfig,
|
||||
reporterOptions: {
|
||||
...baseConfig.reporterOptions,
|
||||
...projectConfig.reporterOptions,
|
||||
...customProjectConfig.reporterOptions,
|
||||
},
|
||||
};
|
||||
} else {
|
||||
// Temporary legacy support for Grafana core (using `yarn start`)
|
||||
return baseConfig;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,73 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const benchmarkPlugin = require('./benchmark');
|
||||
const compareScreenshots = require('./compareScreenshots');
|
||||
const extendConfig = require('./extendConfig');
|
||||
const readProvisions = require('./readProvisions');
|
||||
const typescriptPreprocessor = require('./typescriptPreprocessor');
|
||||
|
||||
module.exports = (on, config) => {
|
||||
if (config.env['BENCHMARK_PLUGIN_ENABLED'] === true) {
|
||||
benchmarkPlugin.initialize(on, config);
|
||||
}
|
||||
|
||||
on('file:preprocessor', typescriptPreprocessor);
|
||||
on('task', { compareScreenshots, readProvisions });
|
||||
on('task', {
|
||||
log({ message, optional }) {
|
||||
optional ? console.log(message, optional) : console.log(message);
|
||||
return null;
|
||||
},
|
||||
});
|
||||
on('task', {
|
||||
getJSONFilesFromDir: async ({ projectPath, relativePath }) => {
|
||||
const directoryPath = path.join(projectPath, relativePath);
|
||||
const jsonFiles = fs.readdirSync(directoryPath);
|
||||
return jsonFiles
|
||||
.filter((fileName) => /.json$/i.test(fileName))
|
||||
.map((fileName) => {
|
||||
const fileBuffer = fs.readFileSync(path.join(directoryPath, fileName));
|
||||
return JSON.parse(fileBuffer);
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// Make recordings higher resolution
|
||||
// https://www.cypress.io/blog/2021/03/01/generate-high-resolution-videos-and-screenshots/
|
||||
on('before:browser:launch', (browser = {}, launchOptions) => {
|
||||
console.log('launching browser %s is headless? %s', browser.name, browser.isHeadless);
|
||||
|
||||
// the browser width and height we want to get
|
||||
// our screenshots and videos will be of that resolution
|
||||
const width = 1920;
|
||||
const height = 1080;
|
||||
|
||||
console.log('setting the browser window size to %d x %d', width, height);
|
||||
|
||||
if (browser.name === 'chrome' && browser.isHeadless) {
|
||||
launchOptions.args.push(`--window-size=${width},${height}`);
|
||||
|
||||
// force screen to be non-retina and just use our given resolution
|
||||
launchOptions.args.push('--force-device-scale-factor=1');
|
||||
}
|
||||
|
||||
if (browser.name === 'electron' && browser.isHeadless) {
|
||||
// might not work on CI for some reason
|
||||
launchOptions.preferences.width = width;
|
||||
launchOptions.preferences.height = height;
|
||||
}
|
||||
|
||||
if (browser.name === 'firefox' && browser.isHeadless) {
|
||||
launchOptions.args.push(`--width=${width}`);
|
||||
launchOptions.args.push(`--height=${height}`);
|
||||
}
|
||||
|
||||
// IMPORTANT: return the updated browser launch options
|
||||
return launchOptions;
|
||||
});
|
||||
|
||||
// Always extend with this library's config and return for diffing
|
||||
// @todo remove this when possible: https://github.com/cypress-io/cypress/issues/5674
|
||||
return extendConfig(config);
|
||||
};
|
||||
@@ -0,0 +1,14 @@
|
||||
'use strict';
|
||||
const {
|
||||
promises: { readFile },
|
||||
} = require('fs');
|
||||
const { resolve: resolvePath } = require('path');
|
||||
const { parse: parseYml } = require('yaml');
|
||||
|
||||
const readProvision = (filePath) => readFile(filePath, 'utf8').then((contents) => parseYml(contents));
|
||||
|
||||
const readProvisions = (filePaths) => Promise.all(filePaths.map(readProvision));
|
||||
|
||||
// Paths are relative to <project-root>/provisioning
|
||||
module.exports = ({ CWD, filePaths }) =>
|
||||
readProvisions(filePaths.map((filePath) => resolvePath(CWD, 'provisioning', filePath)));
|
||||
@@ -0,0 +1,42 @@
|
||||
const wp = require('@cypress/webpack-preprocessor');
|
||||
const { resolve } = require('path');
|
||||
|
||||
const anyNodeModules = /node_modules/;
|
||||
const packageRoot = resolve(`${__dirname}/../../`);
|
||||
const packageModules = `${packageRoot}/node_modules`;
|
||||
|
||||
const webpackOptions = {
|
||||
module: {
|
||||
rules: [
|
||||
{
|
||||
include: (modulePath) => {
|
||||
if (!anyNodeModules.test(modulePath)) {
|
||||
// Is a file within the project
|
||||
return true;
|
||||
} else {
|
||||
// Is a file within this package
|
||||
return modulePath.startsWith(packageRoot) && !modulePath.startsWith(packageModules);
|
||||
}
|
||||
},
|
||||
test: /\.ts$/,
|
||||
use: [
|
||||
{
|
||||
loader: 'ts-loader',
|
||||
options: {
|
||||
transpileOnly: true,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
resolve: {
|
||||
extensions: ['.ts', '.js'],
|
||||
},
|
||||
};
|
||||
|
||||
const options = {
|
||||
webpackOptions,
|
||||
};
|
||||
|
||||
module.exports = wp(options);
|
||||
Reference in New Issue
Block a user