diff --git a/lerna.json b/lerna.json index 83a941b5539..109951a7fef 100644 --- a/lerna.json +++ b/lerna.json @@ -2,5 +2,5 @@ "npmClient": "yarn", "useWorkspaces": true, "packages": ["packages/*"], - "version": "6.3.0-alpha.36" + "version": "6.3.0-alpha.40" } diff --git a/package.json b/package.json index 569bbfd6096..81e53750c40 100644 --- a/package.json +++ b/package.json @@ -148,7 +148,7 @@ "themes:generate": "ts-node --project ./scripts/cli/tsconfig.json ./scripts/cli/generateSassVariableFiles.ts", "packages:prepare": "lerna run clean && npm run test && lerna version --tag-version-prefix=\"packages@\" -m \"Packages: publish %s\" --no-push", "packages:build": "lerna run clean && lerna run build", - "packages:publish": "lerna publish from-package --contents dist --tag-version-prefix=\"packages@\" --dist-tag next" + "packages:publish": "lerna publish from-package --contents dist --dist-tag next --tag-version-prefix=\"packages@\"" }, "husky": { "hooks": { diff --git a/packages/grafana-data/package.json b/packages/grafana-data/package.json index 34d6509bfde..6f0e312c120 100644 --- a/packages/grafana-data/package.json +++ b/packages/grafana-data/package.json @@ -1,6 +1,6 @@ { "name": "@grafana/data", - "version": "6.3.0-alpha.36", + "version": "6.3.0-alpha.39", "description": "Grafana Data Library", "keywords": [ "typescript" diff --git a/packages/grafana-runtime/package.json b/packages/grafana-runtime/package.json index 7798b549dac..3f2fb98bf04 100644 --- a/packages/grafana-runtime/package.json +++ b/packages/grafana-runtime/package.json @@ -1,6 +1,6 @@ { "name": "@grafana/runtime", - "version": "6.3.0-alpha.36", + "version": "6.3.0-alpha.39", "description": "Grafana Runtime Library", "keywords": [ "grafana" diff --git a/packages/grafana-toolkit/package.json b/packages/grafana-toolkit/package.json index 88713004773..5aeea66dc16 100644 --- a/packages/grafana-toolkit/package.json +++ b/packages/grafana-toolkit/package.json @@ -1,6 +1,6 @@ { "name": "@grafana/toolkit", - "version": "6.3.0-alpha.36", + "version": "6.3.0-alpha.40", "description": "Grafana Toolkit", "keywords": [ "grafana", @@ -30,6 +30,7 @@ "@types/node": "^12.0.4", "@types/react-dev-utils": "^9.0.1", "@types/semver": "^6.0.0", + "@types/tmp": "^0.1.0", "@types/webpack": "4.4.34", "axios": "0.19.0", "babel-loader": "8.0.6", diff --git a/packages/grafana-toolkit/src/cli/index.ts b/packages/grafana-toolkit/src/cli/index.ts index 17277a4b4bf..ab9efadae40 100644 --- a/packages/grafana-toolkit/src/cli/index.ts +++ b/packages/grafana-toolkit/src/cli/index.ts @@ -13,7 +13,13 @@ import { pluginTestTask } from './tasks/plugin.tests'; import { searchTestDataSetupTask } from './tasks/searchTestDataSetup'; import { closeMilestoneTask } from './tasks/closeMilestone'; import { pluginDevTask } from './tasks/plugin.dev'; -import { pluginCITask } from './tasks/plugin.ci'; +import { + ciBuildPluginTask, + ciBundlePluginTask, + ciTestPluginTask, + ciDeployPluginTask, + ciSetupPluginTask, +} from './tasks/plugin.ci'; import { buildPackageTask } from './tasks/package.build'; export const run = (includeInternalScripts = false) => { @@ -141,15 +147,47 @@ export const run = (includeInternalScripts = false) => { }); program - .command('plugin:ci') - .option('--dryRun', "Dry run (don't post results)") - .description('Run Plugin CI task') + .command('plugin:ci-build') + .option('--platform ', 'For backend task, which backend to run') + .description('Build the plugin, leaving artifacts in /dist') .action(async cmd => { - await execTask(pluginCITask)({ - dryRun: cmd.dryRun, + await execTask(ciBuildPluginTask)({ + platform: cmd.platform, }); }); + program + .command('plugin:ci-bundle') + .description('Create a zip artifact for the plugin') + .action(async cmd => { + await execTask(ciBundlePluginTask)({}); + }); + + program + .command('plugin:ci-setup') + .option('--installer ', 'Name of installer to download and run') + .description('Install and configure grafana') + .action(async cmd => { + await execTask(ciSetupPluginTask)({ + installer: cmd.installer, + }); + }); + program + .command('plugin:ci-test') + .description('end-to-end test using bundle in /artifacts') + .action(async cmd => { + await execTask(ciTestPluginTask)({ + platform: cmd.platform, + }); + }); + + program + .command('plugin:ci-deploy') + .description('Publish plugin CI results') + .action(async cmd => { + await execTask(ciDeployPluginTask)({}); + }); + program.on('command:*', () => { console.error('Invalid command: %s\nSee --help for a list of available commands.', program.args.join(' ')); process.exit(1); diff --git a/packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts b/packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts index c9ce6db4e65..a30199a70cf 100644 --- a/packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts +++ b/packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts @@ -9,7 +9,8 @@ import path = require('path'); import fs = require('fs'); export interface PluginCIOptions { - dryRun?: boolean; + platform?: string; + installer?: string; } const calcJavascriptSize = (base: string, files?: string[]): number => { @@ -32,22 +33,106 @@ const calcJavascriptSize = (base: string, files?: string[]): number => { return size; }; -const pluginCIRunner: TaskRunner = async ({ dryRun }) => { +const getWorkFolder = () => { + let dir = `${process.cwd()}/work`; + if (process.env.CIRCLE_JOB) { + dir = path.resolve(dir, process.env.CIRCLE_JOB); + } + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + return dir; +}; + +const writeWorkStats = (startTime: number, workDir: string) => { + const elapsed = Date.now() - startTime; + const stats = { + job: `${process.env.CIRCLE_JOB}`, + startTime, + buildTime: elapsed, + endTime: Date.now(), + }; + const f = path.resolve(workDir, 'stats.json'); + fs.writeFile(f, JSON.stringify(stats, null, 2), err => { + if (err) { + throw new Error('Unable to stats: ' + f); + } + }); +}; + +/** + * 1. BUILD + * + * when platform exists it is building backend, otherwise frontend + * + * Each build writes data: + * ~/work/build_xxx/ + * + * Anything that should be put into the final zip file should be put in: + * ~/work/build_xxx/dist + */ +const buildPluginRunner: TaskRunner = async ({ platform }) => { const start = Date.now(); - const distDir = `${process.cwd()}/dist`; - const artifactsDir = `${process.cwd()}/artifacts`; - await execa('rimraf', [`${process.cwd()}/coverage`]); - await execa('rimraf', [artifactsDir]); + const workDir = getWorkFolder(); + await execa('rimraf', [workDir]); + fs.mkdirSync(workDir); - // Do regular build process - await pluginBuildRunner({ coverage: true }); - const elapsed = Date.now() - start; + if (platform) { + console.log('TODO, backend support?'); + const file = path.resolve(workDir, 'README.txt'); + fs.writeFile(workDir + '/README.txt', 'TODO... build it!', err => { + if (err) { + throw new Error('Unable to write: ' + file); + } + }); + } else { + // Do regular build process with coverage + await pluginBuildRunner({ coverage: true }); + } + // Move dist to the scoped work folder + const distDir = path.resolve(process.cwd(), 'dist'); + if (fs.existsSync(distDir)) { + fs.renameSync(distDir, path.resolve(workDir, 'dist')); + } + writeWorkStats(start, workDir); +}; + +export const ciBuildPluginTask = new Task('Build Plugin', buildPluginRunner); + +/** + * 2. BUNDLE + * + * Take everything from `~/work/build_XXX/dist` and zip it into + * artifacts + * + */ +const bundlePluginRunner: TaskRunner = async () => { + const start = Date.now(); + const workDir = getWorkFolder(); + + // Copy all `dist` folders to the root dist folder + const distDir = path.resolve(process.cwd(), 'dist'); + if (!fs.existsSync(distDir)) { + fs.mkdirSync(distDir); + } + fs.mkdirSync(distDir, { recursive: true }); + const dirs = fs.readdirSync(workDir); + for (const dir of dirs) { + if (dir.startsWith('build_')) { + const contents = path.resolve(dir, 'dist'); + if (fs.existsSync(contents)) { + await execa('cp', ['-rp', contents, distDir]); + } + } + } + + // Create an artifact + const artifactsDir = path.resolve(process.cwd(), 'artifacts'); if (!fs.existsSync(artifactsDir)) { - fs.mkdirSync(artifactsDir); + fs.mkdirSync(artifactsDir, { recursive: true }); } - // TODO? can this typed from @grafana/ui? const pluginInfo = getPluginJson(`${distDir}/plugin.json`); const zipName = pluginInfo.id + '-' + pluginInfo.info.version + '.zip'; const zipFile = path.resolve(artifactsDir, zipName); @@ -55,23 +140,165 @@ const pluginCIRunner: TaskRunner = async ({ dryRun }) => { await execa('zip', ['-r', zipFile, '.']); restoreCwd(); - const stats = { - startTime: start, - buildTime: elapsed, - jsSize: calcJavascriptSize(distDir), - zipSize: fs.statSync(zipFile).size, - endTime: Date.now(), + const zipStats = fs.statSync(zipFile); + if (zipStats.size < 100) { + throw new Error('Invalid zip file: ' + zipFile); + } + await execa('sha1sum', [zipFile, '>', zipFile + '.sha1']); + const info = { + name: zipName, + size: zipStats.size, }; - fs.writeFile(artifactsDir + '/stats.json', JSON.stringify(stats, null, 2), err => { + const f = path.resolve(artifactsDir, 'info.json'); + fs.writeFile(f, JSON.stringify(info, null, 2), err => { if (err) { - throw new Error('Unable to write stats'); + throw new Error('Error writing artifact info: ' + f); } - console.log('Stats', stats); }); - if (!dryRun) { - console.log('TODO send info to github?'); - } + writeWorkStats(start, workDir); }; -export const pluginCITask = new Task('Plugin CI', pluginCIRunner); +export const ciBundlePluginTask = new Task('Bundle Plugin', bundlePluginRunner); + +/** + * 3. Setup (install grafana and setup provisioning) + * + * deploy the zip to a running grafana instance + * + */ +const setupPluginRunner: TaskRunner = async ({ installer }) => { + const start = Date.now(); + + if (!installer) { + throw new Error('Missing installer path'); + } + + // Download the grafana installer + const workDir = getWorkFolder(); + const installFile = path.resolve(workDir, installer); + if (!fs.existsSync(installFile)) { + console.log('download', installer); + const exe = await execa('wget', ['-O', installFile, 'https://dl.grafana.com/oss/release/' + installer]); + console.log(exe.stdout); + } + + // Find the plugin zip file + const artifactsDir = path.resolve(process.cwd(), 'artifacts'); + const artifactsInfo = require(path.resolve(artifactsDir, 'info.json')); + const pluginZip = path.resolve(workDir, 'artifacts', artifactsInfo.name); + if (!fs.existsSync(pluginZip)) { + throw new Error('Missing zip file:' + pluginZip); + } + + // Create a grafana runtime folder + const grafanaPluginsDir = path.resolve(require('os').homedir(), 'grafana', 'plugins'); + await execa('rimraf', [grafanaPluginsDir]); + fs.mkdirSync(grafanaPluginsDir, { recursive: true }); + + // unzip package.zip -d /opt + let exe = await execa('unzip', [pluginZip, '-d', grafanaPluginsDir]); + console.log(exe.stdout); + + // Write the custom settings + const customIniPath = '/usr/share/grafana/conf/custom.ini'; + const customIniBody = `[paths] \n` + `plugins = ${grafanaPluginsDir}\n` + ''; + fs.writeFile(customIniPath, customIniBody, err => { + if (err) { + throw new Error('Unable to write: ' + customIniPath); + } + }); + + console.log('Install Grafana'); + exe = await execa('sudo', ['dpkg', 'i', installFile]); + console.log(exe.stdout); + + exe = await execa('sudo', ['grafana-server', 'start']); + console.log(exe.stdout); + exe = await execa('grafana-cli', ['--version']); + + writeWorkStats(start, workDir + '_setup'); +}; + +export const ciSetupPluginTask = new Task('Setup Grafana', setupPluginRunner); + +/** + * 4. Test (end-to-end) + * + * deploy the zip to a running grafana instance + * + */ +const testPluginRunner: TaskRunner = async ({ platform }) => { + const start = Date.now(); + const workDir = getWorkFolder(); + + const args = { + withCredentials: true, + baseURL: process.env.GRAFANA_URL || 'http://localhost:3000/', + responseType: 'json', + auth: { + username: 'admin', + password: 'admin', + }, + }; + + const axios = require('axios'); + const frontendSettings = await axios.get('api/frontend/settings', args); + + console.log('Grafana Version: ' + JSON.stringify(frontendSettings.data.buildInfo, null, 2)); + + const pluginInfo = getPluginJson(`${process.cwd()}/src/plugin.json`); + const pluginSettings = await axios.get(`api/plugins/${pluginInfo.id}/settings`, args); + + console.log('Plugin Info: ' + JSON.stringify(pluginSettings.data, null, 2)); + + console.log('TODO puppeteer'); + + const elapsed = Date.now() - start; + const stats = { + job: `${process.env.CIRCLE_JOB}`, + sha1: `${process.env.CIRCLE_SHA1}`, + startTime: start, + buildTime: elapsed, + endTime: Date.now(), + }; + + console.log('TODO Puppeteer Tests', stats); + writeWorkStats(start, workDir); +}; + +export const ciTestPluginTask = new Task('Test Plugin (e2e)', testPluginRunner); + +/** + * 4. Deploy + * + * deploy the zip to a running grafana instance + * + */ +const deployPluginRunner: TaskRunner = async () => { + const start = Date.now(); + + // TASK Time + if (process.env.CIRCLE_INTERNAL_TASK_DATA) { + const timingInfo = fs.readdirSync(`${process.env.CIRCLE_INTERNAL_TASK_DATA}`); + if (timingInfo) { + timingInfo.forEach(file => { + console.log('TIMING INFO: ', file); + }); + } + } + + const elapsed = Date.now() - start; + const stats = { + job: `${process.env.CIRCLE_JOB}`, + sha1: `${process.env.CIRCLE_SHA1}`, + startTime: start, + buildTime: elapsed, + endTime: Date.now(), + }; + console.log('TODO DEPLOY??', stats); + console.log(' if PR => write a comment to github with difference '); + console.log(' if master | vXYZ ==> upload artifacts to some repo '); +}; + +export const ciDeployPluginTask = new Task('Deploy plugin', deployPluginRunner); diff --git a/packages/grafana-ui/package.json b/packages/grafana-ui/package.json index de0cb7f3eb1..4584e1d0d5e 100644 --- a/packages/grafana-ui/package.json +++ b/packages/grafana-ui/package.json @@ -1,6 +1,6 @@ { "name": "@grafana/ui", - "version": "6.3.0-alpha.36", + "version": "6.3.0-alpha.39", "description": "Grafana Components Library", "keywords": [ "grafana", diff --git a/yarn.lock b/yarn.lock index 44aab000579..0a650dc0496 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2568,6 +2568,11 @@ resolved "https://registry.yarnpkg.com/@types/tinycolor2/-/tinycolor2-1.4.2.tgz#721ca5c5d1a2988b4a886e35c2ffc5735b6afbdf" integrity sha512-PeHg/AtdW6aaIO2a+98Xj7rWY4KC1E6yOy7AFknJQ7VXUGNrMlyxDFxJo7HqLtjQms/ZhhQX52mLVW/EX3JGOw== +"@types/tmp@^0.1.0": + version "0.1.0" + resolved "https://registry.yarnpkg.com/@types/tmp/-/tmp-0.1.0.tgz#19cf73a7bcf641965485119726397a096f0049bd" + integrity sha512-6IwZ9HzWbCq6XoQWhxLpDjuADodH/MKXRUIDFudvgjcVdjFknvmR+DNsoUeer4XPrEnrZs04Jj+kfV9pFsrhmA== + "@types/uglify-js@*": version "3.0.4" resolved "https://registry.yarnpkg.com/@types/uglify-js/-/uglify-js-3.0.4.tgz#96beae23df6f561862a830b4288a49e86baac082" @@ -15757,6 +15762,13 @@ tmp@^0.0.33: dependencies: os-tmpdir "~1.0.2" +tmp@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.1.0.tgz#ee434a4e22543082e294ba6201dcc6eafefa2877" + integrity sha512-J7Z2K08jbGcdA1kkQpJSqLF6T0tdQqpR2pnSUXsIchbPdTI9v3e85cLW0d6WDhwuAleOV71j2xWs8qMPfK7nKw== + dependencies: + rimraf "^2.6.3" + tmpl@1.0.x: version "1.0.4" resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1"