From 3745cccde6d866a83b5f5edc58e52df313e3fc88 Mon Sep 17 00:00:00 2001 From: Stephanie Closson Date: Fri, 6 Mar 2020 09:15:04 -0800 Subject: [PATCH] Toolkit: include a github release utility (#22520) * Adding git publish to toolkit * grafana-toolkit new feature plugin:github-release * Feedback from code review 1. Used async await and exec for executing commands 2. Using dist folder to get plugin information * Reverting change to plugin.json back to original value. * reverted changes to yarn.lock * reverted changes to yarn.lock * feedback from code review * feedback from code review 2 * moved constants to recommended functions * styling changes and reverting yarn.lock * removing changes to package.json * replced execLine with execa * better error detection around the publish token * made simpler with commitHash from build * Testing showed a number of required changes: - Make the sha configurable or through environment variable or through git config. - Allow a release to be recreated - Set name and repo from git config as this is what ghr is expecting anyway. - Appropriate errors if the user tries to run a release without doing a ci-build and ci-package first. * Using spinner. Took out extra dependencies out of project.json wrote tests manually. * Updated tests. Now passing * Adding git publish to toolkit * grafana-toolkit new feature plugin:github-release * Feedback from code review 1. Used async await and exec for executing commands 2. Using dist folder to get plugin information * Reverting change to plugin.json back to original value. * reverted changes to yarn.lock * reverted changes to yarn.lock * feedback from code review * feedback from code review 2 * moved constants to recommended functions * styling changes and reverting yarn.lock * removing changes to package.json * replced execLine with execa * better error detection around the publish token * made simpler with commitHash from build * Testing showed a number of required changes: - Make the sha configurable or through environment variable or through git config. - Allow a release to be recreated - Set name and repo from git config as this is what ghr is expecting anyway. - Appropriate errors if the user tries to run a release without doing a ci-build and ci-package first. * Using spinner. Took out extra dependencies out of project.json wrote tests manually. * Updated tests. Now passing * updated test for reducers, from master * package.json and yarn.lock from master --- packages/grafana-toolkit/src/cli/index.ts | 17 ++ .../src/cli/tasks/manifest.test.ts | 1 + .../src/cli/tasks/plugin.ci.ts | 8 + .../src/cli/tasks/plugin.utils.ts | 180 ++++++++++++++++++ .../src/cli/utils/githubRelease.ts | 106 +++++++++++ 5 files changed, 312 insertions(+) create mode 100755 packages/grafana-toolkit/src/cli/tasks/plugin.utils.ts create mode 100644 packages/grafana-toolkit/src/cli/utils/githubRelease.ts diff --git a/packages/grafana-toolkit/src/cli/index.ts b/packages/grafana-toolkit/src/cli/index.ts index e26915c8829..ac087f5e1b3 100644 --- a/packages/grafana-toolkit/src/cli/index.ts +++ b/packages/grafana-toolkit/src/cli/index.ts @@ -14,6 +14,7 @@ import { pluginTestTask } from './tasks/plugin.tests'; import { searchTestDataSetupTask } from './tasks/searchTestDataSetup'; import { closeMilestoneTask } from './tasks/closeMilestone'; import { pluginDevTask } from './tasks/plugin.dev'; +import { githubPublishTask } from './tasks/plugin.utils'; import { ciBuildPluginTask, ciBuildPluginDocsTask, @@ -208,6 +209,22 @@ export const run = (includeInternalScripts = false) => { }); }); + program + .command('plugin:github-publish') + .option('--dryrun', 'Do a dry run only', false) + .option('--verbose', 'Print verbose', false) + .option('--commitHash ', 'Specify the commit hash') + .option('--recreate', 'Recreate the release if already present') + .description('Publish to github ... etc etc etc') + .action(async cmd => { + await execTask(githubPublishTask)({ + dryrun: cmd.dryrun, + verbose: cmd.verbose, + commitHash: cmd.commitHash, + recreate: cmd.recreate, + }); + }); + // Test the manifest creation program .command('manifest') diff --git a/packages/grafana-toolkit/src/cli/tasks/manifest.test.ts b/packages/grafana-toolkit/src/cli/tasks/manifest.test.ts index 9f4a682530f..7e9d22c164e 100644 --- a/packages/grafana-toolkit/src/cli/tasks/manifest.test.ts +++ b/packages/grafana-toolkit/src/cli/tasks/manifest.test.ts @@ -21,6 +21,7 @@ describe('Manifest', () => { "plugin.create.ts", "plugin.dev.ts", "plugin.tests.ts", + "plugin.utils.ts", "precommit.ts", "searchTestDataSetup.ts", "task.ts", diff --git a/packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts b/packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts index 076fca90391..b54a5ff0367 100644 --- a/packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts +++ b/packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts @@ -118,6 +118,14 @@ const packagePluginRunner: TaskRunner = async () => { const packagesDir = path.resolve(ciDir, 'packages'); const distDir = path.resolve(ciDir, 'dist'); const docsDir = path.resolve(ciDir, 'docs'); + const jobsDir = path.resolve(ciDir, 'jobs'); + + fs.exists(jobsDir, jobsDirExists => { + if (!jobsDirExists) { + throw 'You must run plugin:ci-build prior to running plugin:ci-package'; + } + }); + const grafanaEnvDir = path.resolve(ciDir, 'grafana-test-env'); await execa('rimraf', [packagesDir, distDir, grafanaEnvDir]); fs.mkdirSync(packagesDir); diff --git a/packages/grafana-toolkit/src/cli/tasks/plugin.utils.ts b/packages/grafana-toolkit/src/cli/tasks/plugin.utils.ts new file mode 100755 index 00000000000..de021abe0bc --- /dev/null +++ b/packages/grafana-toolkit/src/cli/tasks/plugin.utils.ts @@ -0,0 +1,180 @@ +import { Task, TaskRunner } from './task'; +import { getPluginJson } from '../../config/utils/pluginValidation'; +import { GitHubRelease } from '../utils/githubRelease'; +import { getPluginId } from '../../config/utils/getPluginId'; +import { getCiFolder } from '../../plugins/env'; +import { useSpinner } from '../utils/useSpinner'; + +import path = require('path'); + +// @ts-ignore +import execa = require('execa'); + +interface Command extends Array {} + +const releaseNotes = async (): Promise => { + const { stdout } = await execa.shell(`awk \'BEGIN {FS="##"; RS=""} FNR==3 {print; exit}\' CHANGELOG.md`); + return stdout; +}; + +const checkoutBranch = async (branchName: string): Promise => { + const currentBranch = await execa.shell(`git rev-parse --abbrev-ref HEAD`); + const branchesAvailable = await execa.shell( + `(git branch -a | grep ${branchName} | grep -v remote) || echo 'No release found'` + ); + + if (currentBranch.stdout !== branchName) { + if (branchesAvailable.stdout.trim() === branchName) { + return ['git', ['checkout', branchName]]; + } else { + return ['git', ['checkout', '-b', branchName]]; + } + } + return []; +}; + +const gitUrlParse = (url: string): { owner: string; name: string } => { + let matchResult: RegExpMatchArray | null = []; + + if (url.match(/^git@github.com/)) { + // We have an ssh style url. + matchResult = url.match(/^git@github.com:(.*?)\/(.*?)\.git/); + } + + if (url.match(/^https:\/\/github.com\//)) { + // We have an https style url + matchResult = url.match(/^https:\/\/github.com\/(.*?)\/(.*?)\/.git/); + } + + if (matchResult && matchResult.length > 2) { + return { + owner: matchResult[1], + name: matchResult[2], + }; + } + + throw `Coult not find a suitable git repository. Received [${url}]`; +}; + +const prepareRelease = useSpinner('Preparing release', async ({ dryrun, verbose }) => { + const ciDir = getCiFolder(); + const distDir = path.resolve(ciDir, 'dist'); + const distContentDir = path.resolve(distDir, getPluginId()); + const pluginJsonFile = path.resolve(distContentDir, 'plugin.json'); + const pluginVersion = getPluginJson(pluginJsonFile).info.version; + const GIT_EMAIL = 'eng@grafana.com'; + const GIT_USERNAME = 'CircleCI Automation'; + + const githubPublishScript: Command = [ + ['git', ['config', 'user.email', GIT_EMAIL]], + ['git', ['config', 'user.name', GIT_USERNAME]], + await checkoutBranch(`release-${pluginVersion}`), + ['git', ['add', '--force', distDir], { dryrun }], + [ + 'git', + ['commit', '-m', `automated release ${pluginVersion} [skip ci]`], + { + dryrun, + okOnError: [/nothing to commit/g, /nothing added to commit/g, /no changes added to commit/g], + }, + ], + ['git', ['tag', '-f', pluginVersion]], + ['git', ['push', '-f', 'origin', `release-${pluginVersion}`], { dryrun }], + ]; + + for (let line of githubPublishScript) { + const opts = line.length === 3 ? line[2] : {}; + const command = line[0]; + const args = line[1]; + + try { + if (verbose) { + console.log('executing >>', line); + } + + if (line.length > 0 && line[0].length > 0) { + if (opts['dryrun']) { + line[1].push('--dry-run'); + } + const { stdout } = await execa(command, args); + if (verbose) { + console.log(stdout); + } + } else { + if (verbose) { + console.log('skipping empty line'); + } + } + } catch (ex) { + const err: string = ex.message; + if (opts['okOnError'] && Array.isArray(opts['okOnError'])) { + let trueError = true; + for (let regex of opts['okOnError']) { + if (err.match(regex)) { + trueError = false; + break; + } + } + + if (!trueError) { + // This is not an error + continue; + } + } + console.error(err); + process.exit(-1); + } + } +}); + +interface GithubPluglishReleaseOptions { + commitHash?: string; + recreate?: boolean; + githubToken: string; + gitRepoOwner: string; + gitRepoName: string; +} + +const createRelease = useSpinner( + 'Creating release', + async ({ commitHash, recreate, githubToken, gitRepoName, gitRepoOwner }) => { + const gitRelease = new GitHubRelease(githubToken, gitRepoOwner, gitRepoName, await releaseNotes(), commitHash); + return gitRelease.release(recreate || false); + } +); + +export interface GithubPublishOptions { + dryrun?: boolean; + verbose?: boolean; + commitHash?: string; + recreate?: boolean; +} + +const githubPublishRunner: TaskRunner = async ({ dryrun, verbose, commitHash, recreate }) => { + if (!process.env['CIRCLE_REPOSITORY_URL']) { + throw `The release plugin requires you specify the repository url as environment variable CIRCLE_REPOSITORY_URL`; + } + + if (!process.env['GITHUB_TOKEN']) { + throw `Github publish requires that you set the environment variable GITHUB_TOKEN to a valid github api token. + See: https://github.com/settings/tokens for more details.`; + } + + const parsedUrl = gitUrlParse(process.env['CIRCLE_REPOSITORY_URL']); + const githubToken = process.env['GITHUB_TOKEN']; + + await prepareRelease({ + dryrun, + verbose, + }); + + await createRelease({ + commitHash, + recreate, + githubToken, + gitRepoOwner: parsedUrl.owner, + gitRepoName: parsedUrl.name, + }); +}; + +export const githubPublishTask = new Task('Github Publish', githubPublishRunner); diff --git a/packages/grafana-toolkit/src/cli/utils/githubRelease.ts b/packages/grafana-toolkit/src/cli/utils/githubRelease.ts new file mode 100644 index 00000000000..e1ab8a67881 --- /dev/null +++ b/packages/grafana-toolkit/src/cli/utils/githubRelease.ts @@ -0,0 +1,106 @@ +import { getPluginId } from '../../config/utils/getPluginId'; +import { getPluginJson } from '../../config/utils/pluginValidation'; +import { getCiFolder } from '../../plugins/env'; +import path = require('path'); +// @ts-ignore +import execa = require('execa'); + +const ghrPlatform = (): string => { + switch (process.platform) { + case 'win32': + return 'windows'; + case 'darwin': + return 'darwin'; + case 'linux': + return 'linux'; + default: + return process.platform; + } +}; + +class GitHubRelease { + token: string; + username: string; + repository: string; + releaseNotes: string; + commitHash?: string; + + constructor(token: string, username: string, repository: string, releaseNotes: string, commitHash?: string) { + this.token = token; + this.username = username; + this.repository = repository; + this.releaseNotes = releaseNotes; + this.commitHash = commitHash; + } + + /** + * Get the ghr binary to perform the release + */ + private async getGhr(): Promise { + const GHR_VERSION = '0.13.0'; + const GHR_ARCH = process.arch === 'x64' ? 'amd64' : '386'; + const GHR_PLATFORM = ghrPlatform(); + const GHR_EXTENSION = process.platform === 'linux' ? 'tar.gz' : 'zip'; + const outName = `./ghr.${GHR_EXTENSION}`; + const archiveName = `ghr_v${GHR_VERSION}_${GHR_PLATFORM}_${GHR_ARCH}`; + const exeName = process.platform === 'linux' ? 'ghr' : 'ghr.exe'; + const exeNameFullPath = path.resolve(process.cwd(), archiveName, exeName); + const ghrUrl = `https://github.com/tcnksm/ghr/releases/download/v${GHR_VERSION}/${archiveName}.${GHR_EXTENSION}`; + await execa('wget', [ghrUrl, `--output-document=${outName}`]); + if (GHR_EXTENSION === 'tar.gz') { + await execa('tar', ['zxvf', outName]); + } else { + await execa('unzip', ['-p', outName]); + } + + if (process.platform === 'linux') { + await execa('chmod', ['755', exeNameFullPath]); + } + + return exeNameFullPath; + } + + async release(recreate: boolean) { + const ciDir = getCiFolder(); + const distDir = path.resolve(ciDir, 'dist'); + const distContentDir = path.resolve(distDir, getPluginId()); + const pluginJsonFile = path.resolve(distContentDir, 'plugin.json'); + const pluginInfo = getPluginJson(pluginJsonFile).info; + const PUBLISH_DIR = path.resolve(getCiFolder(), 'packages'); + const commitHash = this.commitHash || pluginInfo.build?.hash; + + // Get the ghr binary according to platform + const ghrExe = await this.getGhr(); + + if (!commitHash) { + throw 'The release plugin was not able to locate a commithash for release. Either build using the ci, or specify the commit hash with --commitHash '; + } + + const args = [ + '-t', + this.token, + '-u', + this.username, + '-r', + this.repository, // should override --- may not be the same + '-c', + commitHash, + '-n', + `${this.repository}_v${pluginInfo.version}`, + '-b', + this.releaseNotes, + `v${pluginInfo.version}`, + PUBLISH_DIR, + ]; + + if (recreate) { + args.splice(12, 0, '-recreate'); + } + + const { stdout } = await execa(ghrExe, args); + + console.log(stdout); + } +} + +export { GitHubRelease };