From 755f3d0de713ef738581ba56201407d756efc008 Mon Sep 17 00:00:00 2001 From: Arve Knudsen Date: Thu, 24 Sep 2020 15:20:48 +0200 Subject: [PATCH] CI: Migrate to Drone for master builds (#27087) * CI: Migrate to Drone for master builds --- .circleci/config.yml | 149 ++++++----------- .drone.yml | 122 ++++++++++---- scripts/lib.star | 372 ++++++++++++++++++++++++++++++++----------- 3 files changed, 418 insertions(+), 225 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 93ec637a07c..6f25dc67684 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,10 +10,9 @@ aliases: - &filter-only-master branches: only: master - - &filter-master-or-release + - &filter-release branches: only: - - master - chore/test-release-pipeline tags: only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ @@ -86,10 +85,6 @@ jobs: # We're testing the release pipeline /tmp/grabpl build-backend --github-token "${GITHUB_GRAFANABOT_TOKEN}" --edition << parameters.edition >> \ --variants << parameters.variant >> v7.0.0-test - else - # A master or PR build - /tmp/grabpl build-backend --github-token "${GITHUB_GRAFANABOT_TOKEN}" --edition << parameters.edition >> \ - --variants << parameters.variant >> --build-id $CIRCLE_WORKFLOW_ID fi - run: name: Move artifacts @@ -154,9 +149,6 @@ jobs: elif [[ $CIRCLE_BRANCH == "chore/test-release-pipeline" ]]; then # We're testing the release pipeline /tmp/grabpl build-frontend --github-token "${GITHUB_GRAFANABOT_TOKEN}" --edition << parameters.edition >> 7.0.0-test - else - # A master or PR build - /tmp/grabpl build-frontend --github-token "${GITHUB_GRAFANABOT_TOKEN}" --edition << parameters.edition >> --build-id $CIRCLE_WORKFLOW_ID fi - run: name: Move artifacts @@ -410,17 +402,6 @@ jobs: elif [[ $CIRCLE_BRANCH == "chore/test-release-pipeline" ]]; then # We're testing the release pipeline /tmp/grabpl package --github-token "${GITHUB_GRAFANABOT_TOKEN}" --jobs 2 --edition oss --sign v7.0.0-test - elif [[ $CIRCLE_BRANCH == "master" ]]; then - # A master build - /tmp/grabpl package --github-token "${GITHUB_GRAFANABOT_TOKEN}" --jobs 2 --edition oss --sign --build-id $CIRCLE_WORKFLOW_ID - elif [[ -n "$CIRCLE_PR_NUMBER" ]]; then - # A forked PR build, don't sign as it requires an API secret - /tmp/grabpl package --github-token "${GITHUB_GRAFANABOT_TOKEN}" --jobs 2 --edition oss --build-id $CIRCLE_WORKFLOW_ID --variants \ - linux-x64,linux-x64-musl,osx64,win64 - else - # A non-forked PR build - /tmp/grabpl package --github-token "${GITHUB_GRAFANABOT_TOKEN}" --jobs 2 --edition oss --sign --build-id $CIRCLE_WORKFLOW_ID --variants \ - linux-x64,linux-x64-musl,osx64,win64 fi - run: name: Move artifacts @@ -472,17 +453,6 @@ jobs: elif [[ $CIRCLE_BRANCH == "chore/test-release-pipeline" ]]; then # We're testing the release pipeline /tmp/grabpl package --github-token "${GITHUB_GRAFANABOT_TOKEN}" --jobs 2 --edition enterprise --sign v7.0.0-test - elif [[ $CIRCLE_BRANCH == "master" ]]; then - # A master build - /tmp/grabpl package --github-token "${GITHUB_GRAFANABOT_TOKEN}" --jobs 2 --edition enterprise --sign --build-id $CIRCLE_WORKFLOW_ID - elif [[ -n "$CIRCLE_PR_NUMBER" ]]; then - # A forked PR build, don't sign as it requires an API secret - /tmp/grabpl package --github-token "${GITHUB_GRAFANABOT_TOKEN}" --jobs 2 --edition enterprise --build-id $CIRCLE_WORKFLOW_ID --variants \ - linux-x64,linux-x64-musl,osx64,win64 - else - # A PR build - /tmp/grabpl package --github-token "${GITHUB_GRAFANABOT_TOKEN}" --jobs 2 --edition enterprise --sign --build-id $CIRCLE_WORKFLOW_ID --variants \ - linux-x64,linux-x64-musl,osx64,win64 fi - run: name: Move artifacts @@ -566,10 +536,6 @@ jobs: if [[ $CIRCLE_BRANCH == "chore/test-release-pipeline" ]]; then # We're testing the release pipeline echo Testing release - elif [[ $CIRCLE_BRANCH == "master" ]]; then - echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://grafana-storybook/canary elif [[ -n $CIRCLE_TAG ]]; then echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json gcloud auth activate-service-account --key-file=/tmp/gcpkey.json @@ -633,7 +599,7 @@ jobs: name: Build Docker images command: | source google-cloud-sdk/path.bash.inc - if [[ -n $CIRCLE_TAG || $CIRCLE_BRANCH == "chore/test-release-pipeline" || $CIRCLE_BRANCH == "master" ]]; then + if [[ -n $CIRCLE_TAG || $CIRCLE_BRANCH == "chore/test-release-pipeline" ]]; then # It's a full build /tmp/grabpl build-docker --jobs 4 --edition << parameters.edition >> \ --ubuntu=<< parameters.ubuntu >> @@ -645,7 +611,7 @@ jobs: - run: name: Exit if PR command: | - if [[ -z $CIRCLE_TAG && $CIRCLE_BRANCH != "chore/test-release-pipeline" && $CIRCLE_BRANCH != "master" ]]; then + if [[ -z $CIRCLE_TAG && $CIRCLE_BRANCH != "chore/test-release-pipeline" ]]; then echo "Nothing to do for PRs, so marking this step successful" circleci step halt fi @@ -944,19 +910,6 @@ jobs: command: "./scripts/ci-job-succeeded.sh" when: on_success - deploy-to-kubernetes: - description: "Deploy Grafana master Docker image to Kubernetes" - executor: base - steps: - - attach_workspace: - at: /tmp/workspace - - install-grabpl - - run: - name: Deploy to Kubernetes - command: | - cp -r /tmp/workspace/enterprise/dist . - /tmp/grabpl deploy-to-k8s - release-packages: executor: node steps: @@ -1040,193 +993,193 @@ workflows: build-pipeline: jobs: - build-backend: - filters: *filter-master-or-release + filters: *filter-release edition: oss variant: armv6 name: build-oss-backend-armv6 requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release edition: oss variant: armv7 name: build-oss-backend-armv7 requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release edition: oss variant: armv7-musl name: build-oss-backend-armv7-musl requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release edition: oss variant: arm64 name: build-oss-backend-arm64 requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release edition: oss variant: arm64-musl name: build-oss-backend-arm64-musl requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release edition: oss variant: osx64 name: build-oss-backend-osx64 requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release edition: oss variant: win64 name: build-oss-backend-win64 requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release edition: oss variant: linux-x64 name: build-oss-backend-linux-x64 requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release edition: oss variant: linux-x64-musl name: build-oss-backend-linux-x64-musl requires: - lint-go - build-frontend: - filters: *filter-master-or-release + filters: *filter-release name: build-oss-frontend edition: oss - build-plugins: - filters: *filter-master-or-release + filters: *filter-release name: build-oss-plugins edition: oss requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release name: build-enterprise-backend-armv6 edition: enterprise variant: armv6 requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release name: build-enterprise-backend-armv7 edition: enterprise variant: armv7 requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release name: build-enterprise-backend-armv7-musl edition: enterprise variant: armv7-musl requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release name: build-enterprise-backend-arm64 edition: enterprise variant: arm64 requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release name: build-enterprise-backend-arm64-musl edition: enterprise variant: arm64-musl requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release name: build-enterprise-backend-osx64 edition: enterprise variant: osx64 requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release name: build-enterprise-backend-win64 edition: enterprise variant: win64 requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release name: build-enterprise-backend-linux-x64 edition: enterprise variant: linux-x64 requires: - lint-go - build-backend: - filters: *filter-master-or-release + filters: *filter-release name: build-enterprise-backend-linux-x64-musl edition: enterprise variant: linux-x64-musl requires: - lint-go - build-frontend: - filters: *filter-master-or-release + filters: *filter-release name: build-enterprise-frontend edition: enterprise - build-plugins: - filters: *filter-master-or-release + filters: *filter-release name: build-enterprise-plugins edition: enterprise requires: - lint-go - build-release-publisher: - filters: *filter-master-or-release + filters: *filter-release - codespell: - filters: *filter-master-or-release + filters: *filter-release - lint-go: - filters: *filter-master-or-release + filters: *filter-release - shellcheck: - filters: *filter-master-or-release + filters: *filter-release - test-backend: - filters: *filter-master-or-release + filters: *filter-release name: test-oss-backend edition: oss requires: - lint-go - test-backend: - filters: *filter-master-or-release + filters: *filter-release name: test-enterprise-backend edition: enterprise requires: - lint-go - test-frontend: - filters: *filter-master-or-release + filters: *filter-release name: test-oss-frontend edition: oss - test-frontend: - filters: *filter-master-or-release + filters: *filter-release name: test-enterprise-frontend edition: enterprise - mysql-integration-test: - filters: *filter-master-or-release + filters: *filter-release requires: - lint-go - test-oss-backend - test-oss-frontend - postgres-integration-test: - filters: *filter-master-or-release + filters: *filter-release requires: - lint-go - test-oss-backend - test-oss-frontend - package-oss: - filters: *filter-master-or-release + filters: *filter-release requires: - build-oss-backend-armv6 - build-oss-backend-armv7 @@ -1244,7 +1197,7 @@ workflows: - shellcheck - build-oss-plugins - package-enterprise: - filters: *filter-master-or-release + filters: *filter-release requires: - build-enterprise-backend-armv6 - build-enterprise-backend-armv7 @@ -1262,17 +1215,13 @@ workflows: - shellcheck - build-enterprise-plugins - build-oss-windows-installer: - filters: *filter-master-or-release + filters: *filter-release requires: - package-oss - build-enterprise-windows-installer: - filters: *filter-master-or-release + filters: *filter-release requires: - package-enterprise - - release-next-packages: - filters: *filter-only-master - requires: - - end-to-end-tests - release-packages: filters: *filter-only-release requires: @@ -1280,7 +1229,7 @@ workflows: - mysql-integration-test - postgres-integration-test - publish-packages: - filters: *filter-master-or-release + filters: *filter-release name: publish-oss-packages edition: oss requires: @@ -1291,7 +1240,7 @@ workflows: - postgres-integration-test - build-release-publisher - publish-packages: - filters: *filter-master-or-release + filters: *filter-release name: publish-enterprise-packages edition: enterprise requires: @@ -1302,12 +1251,12 @@ workflows: - postgres-integration-test - build-release-publisher - publish-storybook: - filters: *filter-master-or-release + filters: *filter-release requires: - test-oss-backend - test-oss-frontend - build-docker-images: - filters: *filter-master-or-release + filters: *filter-release name: build-oss-docker-images edition: oss ubuntu: false @@ -1318,7 +1267,7 @@ workflows: - package-oss - build-oss-windows-installer - build-docker-images: - filters: *filter-master-or-release + filters: *filter-release name: build-oss-ubuntu-docker-images edition: oss ubuntu: true @@ -1329,7 +1278,7 @@ workflows: - package-oss - build-oss-windows-installer - build-docker-images: - filters: *filter-master-or-release + filters: *filter-release name: build-enterprise-docker-images edition: enterprise ubuntu: false @@ -1340,7 +1289,7 @@ workflows: - package-enterprise - build-enterprise-windows-installer - build-docker-images: - filters: *filter-master-or-release + filters: *filter-release name: build-enterprise-ubuntu-docker-images edition: enterprise ubuntu: true @@ -1351,13 +1300,9 @@ workflows: - package-enterprise - build-enterprise-windows-installer - end-to-end-tests: - filters: *filter-master-or-release + filters: *filter-release requires: - package-oss - - deploy-to-kubernetes: - filters: *filter-only-master - requires: - - build-enterprise-docker-images nightly: triggers: diff --git a/.drone.yml b/.drone.yml index 877ccd89aa9..09838ce1d30 100644 --- a/.drone.yml +++ b/.drone.yml @@ -16,7 +16,7 @@ steps: - name: initialize image: grafana/build-container:1.2.27 commands: - - curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v$${GRABPL_VERSION}/grabpl + - curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.5.10/grabpl - chmod +x grabpl - mkdir -p bin - mv grabpl bin @@ -26,7 +26,6 @@ steps: - yarn install --frozen-lockfile --no-progress environment: DOCKERIZE_VERSION: 0.6.1 - GRABPL_VERSION: 0.5.9 - name: lint-backend image: grafana/build-container:1.2.27 @@ -84,7 +83,7 @@ steps: - name: build-backend image: grafana/build-container:1.2.27 commands: - - ./bin/grabpl build-backend --jobs 8 --edition oss --build-id $DRONE_BUILD_NUMBER --variants linux-x64,linux-x64-musl,osx64,win64 --no-pull-enterprise + - ./bin/grabpl build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --variants linux-x64,linux-x64-musl,osx64,win64 --no-pull-enterprise depends_on: - initialize - lint-backend @@ -93,7 +92,7 @@ steps: - name: build-frontend image: grafana/build-container:1.2.27 commands: - - ./bin/grabpl build-frontend --jobs 8 --no-install-deps --edition oss --build-id $DRONE_BUILD_NUMBER --no-pull-enterprise + - ./bin/grabpl build-frontend --jobs 8 --no-install-deps --edition oss --build-id ${DRONE_BUILD_NUMBER} --no-pull-enterprise depends_on: - initialize - test-frontend @@ -109,7 +108,7 @@ steps: - name: package image: grafana/build-container:1.2.27 commands: - - . scripts/build/gpg-test-vars.sh && ./bin/grabpl package --jobs 8 --edition oss --build-id $DRONE_BUILD_NUMBER --no-pull-enterprise --variants linux-x64,linux-x64-musl,osx64,win64 + - . scripts/build/gpg-test-vars.sh && ./bin/grabpl package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --no-pull-enterprise --variants linux-x64,linux-x64-musl,osx64,win64 depends_on: - build-backend - build-frontend @@ -161,7 +160,7 @@ steps: - package - name: build-docker-images - image: grafana/drone-grafana-docker:0.2.0 + image: grafana/drone-grafana-docker:0.3.1 settings: archs: amd64 dry_run: true @@ -225,7 +224,33 @@ trigger: --- kind: pipeline type: docker -name: test-master +name: windows-pr + +platform: + os: windows + arch: amd64 + version: 1809 + +steps: +- name: identify-runner + image: mcr.microsoft.com/windows:1809 + commands: + - echo $env:DRONE_RUNNER_NAME + +- name: initialize + image: grafana/ci-wix:0.1.1 + commands: + - $$ProgressPreference = "SilentlyContinue" + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.5.10/windows/grabpl.exe -OutFile grabpl.exe + +trigger: + event: + - pull_request + +--- +kind: pipeline +type: docker +name: build-master platform: os: linux @@ -240,7 +265,7 @@ steps: - name: initialize image: grafana/build-container:1.2.27 commands: - - curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v$${GRABPL_VERSION}/grabpl + - curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.5.10/grabpl - chmod +x grabpl - mkdir -p bin - mv grabpl bin @@ -250,7 +275,18 @@ steps: - yarn install --frozen-lockfile --no-progress environment: DOCKERIZE_VERSION: 0.6.1 - GRABPL_VERSION: 0.5.9 + +- name: trigger-enterprise-downstream + image: grafana/drone-downstream + settings: + params: + - SOURCE_BUILD_NUMBER=${DRONE_BUILD_NUMBER} + - SOURCE_COMMIT=${DRONE_COMMIT} + repositories: + - grafana/grafana-enterprise + server: https://drone.grafana.net + token: + from_secret: drone_token - name: lint-backend image: grafana/build-container:1.2.27 @@ -318,7 +354,7 @@ steps: - name: build-backend image: grafana/build-container:1.2.27 commands: - - ./bin/grabpl build-backend --jobs 8 --edition oss --build-id $DRONE_BUILD_NUMBER --no-pull-enterprise + - ./bin/grabpl build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --no-pull-enterprise depends_on: - initialize - lint-backend @@ -327,7 +363,7 @@ steps: - name: build-frontend image: grafana/build-container:1.2.27 commands: - - ./bin/grabpl build-frontend --jobs 8 --no-install-deps --edition oss --build-id $DRONE_BUILD_NUMBER --no-pull-enterprise + - ./bin/grabpl build-frontend --jobs 8 --no-install-deps --edition oss --build-id ${DRONE_BUILD_NUMBER} --no-pull-enterprise depends_on: - initialize - test-frontend @@ -335,7 +371,10 @@ steps: - name: build-plugins image: grafana/build-container:1.2.27 commands: - - ./bin/grabpl build-plugins --jobs 8 --edition oss --no-install-deps + - ./bin/grabpl build-plugins --jobs 8 --edition oss --no-install-deps --sign --signing-admin + environment: + GRAFANA_API_KEY: + from_secret: grafana_api_key depends_on: - initialize - lint-backend @@ -343,7 +382,16 @@ steps: - name: package image: grafana/build-container:1.2.27 commands: - - . scripts/build/gpg-test-vars.sh && ./bin/grabpl package --jobs 8 --edition oss --build-id $DRONE_BUILD_NUMBER --no-pull-enterprise + - ./bin/grabpl package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --no-pull-enterprise --sign + environment: + GPG_KEY_PASSWORD: + from_secret: gpg_key_password + GPG_PRIV_KEY: + from_secret: gpg_priv_key + GPG_PUB_KEY: + from_secret: gpg_pub_key + GRAFANA_API_KEY: + from_secret: grafana_api_key depends_on: - build-backend - build-frontend @@ -383,7 +431,7 @@ steps: commands: - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - echo gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://grafana-storybook/canary + - gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://grafana-storybook/canary environment: GCP_KEY: from_secret: gcp_key @@ -407,19 +455,25 @@ steps: - package - name: build-docker-images - image: grafana/drone-grafana-docker:0.2.0 + image: grafana/drone-grafana-docker:0.3.1 settings: - dry_run: true edition: oss + password: + from_secret: docker_password + username: + from_secret: docker_user depends_on: - copy-packages-for-docker - name: build-docker-images-ubuntu - image: grafana/drone-grafana-docker:0.2.0 + image: grafana/drone-grafana-docker:0.3.1 settings: - dry_run: true edition: oss + password: + from_secret: docker_password ubuntu: true + username: + from_secret: docker_user depends_on: - copy-packages-for-docker @@ -461,7 +515,7 @@ steps: commands: - npx lerna bootstrap - echo "//registry.npmjs.org/:_authToken=$${NPM_TOKEN}" >> ~/.npmrc - - echo ./scripts/circle-release-next-packages.sh + - ./scripts/circle-release-next-packages.sh environment: NPM_TOKEN: from_secret: npm_token @@ -471,7 +525,12 @@ steps: - name: publish-packages image: grafana/grafana-ci-deploy:1.2.6 commands: - - echo ./bin/grabpl publish-packages --edition oss + - ./bin/grabpl publish-packages --edition oss + environment: + GCP_GRAFANA_UPLOAD_KEY: + from_secret: gcp_key + GRAFANA_COM_API_KEY: + from_secret: grafana_api_key depends_on: - package - end-to-end-tests @@ -503,7 +562,7 @@ trigger: --- kind: pipeline type: docker -name: windows-installer-master +name: windows-master platform: os: windows @@ -514,7 +573,13 @@ steps: - name: identify-runner image: mcr.microsoft.com/windows:1809 commands: - - echo $Env:DRONE_RUNNER_NAME + - echo $env:DRONE_RUNNER_NAME + +- name: initialize + image: grafana/ci-wix:0.1.1 + commands: + - $$ProgressPreference = "SilentlyContinue" + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.5.10/windows/grabpl.exe -OutFile grabpl.exe - name: build-windows-installer image: grafana/ci-wix:0.1.1 @@ -524,17 +589,16 @@ steps: - dos2unix gcpkey.json - gcloud auth activate-service-account --key-file=gcpkey.json - rm gcpkey.json - - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.5.9/windows/grabpl.exe -OutFile grabpl.exe - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/oss/master/grafana-7.2.0-9fffe273pre.windows-amd64.zip -OutFile grafana.zip - cp C:\App\nssm-2.24.zip . - - ./grabpl.exe windows-installer --edition oss grafana.zip + - .\grabpl.exe windows-installer --edition oss --build-id $$env:DRONE_BUILD_NUMBER - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - echo "gsutil cp $$fname gs://grafana-downloads/oss/master/" - - echo "gsutil cp $$fname.sha256 gs://grafana-downloads/oss/master/" + - gsutil cp $$fname gs://grafana-downloads/oss/master/ + - gsutil cp $$fname.sha256 gs://grafana-downloads/oss/master/ environment: GCP_KEY: from_secret: gcp_key + depends_on: + - initialize trigger: branch: @@ -543,6 +607,6 @@ trigger: - push depends_on: -- test-master +- build-master ... diff --git a/scripts/lib.star b/scripts/lib.star index bc2e4067762..aeadfb1a33a 100644 --- a/scripts/lib.star +++ b/scripts/lib.star @@ -1,9 +1,12 @@ build_image = 'grafana/build-container:1.2.27' publish_image = 'grafana/grafana-ci-deploy:1.2.6' -grafana_docker_image = 'grafana/drone-grafana-docker:0.2.0' +grafana_docker_image = 'grafana/drone-grafana-docker:0.3.1' alpine_image = 'alpine:3.12' windows_image = 'mcr.microsoft.com/windows:1809' -grabpl_version = '0.5.9' +grabpl_version = '0.5.10' +git_image = 'alpine/git:v2.26.2' +dockerize_version = '0.6.1' +wix_image = 'grafana/ci-wix:0.1.1' def pr_pipelines(edition): services = [ @@ -47,14 +50,51 @@ def pr_pipelines(edition): postgres_integration_tests_step(), mysql_integration_tests_step(), ] + windows_steps = get_windows_steps(edition=edition, version_mode='pr') + trigger = { + 'event': ['pull_request',], + } return [ pipeline( - name='test-pr', edition=edition, trigger={ - 'event': ['pull_request',], - }, services=services, steps=steps + name='test-pr', edition=edition, trigger=trigger, services=services, steps=steps + ), + pipeline( + name='windows-pr', edition=edition, trigger=trigger, steps=windows_steps, platform='windows', ), ] +def master_steps(edition, is_downstream=False): + publish = edition != 'enterprise' or is_downstream + steps = [ + enterprise_downstream_step(edition), + lint_backend_step(edition), + codespell_step(), + shellcheck_step(), + test_backend_step(), + test_frontend_step(), + frontend_metrics_step(edition=edition), + build_backend_step(edition=edition, is_downstream=is_downstream), + build_frontend_step(edition=edition, is_downstream=is_downstream), + build_plugins_step(edition=edition, sign=True), + package_step(edition=edition, sign=True, is_downstream=is_downstream), + e2e_tests_server_step(), + e2e_tests_step(), + build_storybook_step(edition=edition), + publish_storybook_step(edition=edition), + build_docs_website_step(), + copy_packages_for_docker_step(), + build_docker_images_step(edition=edition, publish=publish), + build_docker_images_step(edition=edition, ubuntu=True, publish=publish), + postgres_integration_tests_step(), + mysql_integration_tests_step(), + release_next_npm_packages_step(edition), + publish_packages_step(edition, is_downstream), + deploy_to_kubernetes_step(edition, is_downstream), + ] + windows_steps = get_windows_steps(edition=edition, version_mode='master', is_downstream=is_downstream) + + return steps, windows_steps + def master_pipelines(edition): services = [ { @@ -77,49 +117,38 @@ def master_pipelines(edition): }, }, ] - steps = [ - lint_backend_step(edition), - codespell_step(), - shellcheck_step(), - test_backend_step(), - test_frontend_step(), - frontend_metrics_step(), - build_backend_step(edition=edition), - build_frontend_step(edition=edition), - build_plugins_step(edition=edition), - package_step(edition=edition), - e2e_tests_server_step(), - e2e_tests_step(), - build_storybook_step(edition=edition), - publish_storybook_step(edition=edition), - build_docs_website_step(), - copy_packages_for_docker_step(), - build_docker_images_step(edition=edition), - build_docker_images_step(edition=edition, ubuntu=True), - postgres_integration_tests_step(), - mysql_integration_tests_step(), - release_next_npm_packages_step(edition), - publish_packages_step(edition), - deploy_to_kubernetes_step(edition), - ] - windows_steps = [ - windows_installer_step(edition), - ] trigger = { 'event': ['push',], 'branch': 'master', } - return [ + steps, windows_steps = master_steps(edition=edition) + pipelines = [ pipeline( - name='test-master', edition=edition, trigger=trigger, services=services, steps=steps + name='build-master', edition=edition, trigger=trigger, services=services, steps=steps ), pipeline( - name='windows-installer-master', edition=edition, trigger=trigger, - steps=windows_steps, platform='windows', depends_on=['test-master'], + name='windows-master', edition=edition, trigger=trigger, steps=windows_steps, platform='windows', + depends_on=['build-master'], ), ] + if edition == 'enterprise': + # Add downstream enterprise pipelines triggerable from OSS builds + trigger = { + 'event': ['custom',], + } + steps, windows_steps = master_steps(edition=edition, is_downstream=True) + pipelines.append(pipeline( + name='build-master-downstream', edition=edition, trigger=trigger, services=services, steps=steps, + is_downstream=True, + )) + pipelines.append(pipeline( + name='windows-master-downstream', edition=edition, trigger=trigger, steps=windows_steps, + platform='windows', depends_on=['build-master-downstream'], is_downstream=True, + )) -def pipeline(name, edition, trigger, steps, services=[], platform='linux', depends_on=[]): + return pipelines + +def pipeline(name, edition, trigger, steps, services=[], platform='linux', depends_on=[], is_downstream=False): if platform != 'windows': platform_conf = { 'os': 'linux', @@ -139,7 +168,7 @@ def pipeline(name, edition, trigger, steps, services=[], platform='linux', depen 'name': name, 'trigger': trigger, 'services': services, - 'steps': init_steps(edition, platform) + steps, + 'steps': init_steps(edition, platform, is_downstream=is_downstream) + steps, 'depends_on': depends_on, } @@ -151,14 +180,14 @@ def pipeline(name, edition, trigger, steps, services=[], platform='linux', depen return pipeline -def init_steps(edition, platform): +def init_steps(edition, platform, is_downstream=False): if platform == 'windows': return [ { 'name': 'identify-runner', 'image': windows_image, 'commands': [ - 'echo $Env:DRONE_RUNNER_NAME', + 'echo $env:DRONE_RUNNER_NAME', ], }, ] @@ -178,11 +207,15 @@ def init_steps(edition, platform): 'yarn install --frozen-lockfile --no-progress', ] if edition == 'enterprise': + if is_downstream: + source_commit = ' $${SOURCE_COMMIT}' + else: + source_commit = '' return [ identify_runner_step, { 'name': 'clone', - 'image': 'alpine/git:v2.26.2', + 'image': git_image, 'environment': { 'GITHUB_TOKEN': { 'from_secret': 'github_token', @@ -198,18 +231,19 @@ def init_steps(edition, platform): 'name': 'initialize', 'image': build_image, 'environment': { - 'GRABPL_VERSION': grabpl_version, - 'DOCKERIZE_VERSION': '0.6.1', + 'DOCKERIZE_VERSION': dockerize_version, }, 'depends_on': [ 'clone', ], 'commands': [ - 'curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v$${GRABPL_VERSION}/grabpl', + 'curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/grabpl'.format( + grabpl_version + ), 'chmod +x grabpl', 'mv grabpl /tmp', 'mv grafana-enterprise /tmp/', - '/tmp/grabpl init-enterprise /tmp/grafana-enterprise', + '/tmp/grabpl init-enterprise /tmp/grafana-enterprise{}'.format(source_commit), 'mkdir bin', 'mv /tmp/grabpl bin/' ] + common_cmds, @@ -222,11 +256,12 @@ def init_steps(edition, platform): 'name': 'initialize', 'image': build_image, 'environment': { - 'GRABPL_VERSION': grabpl_version, - 'DOCKERIZE_VERSION': '0.6.1', + 'DOCKERIZE_VERSION': dockerize_version, }, 'commands': [ - 'curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v$${GRABPL_VERSION}/grabpl', + 'curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/grabpl'.format( + grabpl_version + ), 'chmod +x grabpl', 'mkdir -p bin', 'mv grabpl bin', @@ -234,6 +269,28 @@ def init_steps(edition, platform): }, ] +def enterprise_downstream_step(edition): + if edition == 'enterprise': + return None + + return { + 'name': 'trigger-enterprise-downstream', + 'image': 'grafana/drone-downstream', + 'settings': { + 'server': 'https://drone.grafana.net', + 'token': { + 'from_secret': 'drone_token', + }, + 'repositories': [ + 'grafana/grafana-enterprise', + ], + 'params': [ + 'SOURCE_BUILD_NUMBER=${DRONE_BUILD_NUMBER}', + 'SOURCE_COMMIT=${DRONE_COMMIT}', + ], + }, + } + def lint_backend_step(edition): return { 'name': 'lint-backend', @@ -285,11 +342,15 @@ def publish_storybook_step(edition): 'commands': [ 'printenv GCP_KEY | base64 -d > /tmp/gcpkey.json', 'gcloud auth activate-service-account --key-file=/tmp/gcpkey.json', - 'echo gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://grafana-storybook/canary', + 'gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://grafana-storybook/canary', ], } -def build_backend_step(edition, variants=None): +def build_backend_step(edition, variants=None, is_downstream=False): + if not is_downstream: + build_no = '${DRONE_BUILD_NUMBER}' + else: + build_no = '$${SOURCE_BUILD_NUMBER}' if variants: variants_str = ' --variants {}'.format(','.join(variants)) else: @@ -304,13 +365,17 @@ def build_backend_step(edition, variants=None): ], 'commands': [ # TODO: Convert number of jobs to percentage - './bin/grabpl build-backend --jobs 8 --edition {} --build-id $DRONE_BUILD_NUMBER{} --no-pull-enterprise'.format( - edition, variants_str + './bin/grabpl build-backend --jobs 8 --edition {} --build-id {}{} --no-pull-enterprise'.format( + edition, build_no, variants_str, ), ], } -def build_frontend_step(edition): +def build_frontend_step(edition, is_downstream=False): + if not is_downstream: + build_no = '${DRONE_BUILD_NUMBER}' + else: + build_no = '$${SOURCE_BUILD_NUMBER}' return { 'name': 'build-frontend', 'image': build_image, @@ -321,11 +386,21 @@ def build_frontend_step(edition): 'commands': [ # TODO: Use percentage for num jobs './bin/grabpl build-frontend --jobs 8 --no-install-deps --edition {} '.format(edition) + - '--build-id $DRONE_BUILD_NUMBER --no-pull-enterprise', + '--build-id {} --no-pull-enterprise'.format(build_no), ], } -def build_plugins_step(edition): +def build_plugins_step(edition, sign=False): + if sign: + env = { + 'GRAFANA_API_KEY': { + 'from_secret': 'grafana_api_key', + }, + } + sign_args = ' --sign --signing-admin' + else: + env = None + sign_args = '' return { 'name': 'build-plugins', 'image': build_image, @@ -333,9 +408,10 @@ def build_plugins_step(edition): 'initialize', 'lint-backend', ], + 'environment': env, 'commands': [ # TODO: Use percentage for num jobs - './bin/grabpl build-plugins --jobs 8 --edition {} --no-install-deps'.format(edition), + './bin/grabpl build-plugins --jobs 8 --edition {} --no-install-deps{}'.format(edition, sign_args), ], } @@ -370,7 +446,10 @@ def test_frontend_step(): ], } -def frontend_metrics_step(): +def frontend_metrics_step(edition): + if edition == 'enterprise': + return None + return { 'name': 'frontend-metrics', 'image': build_image, @@ -425,11 +504,37 @@ def shellcheck_step(): ], } -def package_step(edition, variants=None): +def package_step(edition, variants=None, sign=False, is_downstream=False): + if not is_downstream: + build_no = '${DRONE_BUILD_NUMBER}' + else: + build_no = '$${SOURCE_BUILD_NUMBER}' if variants: variants_str = ' --variants {}'.format(','.join(variants)) else: variants_str = '' + if sign: + sign_args = ' --sign' + env = { + 'GRAFANA_API_KEY': { + 'from_secret': 'grafana_api_key', + }, + 'GPG_PRIV_KEY': { + 'from_secret': 'gpg_priv_key', + }, + 'GPG_PUB_KEY': { + 'from_secret': 'gpg_pub_key', + }, + 'GPG_KEY_PASSWORD': { + 'from_secret': 'gpg_key_password', + }, + } + test_args = '' + else: + sign_args = '' + env = None + test_args = '. scripts/build/gpg-test-vars.sh && ' + return { 'name': 'package', 'image': build_image, @@ -442,10 +547,11 @@ def package_step(edition, variants=None): 'codespell', 'shellcheck', ], + 'environment': env, 'commands': [ # TODO: Use percentage for jobs - '. scripts/build/gpg-test-vars.sh && ./bin/grabpl package --jobs 8 --edition {} '.format(edition) + - '--build-id $DRONE_BUILD_NUMBER --no-pull-enterprise' + variants_str, + '{}./bin/grabpl package --jobs 8 --edition {} '.format(test_args, edition) + + '--build-id {} --no-pull-enterprise{}{}'.format(build_no, variants_str, sign_args), ], } @@ -507,15 +613,22 @@ def copy_packages_for_docker_step(): ], } -def build_docker_images_step(edition, archs=None, ubuntu=False): +def build_docker_images_step(edition, archs=None, ubuntu=False, publish=False): sfx = '' if ubuntu: sfx = '-ubuntu' settings = { - 'dry_run': True, + 'dry_run': not publish, 'edition': edition, 'ubuntu': ubuntu, } + if publish: + settings['username'] = { + 'from_secret': 'docker_user', + } + settings['password'] = { + 'from_secret': 'docker_password', + } if archs: settings['archs'] = ','.join(archs) return { @@ -593,13 +706,12 @@ def release_next_npm_packages_step(edition): 'commands': [ 'npx lerna bootstrap', 'echo "//registry.npmjs.org/:_authToken=$${NPM_TOKEN}" >> ~/.npmrc', - # TODO: Enable - 'echo ./scripts/circle-release-next-packages.sh', + './scripts/circle-release-next-packages.sh', ], } -def deploy_to_kubernetes_step(edition): - if edition != 'enterprise': +def deploy_to_kubernetes_step(edition, is_downstream): + if edition != 'enterprise' or not is_downstream: return None return { @@ -608,13 +720,20 @@ def deploy_to_kubernetes_step(edition): 'depends_on': [ 'build-docker-images', ], + 'environment': { + 'CIRCLE_TOKEN': { + 'from_secret': 'deployment_tools_circle_token', + }, + }, 'commands': [ - # TODO: Enable - 'echo ./bin/grabpl deploy-to-k8s', + './bin/grabpl deploy-to-k8s', ], } -def publish_packages_step(edition): +def publish_packages_step(edition, is_downstream): + if edition == 'enterprise' and not is_downstream: + return None + return { 'name': 'publish-packages', 'image': publish_image, @@ -624,39 +743,104 @@ def publish_packages_step(edition): 'mysql-integration-tests', 'postgres-integration-tests', ], - 'commands': [ - # TODO: Enable - 'echo ./bin/grabpl publish-packages --edition {}'.format(edition), - ], - } - -def windows_installer_step(edition): - sfx = '' - if edition == 'enterprise': - sfx = '-enterprise' - return { - 'name': 'build-windows-installer', - 'image': 'grafana/ci-wix:0.1.1', 'environment': { - 'GCP_KEY': { + 'GCP_GRAFANA_UPLOAD_KEY': { 'from_secret': 'gcp_key', }, + 'GRAFANA_COM_API_KEY': { + 'from_secret': 'grafana_api_key', + }, }, 'commands': [ + './bin/grabpl publish-packages --edition {}'.format(edition), + ], + } + +def get_windows_steps(edition, version_mode, is_downstream=False): + if not is_downstream: + source_commit = '' + else: + source_commit = ' $$env:SOURCE_COMMIT' + + sfx = '' + if edition == 'enterprise': + sfx = '-enterprise' + if not is_downstream: + build_no = 'DRONE_BUILD_NUMBER' + else: + build_no = 'SOURCE_BUILD_NUMBER' + steps = [ + { + 'name': 'initialize', + 'image': wix_image, + 'commands': [ + '$$ProgressPreference = "SilentlyContinue"', + 'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/windows/grabpl.exe -OutFile grabpl.exe'.format(grabpl_version), + ], + }, + ] + if version_mode == 'master': + installer_commands = [ '$$gcpKey = $$env:GCP_KEY', '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json', # gcloud fails to read the file unless converted with dos2unix 'dos2unix gcpkey.json', 'gcloud auth activate-service-account --key-file=gcpkey.json', 'rm gcpkey.json', - '$$ProgressPreference = "SilentlyContinue"', - 'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/windows/grabpl.exe -OutFile grabpl.exe'.format(grabpl_version), - # TODO: Infer correct Grafana version - 'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/{}/master/grafana{}-7.2.0-9fffe273pre.windows-amd64.zip -OutFile grafana.zip'.format(edition, sfx), 'cp C:\\App\\nssm-2.24.zip .', - './grabpl.exe windows-installer --edition {} grafana.zip'.format(edition), - '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', - 'echo "gsutil cp $$fname gs://grafana-downloads/{}/master/"'.format(edition), - 'echo "gsutil cp $$fname.sha256 gs://grafana-downloads/{}/master/"'.format(edition), - ], - } + '.\\grabpl.exe windows-installer --edition {} --build-id $$env:{}'.format(edition, build_no), + ] + if edition != 'enterprise' or is_downstream: + installer_commands.extend([ + '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', + 'gsutil cp $$fname gs://grafana-downloads/{}/{}/'.format(edition, version_mode), + 'gsutil cp $$fname.sha256 gs://grafana-downloads/{}/{}/'.format(edition, version_mode), + ]) + steps.append({ + 'name': 'build-windows-installer', + 'image': wix_image, + 'environment': { + 'GCP_KEY': { + 'from_secret': 'gcp_key', + }, + }, + 'commands': installer_commands, + 'depends_on': [ + 'initialize', + ], + }) + + if edition == 'enterprise': + # For enterprise, we have to clone both OSS and enterprise and merge the latter into the former + clone_commands = [ + 'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"', + ] + if not is_downstream: + clone_commands.extend([ + 'cd grafana-enterprise', + 'git checkout $$env:DRONE_COMMIT', + ]) + steps.insert(0, { + 'name': 'clone', + 'image': wix_image, + 'environment': { + 'GITHUB_TOKEN': { + 'from_secret': 'github_token', + }, + }, + 'commands': clone_commands, + }) + steps[1]['depends_on'] = [ + 'clone', + ] + steps[1]['commands'].extend([ + # Need to move grafana-enterprise out of the way, so directory is empty and can be cloned into + 'cp -r grafana-enterprise C:\\App\\grafana-enterprise', + 'rm -r -force grafana-enterprise', + 'cp grabpl.exe C:\\App\\grabpl.exe', + 'rm -force grabpl.exe', + 'C:\\App\\grabpl.exe init-enterprise C:\\App\\grafana-enterprise{}'.format(source_commit), + 'cp C:\\App\\grabpl.exe grabpl.exe', + ]) + + return steps