From 6217739bc9e5fa34bb01cbd65ee063c10a5d8965 Mon Sep 17 00:00:00 2001 From: Dimitris Sotirakis Date: Wed, 12 Jan 2022 11:40:11 +0200 Subject: [PATCH] CI: Automate Grafana releases (#43897) (#43911) * CI: Automate releases * Update grabpl version (cherry picked from commit 4f47832bac63142865dbf4279808cd1e5a45294f) --- .drone.star | 5 +- .drone.yml | 536 ++++++++++++++++++++++++++- scripts/drone/pipelines/release.star | 180 +++++++-- scripts/drone/steps/lib.star | 6 +- scripts/drone/utils/utils.star | 7 +- 5 files changed, 688 insertions(+), 46 deletions(-) diff --git a/.drone.star b/.drone.star index 171a65a15b2..a90a3027b23 100644 --- a/.drone.star +++ b/.drone.star @@ -6,7 +6,7 @@ load('scripts/drone/pipelines/pr.star', 'pr_pipelines') load('scripts/drone/pipelines/main.star', 'main_pipelines') -load('scripts/drone/pipelines/release.star', 'release_pipelines', 'test_release_pipelines') +load('scripts/drone/pipelines/release.star', 'release_pipelines', 'test_release_pipelines', 'publish_image_pipelines', 'publish_artifacts_pipelines', 'publish_npm_pipelines', 'publish_packages_pipeline') load('scripts/drone/version.star', 'version_branch_pipelines') load('scripts/drone/pipelines/cron.star', 'cronjobs') load('scripts/drone/vault.star', 'secrets') @@ -14,4 +14,7 @@ load('scripts/drone/vault.star', 'secrets') def main(ctx): edition = 'oss' return pr_pipelines(edition=edition) + main_pipelines(edition=edition) + release_pipelines() + \ + publish_image_pipelines('public') + publish_image_pipelines('security') + \ + publish_artifacts_pipelines('security') + publish_artifacts_pipelines('public') + \ + publish_npm_pipelines('public') + publish_packages_pipeline() + \ test_release_pipelines() + version_branch_pipelines() + cronjobs(edition=edition) + secrets() diff --git a/.drone.yml b/.drone.yml index 87ee2b8614e..08ba8c2e3ec 100644 --- a/.drone.yml +++ b/.drone.yml @@ -932,7 +932,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/vv2.8.1/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.1/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: initialize @@ -1004,7 +1004,7 @@ steps: - ./bin/grabpl store-packages --edition oss --gcp-key /tmp/gcpkey.json --build-id ${DRONE_BUILD_NUMBER} depends_on: - - initialize + - grabpl environment: GCP_KEY: from_secret: gcp_key @@ -1318,7 +1318,7 @@ steps: image: grafana/build-container:1.4.9 name: build-npm-packages - commands: - - ./scripts/build/store-npm-packages.sh ${DRONE_TAG} + - ./bin/grabpl artifacts npm store --tag ${DRONE_TAG} depends_on: - build-npm-packages environment: @@ -1329,6 +1329,9 @@ steps: image: grafana/grafana-ci-deploy:1.3.1 name: store-npm-packages trigger: + event: + exclude: + - promote ref: - refs/tags/v* repo: @@ -1434,6 +1437,9 @@ steps: image: grafana/build-container:1.4.9 name: test-frontend trigger: + event: + exclude: + - promote ref: - refs/tags/v* repo: @@ -1525,6 +1531,9 @@ steps: image: grafana/build-container:1.4.9 name: mysql-integration-tests trigger: + event: + exclude: + - promote ref: - refs/tags/v* repo: @@ -1562,7 +1571,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/vv2.8.1/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.1/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: initialize @@ -1590,6 +1599,9 @@ steps: image: grafana/ci-wix:0.1.1 name: build-windows-installer trigger: + event: + exclude: + - promote ref: - refs/tags/v* repo: @@ -1908,6 +1920,9 @@ steps: image: grafana/grafana-ci-deploy:1.3.1 name: upload-packages-enterprise2 trigger: + event: + exclude: + - promote ref: - refs/tags/v* repo: @@ -2059,6 +2074,9 @@ steps: image: grafana/build-container:1.4.9 name: test-backend-integration-enterprise2 trigger: + event: + exclude: + - promote ref: - refs/tags/v* repo: @@ -2200,6 +2218,9 @@ steps: image: grafana/build-container:1.4.9 name: memcached-integration-tests trigger: + event: + exclude: + - promote ref: - refs/tags/v* repo: @@ -2241,7 +2262,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/vv2.8.1/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.1/windows/grabpl.exe -OutFile grabpl.exe - git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git" - cd grafana-enterprise @@ -2289,6 +2310,9 @@ steps: image: grafana/ci-wix:0.1.1 name: build-windows-installer trigger: + event: + exclude: + - promote ref: - refs/tags/v* repo: @@ -2304,6 +2328,494 @@ volumes: --- depends_on: [] kind: pipeline +name: publish-docker-oss-public +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.1/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - ./bin/grabpl artifacts docker fetch --version-tag ${TAG} --edition oss --base + alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7 + depends_on: + - grabpl + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: fetch-images-oss + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/grabpl artifacts docker publish --version-tag ${TAG} --dockerhub-repo grafana + --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7 + depends_on: + - fetch-images-oss + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: publish-images-grafana + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/grabpl artifacts docker publish --version-tag ${TAG} --dockerhub-repo grafana-oss + --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7 + depends_on: + - fetch-images-oss + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: publish-images-grafana-oss + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + event: + - promote + target: + - public +type: docker +volumes: +- name: cypress_cache + temp: {} +- host: + path: /var/run/docker.sock + name: docker +--- +depends_on: [] +kind: pipeline +name: publish-docker-enterprise-public +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.1/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - ./bin/grabpl artifacts docker fetch --version-tag ${TAG} --edition enterprise + --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7 + depends_on: + - grabpl + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: fetch-images-enterprise + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/grabpl artifacts docker publish --version-tag ${TAG} --dockerhub-repo grafana-enterprise + --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7 + depends_on: + - fetch-images-enterprise + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: publish-images-grafana-enterprise + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + event: + - promote + target: + - public +type: docker +volumes: +- name: cypress_cache + temp: {} +- host: + path: /var/run/docker.sock + name: docker +--- +depends_on: [] +kind: pipeline +name: publish-docker-oss-security +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.1/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - ./bin/grabpl artifacts docker fetch --version-tag ${TAG} --edition oss --base + alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7 + depends_on: + - grabpl + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: fetch-images-oss + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/grabpl artifacts docker publish --security --version-tag ${TAG} --dockerhub-repo + grafana --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7 + depends_on: + - fetch-images-oss + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: publish-images-grafana + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/grabpl artifacts docker publish --security --version-tag ${TAG} --dockerhub-repo + grafana-oss --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7 + depends_on: + - fetch-images-oss + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: publish-images-grafana-oss + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + event: + - promote + target: + - security +type: docker +volumes: +- name: cypress_cache + temp: {} +- host: + path: /var/run/docker.sock + name: docker +--- +depends_on: [] +kind: pipeline +name: publish-docker-enterprise-security +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.1/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - ./bin/grabpl artifacts docker fetch --version-tag ${TAG} --edition enterprise + --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7 + depends_on: + - grabpl + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: fetch-images-enterprise + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/grabpl artifacts docker publish --security --version-tag ${TAG} --dockerhub-repo + grafana-enterprise --base alpine --base ubuntu --arch amd64 --arch arm64 --arch + armv7 + depends_on: + - fetch-images-enterprise + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: publish-images-grafana-enterprise + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + event: + - promote + target: + - security +type: docker +volumes: +- name: cypress_cache + temp: {} +- host: + path: /var/run/docker.sock + name: docker +--- +depends_on: [] +kind: pipeline +name: publish-artifacts-security +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.1/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - ./bin/grabpl artifacts publish --security --tag ${TAG} --src-bucket grafana-prerelease + depends_on: + - grabpl + environment: + GCP_KEY: + from_secret: gcp_key + image: grafana/grafana-ci-deploy:1.3.1 + name: publish-artifacts +trigger: + event: + - promote + target: + - security +type: docker +volumes: +- name: cypress_cache + temp: {} +- host: + path: /var/run/docker.sock + name: docker +--- +depends_on: [] +kind: pipeline +name: publish-artifacts-public +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.1/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - ./bin/grabpl artifacts publish --tag ${TAG} --src-bucket grafana-prerelease + depends_on: + - grabpl + environment: + GCP_KEY: + from_secret: gcp_key + image: grafana/grafana-ci-deploy:1.3.1 + name: publish-artifacts +trigger: + event: + - promote + target: + - public +type: docker +volumes: +- name: cypress_cache + temp: {} +- host: + path: /var/run/docker.sock + name: docker +--- +depends_on: [] +kind: pipeline +name: publish-npm-packages-public +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner +- commands: + - make gen-go + - ./bin/grabpl verify-version ${DRONE_TAG} + - ./bin/grabpl gen-version ${DRONE_TAG} + - yarn install --immutable + image: grafana/build-container:1.4.9 + name: initialize +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.1/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - ./bin/grabpl artifacts npm retrieve --tag v${TAG} + depends_on: + - initialize + environment: + GCP_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.1 + name: retrieve-npm-packages +- commands: + - ./bin/grabpl artifacts npm release --tag v${TAG} + depends_on: + - retrieve-npm-packages + environment: + NPM_TOKEN: + from_secret: npm_token + image: grafana/build-container:1.4.9 + name: release-npm-packages +trigger: + event: + - promote + target: + - public +type: docker +volumes: +- name: cypress_cache + temp: {} +- host: + path: /var/run/docker.sock + name: docker +--- +depends_on: [] +kind: pipeline +name: publish-packages +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.1/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json + - ./bin/grabpl store-packages --edition oss --gcp-key /tmp/gcpkey.json ${DRONE_TAG} + depends_on: + - grabpl + environment: + GCP_KEY: + from_secret: gcp_key + GPG_KEY_PASSWORD: + from_secret: gpg_key_password + GPG_PRIV_KEY: + from_secret: gpg_priv_key + GPG_PUB_KEY: + from_secret: gpg_pub_key + GRAFANA_COM_API_KEY: + from_secret: grafana_api_key + image: grafana/grafana-ci-deploy:1.3.1 + name: store-packages-oss +- commands: + - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json + - ./bin/grabpl store-packages --edition enterprise --gcp-key /tmp/gcpkey.json ${DRONE_TAG} + depends_on: + - grabpl + environment: + GCP_KEY: + from_secret: gcp_key + GPG_KEY_PASSWORD: + from_secret: gpg_key_password + GPG_PRIV_KEY: + from_secret: gpg_priv_key + GPG_PUB_KEY: + from_secret: gpg_pub_key + GRAFANA_COM_API_KEY: + from_secret: grafana_api_key + image: grafana/grafana-ci-deploy:1.3.1 + name: store-packages-enterprise +trigger: + event: + - promote + target: + - public +type: docker +volumes: +- name: cypress_cache + temp: {} +- host: + path: /var/run/docker.sock + name: docker +--- +depends_on: [] +kind: pipeline name: oss-build-publish-e2e-test-release node: type: no-parallel @@ -2775,7 +3287,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/vv2.8.1/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.1/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: initialize @@ -3440,7 +3952,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/vv2.8.1/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.1/windows/grabpl.exe -OutFile grabpl.exe - git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git" - cd grafana-enterprise @@ -3539,7 +4051,7 @@ steps: grafana-downloads-test --rpm-repo-bucket grafana-testing-repo --simulate-release v7.3.0-test depends_on: - - initialize + - grabpl environment: GCP_KEY: from_secret: gcp_key @@ -3560,7 +4072,7 @@ steps: grafana-downloads-test --rpm-repo-bucket grafana-testing-repo --simulate-release v7.3.0-test depends_on: - - initialize + - grabpl environment: GCP_KEY: from_secret: gcp_key @@ -4071,7 +4583,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/vv2.8.1/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.1/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: initialize @@ -4733,7 +5245,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/vv2.8.1/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.1/windows/grabpl.exe -OutFile grabpl.exe - git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git" - cd grafana-enterprise @@ -4847,6 +5359,6 @@ kind: secret name: prerelease_bucket --- kind: signature -hmac: 9f44c7c54d3cd0602007482fadece0062d30e6851aae5b3617a0a5b0c27511aa +hmac: d8f724814ba00078777ee61a1457c76c9bba741d2d6953b94ef68d2ca0f00181 ... diff --git a/scripts/drone/pipelines/release.star b/scripts/drone/pipelines/release.star index 446b04783c1..fe8e75c7327 100644 --- a/scripts/drone/pipelines/release.star +++ b/scripts/drone/pipelines/release.star @@ -68,10 +68,7 @@ def build_npm_packages_step(edition, ver_mode): 'commands': ['./scripts/build/build-npm-packages.sh ${DRONE_TAG}'], } -def store_npm_packages_step(edition, ver_mode): - if edition == 'enterprise' or ver_mode != 'release': - return None - +def store_npm_packages_step(): return { 'name': 'store-npm-packages', 'image': publish_image, @@ -82,31 +79,28 @@ def store_npm_packages_step(edition, ver_mode): 'GCP_KEY': from_secret('gcp_key'), 'PRERELEASE_BUCKET': from_secret(prerelease_bucket) }, - 'commands': ['./scripts/build/store-npm-packages.sh ${DRONE_TAG}'], + 'commands': [ + './bin/grabpl artifacts npm store --tag ${DRONE_TAG}' + ], } -def retrieve_npm_packages_step(edition, ver_mode): - if edition == 'enterprise' or ver_mode != 'release': - return None - +def retrieve_npm_packages_step(): return { 'name': 'retrieve-npm-packages', 'image': publish_image, 'depends_on': [ - # Has to run after store-storybook since this step cleans the files publish-storybook depends on - 'store-storybook', + 'initialize', ], 'environment': { 'GCP_KEY': from_secret('gcp_key'), 'PRERELEASE_BUCKET': from_secret(prerelease_bucket) }, - 'commands': ['./scripts/build/retrieve-npm-packages.sh ${DRONE_TAG}'], + 'commands': [ + './bin/grabpl artifacts npm retrieve --tag v${TAG}' + ], } -def release_npm_packages_step(edition, ver_mode): - if edition == 'enterprise' or ver_mode != 'release': - return None - +def release_npm_packages_step(): return { 'name': 'release-npm-packages', 'image': build_image, @@ -116,9 +110,71 @@ def release_npm_packages_step(edition, ver_mode): 'environment': { 'NPM_TOKEN': from_secret('npm_token'), }, - 'commands': ['./scripts/build/release-npm-packages.sh ${DRONE_TAG}'], + 'commands': [ + './bin/grabpl artifacts npm release --tag v${TAG}' + ], } +def publish_images_step(edition, mode, docker_repo): + if mode == 'security': + mode = '--{} '.format(mode) + else: + mode = '' + return { + 'name': 'publish-images-{}'.format(docker_repo), + 'image': 'google/cloud-sdk', + 'environment': { + 'GCP_KEY': from_secret('gcp_key'), + 'DOCKER_USER': from_secret('docker_username'), + 'DOCKER_PASSWORD': from_secret('docker_password'), + }, + 'commands': ['./bin/grabpl artifacts docker publish {}--version-tag ${{TAG}} --dockerhub-repo {} --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7'.format(mode, docker_repo)], + 'depends_on': ['fetch-images-{}'.format(edition)], + 'volumes': [{ + 'name': 'docker', + 'path': '/var/run/docker.sock' + }], + } + +def fetch_images_step(edition): + return { + 'name': 'fetch-images-{}'.format(edition), + 'image': 'google/cloud-sdk', + 'environment': { + 'GCP_KEY': from_secret('gcp_key'), + 'DOCKER_USER': from_secret('docker_username'), + 'DOCKER_PASSWORD': from_secret('docker_password'), + }, + 'commands': ['./bin/grabpl artifacts docker fetch --version-tag ${{TAG}} --edition {} --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7'.format(edition)], + 'depends_on': ['grabpl'], + 'volumes': [{ + 'name': 'docker', + 'path': '/var/run/docker.sock' + }], + } + +def publish_image_steps(version, mode, docker_repo, additional_docker_repo=""): + steps = [ + download_grabpl_step(), + fetch_images_step(version), + publish_images_step(version, mode, docker_repo), + ] + if additional_docker_repo != "": + steps.extend([publish_images_step(version, mode, additional_docker_repo)]) + + return steps + +def publish_image_pipelines(mode): + trigger = { + 'event': ['promote'], + 'target': [mode], + } + + return [pipeline( + name='publish-docker-oss-{}'.format(mode), trigger=trigger, steps=publish_image_steps(version='oss', mode=mode, docker_repo='grafana', additional_docker_repo='grafana-oss'), edition="" + ), pipeline( + name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(version='enterprise', mode=mode, docker_repo='grafana-enterprise'), edition="" + ),] def get_steps(edition, ver_mode): package_steps = [] @@ -193,7 +249,7 @@ def get_steps(edition, ver_mode): if should_publish: publish_step = store_storybook_step(edition=edition, ver_mode=ver_mode) build_npm_step = build_npm_packages_step(edition=edition, ver_mode=ver_mode) - store_npm_step = store_npm_packages_step(edition=edition, ver_mode=ver_mode) + store_npm_step = store_npm_packages_step() if publish_step: publish_steps.append(publish_step) if build_npm_step and store_npm_step: @@ -308,11 +364,85 @@ def get_enterprise_pipelines(trigger, ver_mode): return pipelines -def release_pipelines(ver_mode='release', trigger=None): +def publish_artifacts_step(mode): + security = '' + if mode == 'security': + security = '--security ' + return { + 'name': 'publish-artifacts', + 'image': publish_image, + 'environment': { + 'GCP_KEY': from_secret('gcp_key'), + }, + 'commands': ['./bin/grabpl artifacts publish {}--tag ${{TAG}} --src-bucket grafana-prerelease'.format(security)], + 'depends_on': ['grabpl'], + } + +def publish_packages_step(edition): + return { + 'name': 'publish-packages-{}'.format(edition), + 'image': publish_image, + 'environment': { + 'GCP_KEY': from_secret('gcp_key'), + }, + 'commands': ['./bin/grabpl store-packages {}'.format(edition)], + 'depends_on': ['grabpl'], + } + +def publish_artifacts_pipelines(mode): + trigger = { + 'event': ['promote'], + 'target': [mode], + } + steps = [ + download_grabpl_step(), + publish_artifacts_step(mode), + ] + + return [pipeline( + name='publish-artifacts-{}'.format(mode), trigger=trigger, steps=steps, edition="all" + )] + +def publish_packages_pipeline(): + trigger = { + 'event': ['promote'], + 'target': ['public'], + } + steps = [ + download_grabpl_step(), + store_packages_step(edition='oss', ver_mode='release'), + store_packages_step(edition='enterprise', ver_mode='release'), + ] + + return [pipeline( + name='publish-packages', trigger=trigger, steps=steps, edition="all" + )] + +def publish_npm_pipelines(mode): + trigger = { + 'event': ['promote'], + 'target': [mode], + } + steps = [ + download_grabpl_step(), + retrieve_npm_packages_step(), + release_npm_packages_step() + ] + + return [pipeline( + name='publish-npm-packages-{}'.format(mode), trigger=trigger, steps = initialize_step(edition='oss', platform='linux', ver_mode='release') + steps, edition="all" + )] + +def release_pipelines(ver_mode='release', trigger=None, environment=None): # 'enterprise' edition services contain both OSS and enterprise services services = integration_test_services(edition='enterprise') if not trigger: trigger = { + 'event': { + 'exclude': [ + 'promote' + ] + }, 'ref': ['refs/tags/v*',], 'repo': { 'exclude': ['grafana/grafana'], @@ -328,17 +458,9 @@ def release_pipelines(ver_mode='release', trigger=None): enterprise_pipelines = get_enterprise_pipelines(ver_mode=ver_mode, trigger=trigger) pipelines = oss_pipelines + enterprise_pipelines - if should_publish: - steps = [ - store_packages_step(edition='oss', ver_mode=ver_mode), - store_packages_step(edition='enterprise', ver_mode=ver_mode), - ] - publish_pipeline = pipeline( - name='publish-{}'.format(ver_mode), trigger=trigger, edition='oss', - steps=[download_grabpl_step()] + initialize_step(edition='oss', platform='linux', ver_mode=ver_mode, install_deps=False) + steps, - depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines], - ) + # if ver_mode == 'release': + # pipelines.append(publish_artifacts_pipelines()) #pipelines.append(notify_pipeline( # name='notify-{}'.format(ver_mode), slack_channel='grafana-ci-notifications', trigger=trigger, # depends_on=[p['name'] for p in pipelines], secret='slack_webhook', diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 124fed197cc..3f5487def03 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -974,7 +974,7 @@ def store_packages_step(edition, ver_mode, is_downstream=False): 'name': 'store-packages-{}'.format(edition), 'image': publish_image, 'depends_on': [ - 'initialize', + 'grabpl', ], 'environment': { 'GRAFANA_COM_API_KEY': from_secret('grafana_api_key'), @@ -1003,7 +1003,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False): else: init_cmds.extend([ '$$ProgressPreference = "SilentlyContinue"', - 'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/windows/grabpl.exe -OutFile grabpl.exe'.format( + 'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format( grabpl_version), ]) steps = [ @@ -1077,7 +1077,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False): # For enterprise, we have to clone both OSS and enterprise and merge the latter into the former download_grabpl_step_cmds = [ '$$ProgressPreference = "SilentlyContinue"', - 'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/windows/grabpl.exe -OutFile grabpl.exe'.format( + 'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format( grabpl_version), ] clone_cmds = [ diff --git a/scripts/drone/utils/utils.star b/scripts/drone/utils/utils.star index 4edb9183e2d..dabc22f0974 100644 --- a/scripts/drone/utils/utils.star +++ b/scripts/drone/utils/utils.star @@ -7,7 +7,7 @@ load( load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token') def pipeline( - name, edition, trigger, steps, services=[], platform='linux', depends_on=[], volumes=[], + name, edition, trigger, steps, services=[], platform='linux', depends_on=[], environment=None, volumes=[], ): if platform != 'windows': platform_conf = { @@ -48,6 +48,11 @@ def pipeline( }], 'depends_on': depends_on, } + if environment: + pipeline.update({ + 'environment': environment, + }) + pipeline['volumes'].extend(volumes) pipeline.update(platform_conf)