From 79f49c964947d363a4117591df889f43052947fa Mon Sep 17 00:00:00 2001 From: Kevin Minehart Date: Tue, 23 May 2023 03:01:09 -0500 Subject: [PATCH] CI: Add release verify pipeline (#68756) --- .drone.star | 2 + .drone.yml | 98 +++++++++++++++++++++++++++-- .github/CODEOWNERS | 1 + scripts/drone/events/release.star | 42 +++++++++++++ scripts/drone/rgm.star | 48 ++++++++------ scripts/list-release-artifacts.sh | 100 ++++++++++++++++++++++++++++++ 6 files changed, 268 insertions(+), 23 deletions(-) create mode 100755 scripts/list-release-artifacts.sh diff --git a/.drone.star b/.drone.star index 6be79486b1f..bc9630209a3 100644 --- a/.drone.star +++ b/.drone.star @@ -19,6 +19,7 @@ load( "publish_artifacts_pipelines", "publish_npm_pipelines", "publish_packages_pipeline", + "verify_release_pipeline", ) load( "scripts/drone/rgm.star", @@ -59,6 +60,7 @@ def main(_ctx): publish_artifacts_pipelines("public") + publish_npm_pipelines() + publish_packages_pipeline() + + [verify_release_pipeline()] + rgm() + [windows_test_backend({ "event": ["promote"], diff --git a/.drone.yml b/.drone.yml index f71bf62cc82..92688f50f3f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -4447,13 +4447,60 @@ volumes: clone: retries: 3 depends_on: +- release-oss-build-e2e-publish +- release-enterprise-build-e2e-publish +- release-enterprise2-build-e2e-publish +- release-oss-windows +- release-enterprise-windows +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: verify-prerelease-assets +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - apt-get update && apt-get install -yq gettext + - printenv GCP_KEY | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - VERSION=${DRONE_TAG} ./scripts/list-release-artifacts.sh | xargs -n1 gsutil stat + depends_on: + - clone + environment: + BUCKET: + from_secret: prerelease_bucket + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: gsutil-stat +trigger: + event: + exclude: + - promote + ref: + exclude: + - refs/tags/*-cloud* + include: + - refs/tags/v* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: - main-test-backend - main-test-frontend image_pull_secrets: - dockerconfigjson kind: pipeline -name: '[RGM] Build and upload a grafana.tar.gz to a prerelease bucket when merging - to main' +name: rgm-main-prerelease node: type: no-parallel platform: @@ -4501,7 +4548,7 @@ depends_on: [] image_pull_secrets: - dockerconfigjson kind: pipeline -name: '[RGM] Build and upload a grafana.tar.gz to a prerelease bucket when tagging' +name: rgm-tag-prerelease node: type: no-parallel platform: @@ -4548,6 +4595,49 @@ volumes: path: /var/run/docker.sock name: docker --- +clone: + retries: 3 +depends_on: +- rgm-tag-prerelease +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-tag-verify-prerelease-assets +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - apt-get update && apt-get install -yq gettext + - printenv GCP_KEY | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - VERSION=${DRONE_TAG} ./scripts/list-release-artifacts.sh | xargs -n1 gsutil stat + depends_on: + - clone + environment: + BUCKET: grafana-prerelease-dev + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: gsutil-stat +trigger: + event: + exclude: + - promote + ref: + exclude: + - refs/tags/*-cloud* + include: + - refs/tags/v* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- clone: disable: true depends_on: [] @@ -7110,6 +7200,6 @@ kind: secret name: delivery-bot-app-private-key --- kind: signature -hmac: b672e3a75c9e262486985d34c31c246dd5dfb291324c53b99588a3da5f92abf5 +hmac: 6b3dd328b2c09e1580a9b5add265651af159349a07488796a937d61e1112a3fe ... diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 45d7522e50e..026393f14ba 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -229,6 +229,7 @@ /Dockerfile @grafana/grafana-delivery /Makefile @grafana/grafana-delivery /scripts/build/ @grafana/grafana-delivery +/scripts/list-release-artifacts.sh @grafana/grafana-delivery # OSS Plugin Partnerships backend code /pkg/tsdb/cloudwatch/ @grafana/aws-datasources diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index a5e3052343e..cc099af5542 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -793,3 +793,45 @@ def integration_test_pipelines(): )) return pipelines + +def verify_release_pipeline( + name = "verify-prerelease-assets", + bucket = from_secret(prerelease_bucket), + gcp_key = from_secret("gcp_key"), + version = "${DRONE_TAG}", + trigger = release_trigger, + depends_on = [ + "release-oss-build-e2e-publish", + "release-enterprise-build-e2e-publish", + "release-enterprise2-build-e2e-publish", + "release-oss-windows", + "release-enterprise-windows", + ]): + """ + Runs a script that 'gsutil stat's every artifact that should have been produced by the pre-release process. + + Returns: + A single Drone pipeline that runs the script. + """ + step = { + "name": "gsutil-stat", + "depends_on": ["clone"], + "image": "google/cloud-sdk", + "environment": { + "BUCKET": bucket, + "GCP_KEY": gcp_key, + }, + "commands": [ + "apt-get update && apt-get install -yq gettext", + "printenv GCP_KEY | base64 -d > /tmp/key.json", + "gcloud auth activate-service-account --key-file=/tmp/key.json", + "VERSION={} ./scripts/list-release-artifacts.sh | xargs -n1 gsutil stat".format(version), + ], + } + return pipeline( + depends_on = depends_on, + name = name, + edition = "all", + trigger = trigger, + steps = [step], + ) diff --git a/scripts/drone/rgm.star b/scripts/drone/rgm.star index e4323a43f10..8b3c3523e41 100644 --- a/scripts/drone/rgm.star +++ b/scripts/drone/rgm.star @@ -8,6 +8,10 @@ load( "scripts/drone/utils/utils.star", "pipeline", ) +load( + "scripts/drone/events/release.star", + "verify_release_pipeline", +) load( "scripts/drone/vault.star", "from_secret", @@ -64,34 +68,34 @@ def rgm_main(): } return pipeline( - name = "[RGM] Build and upload a grafana.tar.gz to a prerelease bucket when merging to main", + name = "rgm-main-prerelease", edition = "all", trigger = trigger, steps = rgm_build(), depends_on = ["main-test-backend", "main-test-frontend"], ) -def rgm_tag(): - trigger = { - "event": { - "exclude": [ - "promote", - ], - }, - "ref": { - "include": [ - "refs/tags/v*", - ], - "exclude": [ - "refs/tags/*-cloud*", - ], - }, - } +tag_trigger = { + "event": { + "exclude": [ + "promote", + ], + }, + "ref": { + "include": [ + "refs/tags/v*", + ], + "exclude": [ + "refs/tags/*-cloud*", + ], + }, +} +def rgm_tag(): return pipeline( - name = "[RGM] Build and upload a grafana.tar.gz to a prerelease bucket when tagging", + name = "rgm-tag-prerelease", edition = "all", - trigger = trigger, + trigger = tag_trigger, steps = rgm_build(script = "drone_publish_tag.sh"), depends_on = [], ) @@ -100,4 +104,10 @@ def rgm(): return [ rgm_main(), rgm_tag(), + verify_release_pipeline( + name = "rgm-tag-verify-prerelease-assets", + trigger = tag_trigger, + depends_on = ["rgm-tag-prerelease"], + bucket = "grafana-prerelease-dev", + ), ] diff --git a/scripts/list-release-artifacts.sh b/scripts/list-release-artifacts.sh new file mode 100755 index 00000000000..a5c3c4eebbc --- /dev/null +++ b/scripts/list-release-artifacts.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +set -e + +VERSION="${1:-v9.5.2}" +ERSION="${VERSION#*v}" + +ASSETS=$(cat << EOF +gs://${BUCKET}/artifacts/static-assets/grafana-oss/${ERSION}/public/robots.txt +gs://${BUCKET}/artifacts/static-assets/grafana/${ERSION}/public/robots.txt +gs://${BUCKET}/artifacts/static-assets/grafana-pro/${ERSION}/public/robots.txt +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}-1.aarch64.rpm +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}-1.aarch64.rpm.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}-1.armhfp.rpm +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}-1.armhfp.rpm.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}-1.x86_64.rpm +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}-1.x86_64.rpm.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.darwin-amd64.tar.gz +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.darwin-amd64.tar.gz.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.linux-amd64-musl.tar.gz +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.linux-amd64-musl.tar.gz.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.linux-amd64.tar.gz +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.linux-amd64.tar.gz.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.linux-arm64-musl.tar.gz +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.linux-arm64-musl.tar.gz.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.linux-arm64.tar.gz +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.linux-arm64.tar.gz.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.linux-armv6.tar.gz +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.linux-armv6.tar.gz.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.linux-armv7-musl.tar.gz +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.linux-armv7-musl.tar.gz.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.linux-armv7.tar.gz +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.linux-armv7.tar.gz.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.windows-amd64.msi +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.windows-amd64.msi.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.windows-amd64.zip +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-${ERSION}.windows-amd64.zip.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-rpi_${ERSION}_armhf.deb +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana-rpi_${ERSION}_armhf.deb.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana_${ERSION}_amd64.deb +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana_${ERSION}_amd64.deb.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana_${ERSION}_arm64.deb +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana_${ERSION}_arm64.deb.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana_${ERSION}_armhf.deb +gs://${BUCKET}/artifacts/downloads/${VERSION}/oss/release/grafana_${ERSION}_armhf.deb.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}-1.aarch64.rpm +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}-1.aarch64.rpm.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}-1.armhfp.rpm +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}-1.armhfp.rpm.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}-1.x86_64.rpm +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}-1.x86_64.rpm.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.darwin-amd64.tar.gz +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.darwin-amd64.tar.gz.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.linux-amd64-musl.tar.gz +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.linux-amd64-musl.tar.gz.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.linux-amd64.tar.gz +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.linux-amd64.tar.gz.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.linux-arm64-musl.tar.gz +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.linux-arm64-musl.tar.gz.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.linux-arm64.tar.gz +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.linux-arm64.tar.gz.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.linux-armv6.tar.gz +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.linux-armv6.tar.gz.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.linux-armv7-musl.tar.gz +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.linux-armv7-musl.tar.gz.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.linux-armv7.tar.gz +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.linux-armv7.tar.gz.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.windows-amd64.msi +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.windows-amd64.msi.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.windows-amd64.zip +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-${ERSION}.windows-amd64.zip.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-rpi_${ERSION}_armhf.deb +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise-rpi_${ERSION}_armhf.deb.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise_${ERSION}_amd64.deb +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise_${ERSION}_amd64.deb.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise_${ERSION}_arm64.deb +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise_${ERSION}_arm64.deb.sha256 +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise_${ERSION}_armhf.deb +gs://${BUCKET}/artifacts/downloads/${VERSION}/enterprise/release/grafana-enterprise_${ERSION}_armhf.deb.sha256 +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-enterprise-${ERSION}-amd64.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-enterprise-${ERSION}-arm64.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-enterprise-${ERSION}-armv7.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-enterprise-${ERSION}-ubuntu-amd64.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-enterprise-${ERSION}-ubuntu-arm64.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-enterprise-${ERSION}-ubuntu-armv7.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-enterprise2-${ERSION}-amd64.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-enterprise2-${ERSION}-arm64.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-enterprise2-${ERSION}-armv7.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-enterprise2-${ERSION}-ubuntu-amd64.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-enterprise2-${ERSION}-ubuntu-arm64.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-enterprise2-${ERSION}-ubuntu-armv7.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-oss-${ERSION}-amd64.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-oss-${ERSION}-arm64.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-oss-${ERSION}-armv7.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-oss-${ERSION}-ubuntu-amd64.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-oss-${ERSION}-ubuntu-arm64.img +gs://${BUCKET}/artifacts/docker/${ERSION}/grafana-oss-${ERSION}-ubuntu-armv7.img +EOF +) + +echo "${ASSETS}" | envsubst