From 9501000856e7f60e578a850a7b721a057b28db97 Mon Sep 17 00:00:00 2001 From: Kevin Minehart Date: Thu, 11 Jan 2024 09:22:14 -0600 Subject: [PATCH] CI: add rgm promotion pipeline (#80296) * add package upload promotion pipeline --- .drone.yml | 102 ++++++++++++++++++++++++++++++++++++++++- scripts/drone/rgm.star | 86 ++++++++++++++++++++++++++++------ 2 files changed, 174 insertions(+), 14 deletions(-) diff --git a/.drone.yml b/.drone.yml index fe5ecc6aa6d..a16b13c0104 100644 --- a/.drone.yml +++ b/.drone.yml @@ -3740,6 +3740,106 @@ volumes: path: /var/run/docker.sock name: docker --- +clone: + retries: 3 +depends_on: [] +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-promotion +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - 'dagger run --silent /src/grafana-build artifacts -a $${ARTIFACTS} --grafana-ref=$${GRAFANA_REF} + --enterprise-ref=$${ENTERPRISE_REF} --version=$${VERSION} ' + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + ALPINE_BASE: alpine:3.18.4 + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.21.5 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + UBUNTU_BASE: ubuntu:22.04 + image: grafana/grafana-build:main + name: rgm-build + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - gcloud storage cp -r dist/* $${UPLOAD_TO} + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: google/cloud-sdk:alpine + name: rgm-copy +trigger: + event: + - promote + target: upload-packages +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- clone: disable: true depends_on: [] @@ -4631,6 +4731,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: 013b133b2c65edbc2220813ae96f6b55478f6b67643639ed54ddfc347de4cb55 +hmac: 5a32e13ad67622913b4ce5c63931d2fc6ba0c118b4f32452724c441aaabf53c5 ... diff --git a/scripts/drone/rgm.star b/scripts/drone/rgm.star index b411178cee8..c578febabc6 100644 --- a/scripts/drone/rgm.star +++ b/scripts/drone/rgm.star @@ -166,7 +166,7 @@ def rgm_copy(src, dst): dst: destination of the files. Returns: - Drone steps. + Drone step. """ commands = [ "printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json", @@ -174,20 +174,13 @@ def rgm_copy(src, dst): "gcloud storage cp -r {} {}".format(src, dst), ] - if not dst.startswith("gs://"): - commands.insert(0, "mkdir -p {}".format(dst)) - - rgm_copy_step = { + return { "name": "rgm-copy", "image": "google/cloud-sdk:alpine", "commands": commands, "environment": rgm_env_secrets({}), } - return [ - rgm_copy_step, - ] - def rgm_publish_packages(bucket = "grafana-packages"): """Publish deb and rpm packages. @@ -258,10 +251,18 @@ def rgm_version_branch(): ) def rgm_nightly_build(): + """Nightly build pipeline. + + Returns: + Drone pipeline. + """ src = "$${DRONE_WORKSPACE}/dist/*" dst = "$${DESTINATION}/$${DRONE_BUILD_EVENT}" + copy_step = rgm_copy(src, dst) + if not dst.startswith("gs://"): + copy_step["commands"].insert(0, "mkdir -p {}".format(dst)) - copy_steps = with_deps(rgm_copy(src, dst), ["rgm-build"]) + copy_steps = with_deps([copy_step], ["rgm-build"]) return pipeline( name = "rgm-nightly-build", @@ -281,11 +282,13 @@ def rgm_nightly_publish(): publish_steps = with_deps(rgm_run("rgm-publish", "drone_publish_nightly_grafana.sh"), ["rgm-copy"]) package_steps = with_deps(rgm_publish_packages(), ["rgm-publish"]) - + copy_step = rgm_copy(src, dst) + if not dst.startswith("gs://"): + copy_step["commands"].insert(0, "mkdir -p {}".format(dst)) return pipeline( name = "rgm-nightly-publish", trigger = nightly_trigger, - steps = rgm_copy(src, dst) + publish_steps + package_steps, + steps = [copy_step] + publish_steps + package_steps, depends_on = ["rgm-nightly-build"], ) @@ -333,10 +336,67 @@ def rgm_main_pipeline(): rgm_main(), ] +def rgm_promotion_pipeline(): + """Promotion build pipeline. + + Returns: + Drone pipeline. + """ + promotion_trigger = { + "event": ["promote"], + "target": "upload-packages", + } + + env = { + "GO_VERSION": golang_version, + "ALPINE_BASE": images["alpine"], + "UBUNTU_BASE": images["ubuntu"], + } + + # Expected promotion args: + # * GRAFANA_REF = commit hash, branch name, or tag name + # * ENTERPRISE_REF = commit hash, branch name, or tag name. If not building an enterprise artifact, then this can be + # left empty. + # * ARTIFACTS = comma delimited list of artifacts (ex: "targz:grafana:linux/amd64,rpm:grafana:linux/amd64") + # * VERSION = version string of Grafana that is being built (ex: v10.0.0) + # * UPLOAD_TO = Google Cloud Storage URL to upload the built artifacts to. (ex: gs://some-bucket/path) + build_step = { + "name": "rgm-build", + "image": "grafana/grafana-build:main", + "pull": "always", + "commands": [ + "dagger run --silent /src/grafana-build artifacts " + + "-a $${ARTIFACTS} " + + "--grafana-ref=$${GRAFANA_REF} " + + "--enterprise-ref=$${ENTERPRISE_REF} " + + "--version=$${VERSION} ", + ], + "environment": rgm_env_secrets(env), + # The docker socket is a requirement for running dagger programs + # In the future we should find a way to use dagger without mounting the docker socket. + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + } + + publish_step = rgm_copy("dist/*", "$${UPLOAD_TO}") + + steps = [ + build_step, + publish_step, + ] + + return [ + pipeline( + name = "rgm-promotion", + trigger = promotion_trigger, + steps = steps, + ), + ] + def rgm(): return ( rgm_main_pipeline() + rgm_tag_pipeline() + rgm_version_branch_pipeline() + - rgm_nightly_pipeline() + rgm_nightly_pipeline() + + rgm_promotion_pipeline() )