From 58261a814ac074d565de23730075dc0de35c5cc4 Mon Sep 17 00:00:00 2001 From: Kevin Minehart Date: Thu, 2 Feb 2023 10:22:57 -0600 Subject: [PATCH] [v8.5.x] CI: backport the rpm and deb packaging to 8.5.x (#62639) * backport the rpm and deb packaging to 8.5.x Co-authored-by: Julien Duchesne --- .drone.yml | 128 ++++++++++++++++++++++----- scripts/drone/pipelines/release.star | 15 +++- scripts/drone/steps/lib.star | 69 +++++++++++++++ 3 files changed, 188 insertions(+), 24 deletions(-) diff --git a/.drone.yml b/.drone.yml index 1e9446795a5..6b248d65c70 100644 --- a/.drone.yml +++ b/.drone.yml @@ -3124,23 +3124,66 @@ steps: image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - - ./bin/grabpl store-packages --edition oss --packages-bucket grafana-downloads - --gcp-key /tmp/gcpkey.json ${DRONE_TAG} - depends_on: - - grabpl + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] environment: - GCP_KEY: - from_secret: gcp_key - GPG_KEY_PASSWORD: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd +- depends_on: + - grabpl + failure: ignore + image: us.gcr.io/kubernetes-dev/package-publish:latest + name: publish-linux-packages-deb + privileged: true + settings: + access_key_id: + from_secret: packages_access_key_id + deb_distribution: stable + gpg_passphrase: from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: + gpg_private_key: from_secret: packages_gpg_private_key - GPG_PUB_KEY: + gpg_public_key: from_secret: packages_gpg_public_key + package_path: gs://grafana-prerelease/artifacts/downloads/*${DRONE_TAG}/oss/**.deb + secret_access_key: + from_secret: packages_secret_access_key + service_account_json: + from_secret: packages_service_account + target_bucket: grafana-packages +- depends_on: + - grabpl + failure: ignore + image: us.gcr.io/kubernetes-dev/package-publish:latest + name: publish-linux-packages-rpm + privileged: true + settings: + access_key_id: + from_secret: packages_access_key_id + deb_distribution: stable + gpg_passphrase: + from_secret: packages_gpg_passphrase + gpg_private_key: + from_secret: packages_gpg_private_key + gpg_public_key: + from_secret: packages_gpg_public_key + package_path: gs://grafana-prerelease/artifacts/downloads/*${DRONE_TAG}/oss/**.rpm + secret_access_key: + from_secret: packages_secret_access_key + service_account_json: + from_secret: packages_service_account + target_bucket: grafana-packages +- commands: + - ./bin/build publish grafana-com --edition oss ${DRONE_TAG} + depends_on: + - publish-linux-packages-deb + - publish-linux-packages-rpm + environment: GRAFANA_COM_API_KEY: from_secret: grafana_api_key image: grafana/grafana-ci-deploy:1.3.1 - name: store-packages-oss + name: publish-grafanacom-oss trigger: event: - promote @@ -3172,23 +3215,66 @@ steps: image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - - ./bin/grabpl store-packages --edition enterprise --packages-bucket grafana-downloads - --gcp-key /tmp/gcpkey.json ${DRONE_TAG} - depends_on: - - grabpl + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] environment: - GCP_KEY: - from_secret: gcp_key - GPG_KEY_PASSWORD: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd +- depends_on: + - grabpl + failure: ignore + image: us.gcr.io/kubernetes-dev/package-publish:latest + name: publish-linux-packages-deb + privileged: true + settings: + access_key_id: + from_secret: packages_access_key_id + deb_distribution: stable + gpg_passphrase: from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: + gpg_private_key: from_secret: packages_gpg_private_key - GPG_PUB_KEY: + gpg_public_key: from_secret: packages_gpg_public_key + package_path: gs://grafana-prerelease/artifacts/downloads/*${DRONE_TAG}/enterprise/**.deb + secret_access_key: + from_secret: packages_secret_access_key + service_account_json: + from_secret: packages_service_account + target_bucket: grafana-packages +- depends_on: + - grabpl + failure: ignore + image: us.gcr.io/kubernetes-dev/package-publish:latest + name: publish-linux-packages-rpm + privileged: true + settings: + access_key_id: + from_secret: packages_access_key_id + deb_distribution: stable + gpg_passphrase: + from_secret: packages_gpg_passphrase + gpg_private_key: + from_secret: packages_gpg_private_key + gpg_public_key: + from_secret: packages_gpg_public_key + package_path: gs://grafana-prerelease/artifacts/downloads/*${DRONE_TAG}/enterprise/**.rpm + secret_access_key: + from_secret: packages_secret_access_key + service_account_json: + from_secret: packages_service_account + target_bucket: grafana-packages +- commands: + - ./bin/build publish grafana-com --edition enterprise ${DRONE_TAG} + depends_on: + - publish-linux-packages-deb + - publish-linux-packages-rpm + environment: GRAFANA_COM_API_KEY: from_secret: grafana_api_key image: grafana/grafana-ci-deploy:1.3.1 - name: store-packages-enterprise + name: publish-grafanacom-enterprise trigger: event: - promote @@ -4638,6 +4724,6 @@ kind: secret name: aws_secret_access_key --- kind: signature -hmac: 2b78b3489ea0d6afba76f0ccf5b59cb5c77e92d5f5ee7b39bc9ff4010cd32ed6 +hmac: b7359fe83b6b5a8bfcd1c4ecb97c9242d94e2b95364258fdd93dba626d31d363 ... diff --git a/scripts/drone/pipelines/release.star b/scripts/drone/pipelines/release.star index e85610016b5..259276dd095 100644 --- a/scripts/drone/pipelines/release.star +++ b/scripts/drone/pipelines/release.star @@ -11,6 +11,8 @@ load( 'build_image', 'identify_runner_step', 'publish_image', + 'publish_linux_packages_step', + 'publish_grafanacom_step', 'lint_backend_step', 'lint_frontend_step', 'codespell_step', @@ -42,7 +44,8 @@ load( 'validate_scuemata_step', 'ensure_cuetsified_step', 'publish_images_step', - 'trigger_oss' + 'trigger_oss', + 'compile_build_cmd', ) load( @@ -404,12 +407,18 @@ def publish_packages_pipeline(): } oss_steps = [ download_grabpl_step(), - store_packages_step(edition='oss', ver_mode='release'), + compile_build_cmd(), + publish_linux_packages_step(edition = "oss", package_manager = "deb"), + publish_linux_packages_step(edition = "oss", package_manager = "rpm"), + publish_grafanacom_step(edition = "oss", ver_mode = "release"), ] enterprise_steps = [ download_grabpl_step(), - store_packages_step(edition='enterprise', ver_mode='release'), + compile_build_cmd(), + publish_linux_packages_step(edition = "enterprise", package_manager = "deb"), + publish_linux_packages_step(edition = "enterprise", package_manager = "rpm"), + publish_grafanacom_step(edition = "enterprise", ver_mode = "release"), ] deps = [ 'publish-artifacts-public', diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index f14958c2570..67dca67f44d 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -8,6 +8,7 @@ alpine_image = 'alpine:3.15' curl_image = 'byrnedo/alpine-curl:0.1.8' windows_image = 'mcr.microsoft.com/windows:1809' wix_image = 'grafana/ci-wix:0.1.1' +go_image = "golang:1.19.4" disable_tests = False trigger_oss = { @@ -1005,6 +1006,56 @@ def store_packages_step(edition, ver_mode): ], } +def publish_grafanacom_step(edition, ver_mode): + if ver_mode == 'release': + cmd = './bin/build publish grafana-com --edition {} ${{DRONE_TAG}}'.format( + edition, + ) + elif ver_mode == 'main': + build_no = '${DRONE_BUILD_NUMBER}' + cmd = './bin/build publish grafana-com --edition {} --build-id {}'.format( + edition, build_no, + ) + else: + fail('Unexpected version mode {}'.format(ver_mode)) + + return { + 'name': 'publish-grafanacom-{}'.format(edition), + 'image': publish_image, + 'depends_on': [ + "publish-linux-packages-deb", + "publish-linux-packages-rpm", + ], + 'environment': { + 'GRAFANA_COM_API_KEY': from_secret('grafana_api_key'), + }, + 'commands': [ + cmd, + ], + } + +def publish_linux_packages_step(edition, package_manager='deb'): + return { + 'name': 'publish-linux-packages-{}'.format(package_manager), + # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image + 'image': 'us.gcr.io/kubernetes-dev/package-publish:latest', + 'depends_on': [ + 'grabpl' + ], + 'privileged': True, + 'failure': 'ignore', # While we're testing it + 'settings': { + 'access_key_id': from_secret('packages_access_key_id'), + 'secret_access_key': from_secret('packages_secret_access_key'), + 'service_account_json': from_secret('packages_service_account'), + 'target_bucket': 'grafana-packages', + 'deb_distribution': 'stable', + 'gpg_passphrase': from_secret('packages_gpg_passphrase'), + 'gpg_public_key': from_secret('packages_gpg_public_key'), + 'gpg_private_key': from_secret('packages_gpg_private_key'), + 'package_path': 'gs://grafana-prerelease/artifacts/downloads/*${{DRONE_TAG}}/{}/**.{}'.format(edition, package_manager) + } + } def get_windows_steps(edition, ver_mode): init_cmds = [] @@ -1166,3 +1217,21 @@ def end_to_end_tests_deps(edition): 'end-to-end-tests-smoke-tests-suite' + enterprise2_suffix(edition), 'end-to-end-tests-various-suite' + enterprise2_suffix(edition), ] + +def compile_build_cmd(edition = "oss"): + dependencies = [] + if edition in ("enterprise", "enterprise2"): + dependencies = [ + "init-enterprise", + ] + return { + "name": "compile-build-cmd", + "image": go_image, + "commands": [ + "go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd", + ], + "depends_on": dependencies, + "environment": { + "CGO_ENABLED": 0, + }, + }