From 67f1778bf1efa1124f248e76e651cd4e8ed00d23 Mon Sep 17 00:00:00 2001 From: Julien Duchesne Date: Thu, 1 Sep 2022 07:13:44 -0400 Subject: [PATCH] Add package publishing step (#53553) Issue: https://github.com/grafana/deployment_tools/issues/36289 Based on the new image: https://github.com/grafana/deployment_tools/tree/master/docker/package-publish This is a new step meant to replace the store-packages command. It will greatly improve publishing performace and it publishes to a common repository shared with all Grafana products Co-authored-by: dsotirakis --- .drone.yml | 80 ++++++++++++++++++++++++++++++- scripts/drone/events/release.star | 3 ++ scripts/drone/steps/lib.star | 22 +++++++++ scripts/drone/vault.star | 8 ++++ 4 files changed, 112 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 9eef6791bd8..95b7d96e726 100644 --- a/.drone.yml +++ b/.drone.yml @@ -3617,6 +3617,27 @@ steps: from_secret: grafana_api_key image: grafana/grafana-ci-deploy:1.3.3 name: publish-grafanacom-oss +- depends_on: + - gen-version + failure: ignore + image: us.gcr.io/kubernetes-dev/package-publish:latest + name: publish-linux-packages + settings: + access_key_id: + from_secret: packages_access_key_id + deb_distribution: stable + gpg_passphrase: + from_secret: packages_gpg_passphrase + gpg_private_key: + from_secret: packages_gpg_private_key + gpg_public_key: + from_secret: packages_gpg_public_key + package_path: gs://grafana-prerelease/artifacts/downloads/*${DRONE_TAG}/oss/**.deb + secret_access_key: + from_secret: packages_secret_access_key + service_account_json: + from_secret: packages_service_account_json + target_bucket: grafana-packages trigger: event: - promote @@ -3682,6 +3703,27 @@ steps: from_secret: grafana_api_key image: grafana/grafana-ci-deploy:1.3.3 name: publish-grafanacom-enterprise +- depends_on: + - gen-version + failure: ignore + image: us.gcr.io/kubernetes-dev/package-publish:latest + name: publish-linux-packages + settings: + access_key_id: + from_secret: packages_access_key_id + deb_distribution: stable + gpg_passphrase: + from_secret: packages_gpg_passphrase + gpg_private_key: + from_secret: packages_gpg_private_key + gpg_public_key: + from_secret: packages_gpg_public_key + package_path: gs://grafana-prerelease/artifacts/downloads/*${DRONE_TAG}/enterprise/**.deb + secret_access_key: + from_secret: packages_secret_access_key + service_account_json: + from_secret: packages_service_account_json + target_bucket: grafana-packages trigger: event: - promote @@ -5174,7 +5216,43 @@ get: kind: secret name: gcp_upload_artifacts_key --- +get: + name: public-key + path: infra/data/ci/packages-publish/gpg +kind: secret +name: packages_gpg_public_key +--- +get: + name: private-key + path: infra/data/ci/packages-publish/gpg +kind: secret +name: packages_gpg_private_key +--- +get: + name: passphrase + path: infra/data/ci/packages-publish/gpg +kind: secret +name: packages_gpg_passphrase +--- +get: + name: credentials.json + path: infra/data/ci/packages-publish/service-account +kind: secret +name: packages_service_account +--- +get: + name: AccessID + path: infra/data/ci/packages-publish/bucket-credentials +kind: secret +name: packages_access_key_id +--- +get: + name: Secret + path: infra/data/ci/packages-publish/bucket-credentials +kind: secret +name: packages_secret_access_key +--- kind: signature -hmac: 50fdfb13ce364613fc8359736da906ba838f9ac60e32b8608ec08b4ca2c335aa +hmac: ce0208c16afae599091c3df008a9bbe3cf98bdfdd962554b722c9bc6985ad5ac ... diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index 289c57059f8..6cdb1fe565a 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -42,6 +42,7 @@ load( 'upload_cdn_step', 'verify_gen_cue_step', 'publish_images_step', + 'publish_linux_packages_step', 'trigger_oss', 'artifacts_page_step', 'compile_build_cmd', @@ -394,6 +395,7 @@ def publish_packages_pipeline(): gen_version_step(ver_mode='release'), publish_packages_step(edition='oss', ver_mode='release'), publish_grafanacom_step(edition='oss', ver_mode='release'), + publish_linux_packages_step(edition='oss'), ] enterprise_steps = [ @@ -401,6 +403,7 @@ def publish_packages_pipeline(): gen_version_step(ver_mode='release'), publish_packages_step(edition='enterprise', ver_mode='release'), publish_grafanacom_step(edition='enterprise', ver_mode='release'), + publish_linux_packages_step(edition='enterprise'), ] deps = [ 'publish-artifacts-public', diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 49539bde70d..e36a84917b6 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -1055,6 +1055,28 @@ def publish_grafanacom_step(edition, ver_mode): ], } +def publish_linux_packages_step(edition): + return { + 'name': 'publish-linux-packages', + # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image + 'image': 'us.gcr.io/kubernetes-dev/package-publish:latest', + 'depends_on': [ + 'gen-version' + ], + 'failure': 'ignore', # While we're testing it + 'settings': { + 'access_key_id': from_secret('packages_access_key_id'), + 'secret_access_key': from_secret('packages_secret_access_key'), + 'service_account_json': from_secret('packages_service_account_json'), + 'target_bucket': 'grafana-packages', + 'deb_distribution': 'stable', + 'gpg_passphrase': from_secret('packages_gpg_passphrase'), + 'gpg_public_key': from_secret('packages_gpg_public_key'), + 'gpg_private_key': from_secret('packages_gpg_private_key'), + 'package_path': 'gs://grafana-prerelease/artifacts/downloads/*${{DRONE_TAG}}/{}/**.deb'.format(edition) + } + } + def get_windows_steps(edition, ver_mode): init_cmds = [] diff --git a/scripts/drone/vault.star b/scripts/drone/vault.star index 997f8b443bb..b30d486cfec 100644 --- a/scripts/drone/vault.star +++ b/scripts/drone/vault.star @@ -26,4 +26,12 @@ def secrets(): vault_secret(drone_token, 'infra/data/ci/drone', 'machine-user-token'), vault_secret(prerelease_bucket, 'infra/data/ci/grafana/prerelease', 'bucket'), vault_secret(gcp_upload_artifacts_key, 'infra/data/ci/grafana/releng/artifacts-uploader-service-account', 'credentials.json'), + + # Package publishing + vault_secret('packages_gpg_public_key', 'infra/data/ci/packages-publish/gpg', 'public-key'), + vault_secret('packages_gpg_private_key', 'infra/data/ci/packages-publish/gpg', 'private-key'), + vault_secret('packages_gpg_passphrase', 'infra/data/ci/packages-publish/gpg', 'passphrase'), + vault_secret('packages_service_account', 'infra/data/ci/packages-publish/service-account', 'credentials.json'), + vault_secret('packages_access_key_id', 'infra/data/ci/packages-publish/bucket-credentials', 'AccessID'), + vault_secret('packages_secret_access_key', 'infra/data/ci/packages-publish/bucket-credentials', 'Secret'), ]