Add package publishing step (#53553) (#54846)

Issue: https://github.com/grafana/deployment_tools/issues/36289
Based on the new image: https://github.com/grafana/deployment_tools/tree/master/docker/package-publish
This is a new step meant to replace the store-packages command. It will greatly improve publishing performace and it publishes to a common repository shared with all Grafana products

Co-authored-by: dsotirakis <dimitrios.sotirakis@grafana.com>
(cherry picked from commit 67f1778bf1)

Co-authored-by: Julien Duchesne <julien.duchesne@grafana.com>

.drone.yml

@@ -3500,6 +3500,27 @@ steps:
       from_secret: grafana_api_key
   image: grafana/grafana-ci-deploy:1.3.3
   name: store-packages-oss
+- depends_on:
+  - gen-version
+  failure: ignore
+  image: us.gcr.io/kubernetes-dev/package-publish:latest
+  name: publish-linux-packages
+  settings:
+    access_key_id:
+      from_secret: packages_access_key_id
+    deb_distribution: stable
+    gpg_passphrase:
+      from_secret: packages_gpg_passphrase
+    gpg_private_key:
+      from_secret: packages_gpg_private_key
+    gpg_public_key:
+      from_secret: packages_gpg_public_key
+    package_path: gs://grafana-prerelease/artifacts/downloads/*${DRONE_TAG}/oss/**.deb
+    secret_access_key:
+      from_secret: packages_secret_access_key
+    service_account_json:
+      from_secret: packages_service_account_json
+    target_bucket: grafana-packages
 trigger:
   event:
   - promote
@@ -3554,6 +3575,27 @@ steps:
       from_secret: grafana_api_key
   image: grafana/grafana-ci-deploy:1.3.3
   name: store-packages-enterprise
+- depends_on:
+  - gen-version
+  failure: ignore
+  image: us.gcr.io/kubernetes-dev/package-publish:latest
+  name: publish-linux-packages
+  settings:
+    access_key_id:
+      from_secret: packages_access_key_id
+    deb_distribution: stable
+    gpg_passphrase:
+      from_secret: packages_gpg_passphrase
+    gpg_private_key:
+      from_secret: packages_gpg_private_key
+    gpg_public_key:
+      from_secret: packages_gpg_public_key
+    package_path: gs://grafana-prerelease/artifacts/downloads/*${DRONE_TAG}/enterprise/**.deb
+    secret_access_key:
+      from_secret: packages_secret_access_key
+    service_account_json:
+      from_secret: packages_service_account_json
+    target_bucket: grafana-packages
 trigger:
   event:
   - promote
@@ -5021,7 +5063,43 @@ get:
 kind: secret
 name: gcp_upload_artifacts_key
 ---
+get:
+  name: public-key
+  path: infra/data/ci/packages-publish/gpg
+kind: secret
+name: packages_gpg_public_key
+---
+get:
+  name: private-key
+  path: infra/data/ci/packages-publish/gpg
+kind: secret
+name: packages_gpg_private_key
+---
+get:
+  name: passphrase
+  path: infra/data/ci/packages-publish/gpg
+kind: secret
+name: packages_gpg_passphrase
+---
+get:
+  name: credentials.json
+  path: infra/data/ci/packages-publish/service-account
+kind: secret
+name: packages_service_account
+---
+get:
+  name: AccessID
+  path: infra/data/ci/packages-publish/bucket-credentials
+kind: secret
+name: packages_access_key_id
+---
+get:
+  name: Secret
+  path: infra/data/ci/packages-publish/bucket-credentials
+kind: secret
+name: packages_secret_access_key
+---
 kind: signature
-hmac: a03ea93b638a318e1465d1d24d1a52337f473e90e7cc1ac7d6762f42472986a4
+hmac: f8c3274434ad2e4a4a67a25e927256d22ba0dd49f99a9eb498b2bf0f88835933
 
 ...

scripts/drone/events/release.star

@@ -41,6 +41,7 @@ load(
     'upload_cdn_step',
     'verify_gen_cue_step',
     'publish_images_step',
+    'publish_linux_packages_step',
     'trigger_oss',
     'artifacts_page_step',
     'compile_build_cmd',
@@ -403,12 +404,14 @@ def publish_packages_pipeline():
         download_grabpl_step(),
         gen_version_step(ver_mode='release'),
         store_packages_step(edition='oss', ver_mode='release'),
+        publish_linux_packages_step(edition='oss'),
     ]
 
     enterprise_steps = [
         download_grabpl_step(),
         gen_version_step(ver_mode='release'),
         store_packages_step(edition='enterprise', ver_mode='release'),
+        publish_linux_packages_step(edition='enterprise'),
     ]
     deps = [
         'publish-artifacts-public',

scripts/drone/steps/lib.star

@@ -1023,6 +1023,28 @@ def store_packages_step(edition, ver_mode):
         ],
     }
 
+def publish_linux_packages_step(edition):
+    return {
+        'name': 'publish-linux-packages',
+        # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image
+        'image': 'us.gcr.io/kubernetes-dev/package-publish:latest',
+        'depends_on': [
+            'gen-version'
+        ],
+        'failure': 'ignore', # While we're testing it
+        'settings': {
+            'access_key_id': from_secret('packages_access_key_id'),
+            'secret_access_key': from_secret('packages_secret_access_key'),
+            'service_account_json': from_secret('packages_service_account_json'),
+            'target_bucket': 'grafana-packages',
+            'deb_distribution': 'stable',
+            'gpg_passphrase': from_secret('packages_gpg_passphrase'),
+            'gpg_public_key': from_secret('packages_gpg_public_key'),
+            'gpg_private_key': from_secret('packages_gpg_private_key'),
+            'package_path': 'gs://grafana-prerelease/artifacts/downloads/*${{DRONE_TAG}}/{}/**.deb'.format(edition)
+        }
+    }
+
 
 def get_windows_steps(edition, ver_mode):
     init_cmds = []

scripts/drone/vault.star

@@ -26,4 +26,12 @@ def secrets():
         vault_secret(drone_token, 'infra/data/ci/drone', 'machine-user-token'),
         vault_secret(prerelease_bucket, 'infra/data/ci/grafana/prerelease', 'bucket'),
         vault_secret(gcp_upload_artifacts_key, 'infra/data/ci/grafana/releng/artifacts-uploader-service-account', 'credentials.json'),
+    
+        # Package publishing
+        vault_secret('packages_gpg_public_key', 'infra/data/ci/packages-publish/gpg', 'public-key'),
+        vault_secret('packages_gpg_private_key', 'infra/data/ci/packages-publish/gpg', 'private-key'),
+        vault_secret('packages_gpg_passphrase', 'infra/data/ci/packages-publish/gpg', 'passphrase'),
+        vault_secret('packages_service_account', 'infra/data/ci/packages-publish/service-account', 'credentials.json'),
+        vault_secret('packages_access_key_id', 'infra/data/ci/packages-publish/bucket-credentials', 'AccessID'),
+        vault_secret('packages_secret_access_key', 'infra/data/ci/packages-publish/bucket-credentials', 'Secret'),
     ]