[v8.5.x] CI: backport the rpm and deb packaging to 8.5.x (#62639)

* backport the rpm and deb packaging to 8.5.x

Co-authored-by: Julien Duchesne <julien.duchesne@grafana.com>

.drone.yml

@@ -3124,23 +3124,66 @@ steps:
   image: byrnedo/alpine-curl:0.1.8
   name: grabpl
 - commands:
-  - ./bin/grabpl store-packages --edition oss --packages-bucket grafana-downloads
-    --gcp-key /tmp/gcpkey.json ${DRONE_TAG}
-  depends_on:
-  - grabpl
+  - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd
+  depends_on: []
   environment:
-    GCP_KEY:
-      from_secret: gcp_key
-    GPG_KEY_PASSWORD:
+    CGO_ENABLED: 0
+  image: golang:1.19.4
+  name: compile-build-cmd
+- depends_on:
+  - grabpl
+  failure: ignore
+  image: us.gcr.io/kubernetes-dev/package-publish:latest
+  name: publish-linux-packages-deb
+  privileged: true
+  settings:
+    access_key_id:
+      from_secret: packages_access_key_id
+    deb_distribution: stable
+    gpg_passphrase:
       from_secret: packages_gpg_passphrase
-    GPG_PRIV_KEY:
+    gpg_private_key:
       from_secret: packages_gpg_private_key
-    GPG_PUB_KEY:
+    gpg_public_key:
       from_secret: packages_gpg_public_key
+    package_path: gs://grafana-prerelease/artifacts/downloads/*${DRONE_TAG}/oss/**.deb
+    secret_access_key:
+      from_secret: packages_secret_access_key
+    service_account_json:
+      from_secret: packages_service_account
+    target_bucket: grafana-packages
+- depends_on:
+  - grabpl
+  failure: ignore
+  image: us.gcr.io/kubernetes-dev/package-publish:latest
+  name: publish-linux-packages-rpm
+  privileged: true
+  settings:
+    access_key_id:
+      from_secret: packages_access_key_id
+    deb_distribution: stable
+    gpg_passphrase:
+      from_secret: packages_gpg_passphrase
+    gpg_private_key:
+      from_secret: packages_gpg_private_key
+    gpg_public_key:
+      from_secret: packages_gpg_public_key
+    package_path: gs://grafana-prerelease/artifacts/downloads/*${DRONE_TAG}/oss/**.rpm
+    secret_access_key:
+      from_secret: packages_secret_access_key
+    service_account_json:
+      from_secret: packages_service_account
+    target_bucket: grafana-packages
+- commands:
+  - ./bin/build publish grafana-com --edition oss ${DRONE_TAG}
+  depends_on:
+  - publish-linux-packages-deb
+  - publish-linux-packages-rpm
+  environment:
     GRAFANA_COM_API_KEY:
       from_secret: grafana_api_key
   image: grafana/grafana-ci-deploy:1.3.1
-  name: store-packages-oss
+  name: publish-grafanacom-oss
 trigger:
   event:
   - promote
@@ -3172,23 +3215,66 @@ steps:
   image: byrnedo/alpine-curl:0.1.8
   name: grabpl
 - commands:
-  - ./bin/grabpl store-packages --edition enterprise --packages-bucket grafana-downloads
-    --gcp-key /tmp/gcpkey.json ${DRONE_TAG}
-  depends_on:
-  - grabpl
+  - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd
+  depends_on: []
   environment:
-    GCP_KEY:
-      from_secret: gcp_key
-    GPG_KEY_PASSWORD:
+    CGO_ENABLED: 0
+  image: golang:1.19.4
+  name: compile-build-cmd
+- depends_on:
+  - grabpl
+  failure: ignore
+  image: us.gcr.io/kubernetes-dev/package-publish:latest
+  name: publish-linux-packages-deb
+  privileged: true
+  settings:
+    access_key_id:
+      from_secret: packages_access_key_id
+    deb_distribution: stable
+    gpg_passphrase:
       from_secret: packages_gpg_passphrase
-    GPG_PRIV_KEY:
+    gpg_private_key:
       from_secret: packages_gpg_private_key
-    GPG_PUB_KEY:
+    gpg_public_key:
       from_secret: packages_gpg_public_key
+    package_path: gs://grafana-prerelease/artifacts/downloads/*${DRONE_TAG}/enterprise/**.deb
+    secret_access_key:
+      from_secret: packages_secret_access_key
+    service_account_json:
+      from_secret: packages_service_account
+    target_bucket: grafana-packages
+- depends_on:
+  - grabpl
+  failure: ignore
+  image: us.gcr.io/kubernetes-dev/package-publish:latest
+  name: publish-linux-packages-rpm
+  privileged: true
+  settings:
+    access_key_id:
+      from_secret: packages_access_key_id
+    deb_distribution: stable
+    gpg_passphrase:
+      from_secret: packages_gpg_passphrase
+    gpg_private_key:
+      from_secret: packages_gpg_private_key
+    gpg_public_key:
+      from_secret: packages_gpg_public_key
+    package_path: gs://grafana-prerelease/artifacts/downloads/*${DRONE_TAG}/enterprise/**.rpm
+    secret_access_key:
+      from_secret: packages_secret_access_key
+    service_account_json:
+      from_secret: packages_service_account
+    target_bucket: grafana-packages
+- commands:
+  - ./bin/build publish grafana-com --edition enterprise ${DRONE_TAG}
+  depends_on:
+  - publish-linux-packages-deb
+  - publish-linux-packages-rpm
+  environment:
     GRAFANA_COM_API_KEY:
       from_secret: grafana_api_key
   image: grafana/grafana-ci-deploy:1.3.1
-  name: store-packages-enterprise
+  name: publish-grafanacom-enterprise
 trigger:
   event:
   - promote
@@ -4638,6 +4724,6 @@ kind: secret
 name: aws_secret_access_key
 ---
 kind: signature
-hmac: 2b78b3489ea0d6afba76f0ccf5b59cb5c77e92d5f5ee7b39bc9ff4010cd32ed6
+hmac: b7359fe83b6b5a8bfcd1c4ecb97c9242d94e2b95364258fdd93dba626d31d363
 
 ...

scripts/drone/pipelines/release.star

@@ -11,6 +11,8 @@ load(
     'build_image',
     'identify_runner_step',
     'publish_image',
+    'publish_linux_packages_step',
+    'publish_grafanacom_step',
     'lint_backend_step',
     'lint_frontend_step',
     'codespell_step',
@@ -42,7 +44,8 @@ load(
     'validate_scuemata_step',
     'ensure_cuetsified_step',
     'publish_images_step',
-    'trigger_oss'
+    'trigger_oss',
+    'compile_build_cmd',
 )
 
 load(
@@ -404,12 +407,18 @@ def publish_packages_pipeline():
     }
     oss_steps = [
         download_grabpl_step(),
-        store_packages_step(edition='oss', ver_mode='release'),
+        compile_build_cmd(),
+        publish_linux_packages_step(edition = "oss", package_manager = "deb"),
+        publish_linux_packages_step(edition = "oss", package_manager = "rpm"),
+        publish_grafanacom_step(edition = "oss", ver_mode = "release"),
     ]
 
     enterprise_steps = [
         download_grabpl_step(),
-        store_packages_step(edition='enterprise', ver_mode='release'),
+        compile_build_cmd(),
+        publish_linux_packages_step(edition = "enterprise", package_manager = "deb"),
+        publish_linux_packages_step(edition = "enterprise", package_manager = "rpm"),
+        publish_grafanacom_step(edition = "enterprise", ver_mode = "release"),
     ]
     deps = [
         'publish-artifacts-public',

scripts/drone/steps/lib.star

@@ -8,6 +8,7 @@ alpine_image = 'alpine:3.15'
 curl_image = 'byrnedo/alpine-curl:0.1.8'
 windows_image = 'mcr.microsoft.com/windows:1809'
 wix_image = 'grafana/ci-wix:0.1.1'
+go_image = "golang:1.19.4"
 
 disable_tests = False
 trigger_oss = {
@@ -1005,6 +1006,56 @@ def store_packages_step(edition, ver_mode):
         ],
     }
 
+def publish_grafanacom_step(edition, ver_mode):
+    if ver_mode == 'release':
+        cmd = './bin/build publish grafana-com --edition {} ${{DRONE_TAG}}'.format(
+            edition,
+        )
+    elif ver_mode == 'main':
+        build_no = '${DRONE_BUILD_NUMBER}'
+        cmd = './bin/build publish grafana-com --edition {} --build-id {}'.format(
+            edition, build_no,
+        )
+    else:
+        fail('Unexpected version mode {}'.format(ver_mode))
+
+    return {
+        'name': 'publish-grafanacom-{}'.format(edition),
+        'image': publish_image,
+        'depends_on': [
+            "publish-linux-packages-deb",
+            "publish-linux-packages-rpm",
+        ],
+        'environment': {
+            'GRAFANA_COM_API_KEY': from_secret('grafana_api_key'),
+        },
+        'commands': [
+            cmd,
+        ],
+    }
+
+def publish_linux_packages_step(edition, package_manager='deb'):
+    return {
+        'name': 'publish-linux-packages-{}'.format(package_manager),
+        # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image
+        'image': 'us.gcr.io/kubernetes-dev/package-publish:latest',
+        'depends_on': [
+            'grabpl'
+        ],
+        'privileged': True,
+        'failure': 'ignore', # While we're testing it
+        'settings': {
+            'access_key_id': from_secret('packages_access_key_id'),
+            'secret_access_key': from_secret('packages_secret_access_key'),
+            'service_account_json': from_secret('packages_service_account'),
+            'target_bucket': 'grafana-packages',
+            'deb_distribution': 'stable',
+            'gpg_passphrase': from_secret('packages_gpg_passphrase'),
+            'gpg_public_key': from_secret('packages_gpg_public_key'),
+            'gpg_private_key': from_secret('packages_gpg_private_key'),
+            'package_path': 'gs://grafana-prerelease/artifacts/downloads/*${{DRONE_TAG}}/{}/**.{}'.format(edition, package_manager)
+        }
+    }
 
 def get_windows_steps(edition, ver_mode):
     init_cmds = []
@@ -1166,3 +1217,21 @@ def end_to_end_tests_deps(edition):
         'end-to-end-tests-smoke-tests-suite' + enterprise2_suffix(edition),
         'end-to-end-tests-various-suite' + enterprise2_suffix(edition),
     ]
+
+def compile_build_cmd(edition = "oss"):
+    dependencies = []
+    if edition in ("enterprise", "enterprise2"):
+        dependencies = [
+            "init-enterprise",
+        ]
+    return {
+        "name": "compile-build-cmd",
+        "image": go_image,
+        "commands": [
+            "go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd",
+        ],
+        "depends_on": dependencies,
+        "environment": {
+            "CGO_ENABLED": 0,
+        },
+    }