CI: Automate Grafana releases (#43897) (#43911)

* CI: Automate releases

* Update grabpl version

(cherry picked from commit 4f47832bac)

scripts/drone/pipelines/release.star

@@ -68,10 +68,7 @@ def build_npm_packages_step(edition, ver_mode):
         'commands': ['./scripts/build/build-npm-packages.sh ${DRONE_TAG}'],
     }
 
-def store_npm_packages_step(edition, ver_mode):
-    if edition == 'enterprise' or ver_mode != 'release':
-        return None
-
+def store_npm_packages_step():
     return {
         'name': 'store-npm-packages',
         'image': publish_image,
@@ -82,31 +79,28 @@ def store_npm_packages_step(edition, ver_mode):
             'GCP_KEY': from_secret('gcp_key'),
             'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
         },
-        'commands': ['./scripts/build/store-npm-packages.sh ${DRONE_TAG}'],
+        'commands': [
+            './bin/grabpl artifacts npm store --tag ${DRONE_TAG}'
+        ],
     }
 
-def retrieve_npm_packages_step(edition, ver_mode):
-    if edition == 'enterprise' or ver_mode != 'release':
-        return None
-
+def retrieve_npm_packages_step():
     return {
         'name': 'retrieve-npm-packages',
         'image': publish_image,
         'depends_on': [
-            # Has to run after store-storybook since this step cleans the files publish-storybook depends on
-            'store-storybook',
+            'initialize',
         ],
         'environment': {
             'GCP_KEY': from_secret('gcp_key'),
             'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
         },
-        'commands': ['./scripts/build/retrieve-npm-packages.sh ${DRONE_TAG}'],
+        'commands': [
+            './bin/grabpl artifacts npm retrieve --tag v${TAG}'
+        ],
     }
 
-def release_npm_packages_step(edition, ver_mode):
-    if edition == 'enterprise' or ver_mode != 'release':
-        return None
-
+def release_npm_packages_step():
     return {
         'name': 'release-npm-packages',
         'image': build_image,
@@ -116,9 +110,71 @@ def release_npm_packages_step(edition, ver_mode):
         'environment': {
             'NPM_TOKEN': from_secret('npm_token'),
         },
-        'commands': ['./scripts/build/release-npm-packages.sh ${DRONE_TAG}'],
+        'commands': [
+            './bin/grabpl artifacts npm release --tag v${TAG}'
+        ],
     }
 
+def publish_images_step(edition, mode, docker_repo):
+    if mode == 'security':
+        mode = '--{} '.format(mode)
+    else:
+        mode = ''
+    return {
+        'name': 'publish-images-{}'.format(docker_repo),
+        'image': 'google/cloud-sdk',
+        'environment': {
+            'GCP_KEY': from_secret('gcp_key'),
+            'DOCKER_USER': from_secret('docker_username'),
+            'DOCKER_PASSWORD': from_secret('docker_password'),
+        },
+        'commands': ['./bin/grabpl artifacts docker publish {}--version-tag ${{TAG}} --dockerhub-repo {} --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7'.format(mode, docker_repo)],
+        'depends_on': ['fetch-images-{}'.format(edition)],
+        'volumes': [{
+            'name': 'docker',
+            'path': '/var/run/docker.sock'
+        }],
+    }
+
+def fetch_images_step(edition):
+    return {
+        'name': 'fetch-images-{}'.format(edition),
+        'image': 'google/cloud-sdk',
+        'environment': {
+            'GCP_KEY': from_secret('gcp_key'),
+            'DOCKER_USER': from_secret('docker_username'),
+            'DOCKER_PASSWORD': from_secret('docker_password'),
+        },
+        'commands': ['./bin/grabpl artifacts docker fetch --version-tag ${{TAG}} --edition {} --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7'.format(edition)],
+        'depends_on': ['grabpl'],
+        'volumes': [{
+            'name': 'docker',
+            'path': '/var/run/docker.sock'
+        }],
+    }
+
+def publish_image_steps(version, mode, docker_repo, additional_docker_repo=""):
+    steps = [
+        download_grabpl_step(),
+        fetch_images_step(version),
+        publish_images_step(version, mode, docker_repo),
+    ]
+    if additional_docker_repo != "":
+        steps.extend([publish_images_step(version, mode, additional_docker_repo)])
+
+    return steps
+
+def publish_image_pipelines(mode):
+    trigger = {
+        'event': ['promote'],
+        'target': [mode],
+    }
+
+    return [pipeline(
+        name='publish-docker-oss-{}'.format(mode), trigger=trigger, steps=publish_image_steps(version='oss',  mode=mode, docker_repo='grafana', additional_docker_repo='grafana-oss'), edition=""
+    ), pipeline(
+        name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(version='enterprise',  mode=mode, docker_repo='grafana-enterprise'), edition=""
+    ),]
 
 def get_steps(edition, ver_mode):
     package_steps = []
@@ -193,7 +249,7 @@ def get_steps(edition, ver_mode):
     if should_publish:
         publish_step = store_storybook_step(edition=edition, ver_mode=ver_mode)
         build_npm_step = build_npm_packages_step(edition=edition, ver_mode=ver_mode)
-        store_npm_step = store_npm_packages_step(edition=edition, ver_mode=ver_mode)
+        store_npm_step = store_npm_packages_step()
         if publish_step:
             publish_steps.append(publish_step)
         if build_npm_step and store_npm_step:
@@ -308,11 +364,85 @@ def get_enterprise_pipelines(trigger, ver_mode):
 
     return pipelines
 
-def release_pipelines(ver_mode='release', trigger=None):
+def publish_artifacts_step(mode):
+    security = ''
+    if mode == 'security':
+        security = '--security '
+    return {
+        'name': 'publish-artifacts',
+        'image': publish_image,
+        'environment': {
+            'GCP_KEY': from_secret('gcp_key'),
+        },
+        'commands': ['./bin/grabpl artifacts publish {}--tag ${{TAG}} --src-bucket grafana-prerelease'.format(security)],
+        'depends_on': ['grabpl'],
+    }
+
+def publish_packages_step(edition):
+    return {
+        'name': 'publish-packages-{}'.format(edition),
+        'image': publish_image,
+        'environment': {
+            'GCP_KEY': from_secret('gcp_key'),
+        },
+        'commands': ['./bin/grabpl store-packages {}'.format(edition)],
+        'depends_on': ['grabpl'],
+    }
+
+def publish_artifacts_pipelines(mode):
+    trigger = {
+        'event': ['promote'],
+        'target': [mode],
+    }
+    steps = [
+        download_grabpl_step(),
+        publish_artifacts_step(mode),
+    ]
+
+    return [pipeline(
+        name='publish-artifacts-{}'.format(mode), trigger=trigger, steps=steps, edition="all"
+    )]
+
+def publish_packages_pipeline():
+    trigger = {
+        'event': ['promote'],
+        'target': ['public'],
+    }
+    steps = [
+        download_grabpl_step(),
+        store_packages_step(edition='oss', ver_mode='release'),
+        store_packages_step(edition='enterprise', ver_mode='release'),
+    ]
+
+    return [pipeline(
+        name='publish-packages', trigger=trigger, steps=steps, edition="all"
+    )]
+
+def publish_npm_pipelines(mode):
+    trigger = {
+        'event': ['promote'],
+        'target': [mode],
+    }
+    steps = [
+        download_grabpl_step(),
+        retrieve_npm_packages_step(),
+        release_npm_packages_step()
+    ]
+
+    return [pipeline(
+        name='publish-npm-packages-{}'.format(mode), trigger=trigger, steps = initialize_step(edition='oss', platform='linux', ver_mode='release') + steps, edition="all"
+    )]
+
+def release_pipelines(ver_mode='release', trigger=None, environment=None):
     # 'enterprise' edition services contain both OSS and enterprise services
     services = integration_test_services(edition='enterprise')
     if not trigger:
         trigger = {
+            'event': {
+                'exclude': [
+                    'promote'
+                ]
+            },
             'ref': ['refs/tags/v*',],
             'repo': {
               'exclude': ['grafana/grafana'],
@@ -328,17 +458,9 @@ def release_pipelines(ver_mode='release', trigger=None):
     enterprise_pipelines = get_enterprise_pipelines(ver_mode=ver_mode, trigger=trigger)
 
     pipelines = oss_pipelines + enterprise_pipelines
-    if should_publish:
-        steps = [
-            store_packages_step(edition='oss', ver_mode=ver_mode),
-            store_packages_step(edition='enterprise', ver_mode=ver_mode),
-        ]
-        publish_pipeline = pipeline(
-            name='publish-{}'.format(ver_mode), trigger=trigger, edition='oss',
-            steps=[download_grabpl_step()] + initialize_step(edition='oss', platform='linux', ver_mode=ver_mode, install_deps=False) + steps,
-            depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines],
-        )
 
+    # if ver_mode == 'release':
+    #   pipelines.append(publish_artifacts_pipelines())
     #pipelines.append(notify_pipeline(
     #    name='notify-{}'.format(ver_mode), slack_channel='grafana-ci-notifications', trigger=trigger,
     #    depends_on=[p['name'] for p in pipelines], secret='slack_webhook',

scripts/drone/steps/lib.star

@@ -974,7 +974,7 @@ def store_packages_step(edition, ver_mode, is_downstream=False):
         'name': 'store-packages-{}'.format(edition),
         'image': publish_image,
         'depends_on': [
-            'initialize',
+            'grabpl',
         ],
         'environment': {
             'GRAFANA_COM_API_KEY': from_secret('grafana_api_key'),
@@ -1003,7 +1003,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
     else:
         init_cmds.extend([
             '$$ProgressPreference = "SilentlyContinue"',
-            'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/windows/grabpl.exe -OutFile grabpl.exe'.format(
+            'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format(
                 grabpl_version),
         ])
     steps = [
@@ -1077,7 +1077,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
         # For enterprise, we have to clone both OSS and enterprise and merge the latter into the former
         download_grabpl_step_cmds = [
             '$$ProgressPreference = "SilentlyContinue"',
-            'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/windows/grabpl.exe -OutFile grabpl.exe'.format(
+            'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format(
                 grabpl_version),
         ]
         clone_cmds = [

scripts/drone/utils/utils.star

@@ -7,7 +7,7 @@ load(
 load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token')
 
 def pipeline(
-    name, edition, trigger, steps, services=[], platform='linux', depends_on=[], volumes=[],
+    name, edition, trigger, steps, services=[], platform='linux', depends_on=[], environment=None, volumes=[],
     ):
     if platform != 'windows':
         platform_conf = {
@@ -48,6 +48,11 @@ def pipeline(
         }],
         'depends_on': depends_on,
     }
+    if environment:
+        pipeline.update({
+            'environment': environment,
+        })
+
     pipeline['volumes'].extend(volumes)
     pipeline.update(platform_conf)