[10.4.x] backport CI / release changes (#89631)

* Chore: Add go workspace (#83191) --------- Co-authored-by: ismail simsek <ismailsimsek09@gmail.com> * CI: Make pkg/build its own module, remove unused Grafana modules in go.mo… (#89243) * Make pkg/build its own module, remove unused Grafana modules in go.mod/go.sum * fix go.work format * log errors on file close errors * CI: Add release-pr workflow (#89005) * Add release-pr workflow * update CODEOWNERS * CI: Trigger release pr workflow when a release is completed (#89062) * Automation: Verify release artifacts on grafana.com (#89197) * baldm0mma/verify_release/ create verify_release_for_download function * baldm0mma/verify_release/ add name, image, env * baldm0mma/verify_release/ add initial commands * baldm0mma/verify_release/ add deps? * baldm0mma/verify_release/ update location * baldm0mma/verify_release/ add anno to lib-star * bald0mma/verify_release/ update func name to verify_grafanacom_step * baldm0mma/verify_release/ add verify shell script * baldm0mma/verify_release/ add script content, first attempt * baldm0mma/verify_release/ add node image to verify_grafanacom_step * baldm0mma/verify_release/ add gcom secret note * baldm0mma/verify_release/ add sudo to apt-get * baldm0mma/verify_release/ add anno * baldm0mma/verify_release/ add anno to secrets * baldm0mma/verify_release/ update commands to reflect node env image * baldm0mma/verify_release/ update annos * baldm0mma/verify_release/ update tag variable * baldm0mma/verify release/ add whitespace * baldm0mma/verify_releases/ update with no bash loops * baldm0mma/verify_release/ update exit logic * baldm0mma/verify_release/ remove annos * baldm0mma/verify_releasse/ resign and build yml * baldm0mma/verify_release/ remove annos * baldm0mma/verify_release/ update signature * baldm0mma/verify_release/ download curl * baldm0mma/verify_release/ remove temp key folder removal * baldm0mma/verify_release/ account for artifact download time * baldm0mma/verify_release/ add anno * baldm0mma/verify_release/ update location * baldm0mma/verify_release/ update script * baldm0mma/verify_release/ make drone * baldm0mma/verify_release/ update script for oss or ent * baldm0mma/verify_release/ add promotion option * baldm0mma/verify_release/ make drone * Update scripts/drone/events/release.star Co-authored-by: Kevin Minehart <5140827+kminehart@users.noreply.github.com> * Update scripts/drone/steps/lib.star Co-authored-by: Kevin Minehart <5140827+kminehart@users.noreply.github.com> * Update scripts/drone/steps/lib.star Co-authored-by: Kevin Minehart <5140827+kminehart@users.noreply.github.com> * baldm0mma/verify_release/ update drone * Update scripts/drone/events/release.star Co-authored-by: Kevin Minehart <5140827+kminehart@users.noreply.github.com> * baldm0mma/verify_release/ update drone * Update scripts/drone/steps/lib.star Co-authored-by: Kevin Minehart <5140827+kminehart@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: Kevin Minehart <5140827+kminehart@users.noreply.github.com> * baldm0mma/update drone * baldm0mma/verify_release/ update path * baldm0mma/verify_release/ make drone * baldm0mma/update drone * Apply suggestions from code review Co-authored-by: Kevin Minehart <5140827+kminehart@users.noreply.github.com> * baldm0mma/verify_release/ update for loop to account for failure * baldm0mma/verify_release/ make drone * baldm0mma/verify_release/ make format-drone * baldm0mma/verify_release/ rem unused var --------- Co-authored-by: Kevin Minehart <5140827+kminehart@users.noreply.github.com> --------- Co-authored-by: Todd Treece <360020+toddtreece@users.noreply.github.com> Co-authored-by: ismail simsek <ismailsimsek09@gmail.com> Co-authored-by: Jev Forsberg <46619047+baldm0mma@users.noreply.github.com>
2024-06-24 15:06:53 -05:00
parent e95b1e838c
commit 8e9aa45716
22 changed files with 1802 additions and 124 deletions
@@ -21,6 +21,7 @@ load(
    "remote_alertmanager_integration_tests_steps",
    "verify_gen_cue_step",
    "verify_gen_jsonnet_step",
+    "verify_grafanacom_step",
    "wire_install_step",
    "yarn_install_step",
 )
@@ -42,6 +43,7 @@ load(
 )

 ver_mode = "release"
+semver_regex = r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"

 def retrieve_npm_packages_step():
    return {
@@ -59,6 +61,34 @@ def retrieve_npm_packages_step():
        "commands": ["./bin/build artifacts npm retrieve --tag ${DRONE_TAG}"],
    }

+def release_pr_step(depends_on = []):
+    return {
+        "name": "create-release-pr",
+        "image": images["curl"],
+        "depends_on": depends_on,
+        "environment": {
+            "GITHUB_TOKEN": from_secret("github_token"),
+            "GH_CLI_URL": "https://github.com/cli/cli/releases/download/v2.50.0/gh_2.50.0_linux_amd64.tar.gz",
+        },
+        "commands": [
+            "apk add perl",
+            "v_target=`echo $${{TAG}} | perl -pe 's/{}/v\\1.\\2.x/'`".format(semver_regex),
+            "default_target=`if [[ -n $$LATEST ]]; then echo 'main'; else echo $$v_target; fi`",
+            "backport=`if [[ -n $$LATEST ]]; then echo $$v_target; fi`",
+            # Install gh CLI
+            "curl -L $${GH_CLI_URL} | tar -xz --strip-components=1 -C /usr",
+            # Run the release-pr workflow
+            "gh workflow run " +
+            "-f dry_run=$${DRY_RUN} " +
+            "-f version=$${TAG} " +
+            # If the submitter has set a target branch, then use that, otherwise use the default
+            "-f target=$${TARGET:-$default_target} " +
+            # If the submitter has set a backport branch, then use that, otherwise use the default
+            "-f backport=$${BACKPORT:-$default_backport} " +
+            "--repo=grafana/grafana release-pr.yml",
+        ],
+    }
+
 def release_npm_packages_step():
    return {
        "name": "release-npm-packages",
@@ -136,9 +166,20 @@ def publish_artifacts_pipelines(mode):
        publish_artifacts_step(),
        publish_static_assets_step(),
        publish_storybook_step(),
+        release_pr_step(depends_on = ["publish-artifacts", "publish-static-assets"]),
    ]

    return [
+        pipeline(
+            name = "create-release-pr",
+            trigger = {
+                "event": ["promote"],
+                "target": "release-pr",
+            },
+            steps = [
+                release_pr_step(),
+            ],
+        ),
        pipeline(
            name = "publish-artifacts-{}".format(mode),
            trigger = trigger,
@@ -163,6 +204,7 @@ def publish_packages_pipeline():
        publish_linux_packages_step(package_manager = "deb"),
        publish_linux_packages_step(package_manager = "rpm"),
        publish_grafanacom_step(ver_mode = "release"),
+        verify_grafanacom_step(),
    ]

    deps = [
@@ -171,6 +213,16 @@ def publish_packages_pipeline():
    ]

    return [
+        pipeline(
+            name = "verify-grafanacom-artifacts",
+            trigger = {
+                "event": ["promote"],
+                "target": "verify-grafanacom-artifacts",
+            },
+            steps = [
+                verify_grafanacom_step(depends_on = []),
+            ],
+        ),
        pipeline(
            name = "publish-packages",
            trigger = trigger,
@@ -506,7 +506,7 @@ def test_backend_step():
            # shared-mime-info and shared-mime-info-lang is used for exactly 1 test for the
            # mime.TypeByExtension function.
            "apk add --update build-base shared-mime-info shared-mime-info-lang",
-            "go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/...",
+            "go list -f '{{.Dir}}/...' -m | xargs go test -tags requires_buildifer -short -covermode=atomic -timeout=5m",
        ],
    }

@@ -958,7 +958,7 @@ def mysql_integration_tests_steps(hostname, version):
 def redis_integration_tests_steps():
    cmds = [
        "go clean -testcache",
-        "go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...",
+        "go list -f '{{.Dir}}/...' -m | xargs go test -run IntegrationRedis -covermode=atomic -timeout=2m",
    ]

    environment = {
@@ -983,7 +983,7 @@ def remote_alertmanager_integration_tests_steps():
 def memcached_integration_tests_steps():
    cmds = [
        "go clean -testcache",
-        "go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...",
+        "go list -f '{{.Dir}}/...' -m | xargs go test -run IntegrationMemcached -covermode=atomic -timeout=2m",
    ]

    environment = {
@@ -1096,6 +1096,34 @@ def publish_grafanacom_step(ver_mode):
        ],
    }

+def verify_grafanacom_step(depends_on = ["publish-grafanacom"]):
+    return {
+        "name": "verify-grafanacom",
+        "image": images["node"],
+        "commands": [
+            # Download and install `curl` and `bash` - both of which aren't available inside of the `node:{version}-alpine` docker image.
+            "apk add curl bash",
+
+            # There may be a slight lag between when artifacts are uploaded to Google Storage,
+            # and when they become available on the website. This `for` loop sould account for that discrepancy.
+            # We attempt the verification up to 5 times. If successful, exit the loop with a success (0) status.
+            # If any attempt fails, but it's not the final attempt, wait 60 seconds before the next attempt.
+            # If the 5th (final) attempt fails, exit with error (1) status.
+            """
+            for i in {1..5}; do
+                if ./scripts/drone/verify-grafanacom.sh; then
+                    exit 0
+                elif [ $i -eq 5 ]; then
+                    exit 1
+                else
+                    sleep 60
+                fi
+            done
+            """,
+        ],
+        "depends_on": depends_on,
+    }
+
 def publish_linux_packages_step(package_manager = "deb"):
    return {
        "name": "publish-linux-packages-{}".format(package_manager),
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+version=${1:-$TAG}
+
+# Construct the URL based on the provided version and edition
+if [ "$EDITION" = "enterprise" ]; then
+    url="https://grafana.com/api/downloads/grafana-enterprise/versions/$version"
+else
+    url="https://grafana.com/api/downloads/grafana/versions/$version"
+fi
+
+# Make a request to the GCOM API to retrieve the artifacts for the specified version. Exit if the request fails.
+if ! artifacts=$(curl "$url"); then
+  echo "Failed to retrieve artifact URLs from Grafana.com API. Please check the API key, authentication, edition, and version."
+  exit 1
+fi
+
+# Use Node.js to parse the JSON response and extract the download URLs
+url_string=$(node -e "
+  const artifacts = JSON.parse(JSON.stringify($artifacts));
+  const downloadUrls = artifacts.packages.map((package) => package.links.find((link) => link.rel === 'download').href);
+  console.log(downloadUrls.join(' '));
+")
+
+# Convert the url_string to a Bash array
+read -r -a urls <<< "$url_string"
+
+# If empty, no artifact URLs were found for the specified version. Exit with an error.
+if [ ${#urls[@]} -eq 0 ]; then
+  echo "No artifact URLs found for version $version. Please check the provided version."
+  exit 1
+fi
+
+# Iterate over the URLs and check the status code of each. If any URL does not return a 200 status code, add it to the failed_urls string.
+failed_urls=""
+for url in "${urls[@]}"; do
+  status_code=$(curl -L -s -o /dev/null -w "%{http_code}" "$url")
+  if [ "$status_code" -ne 200 ]; then
+    failed_urls+="$url\n"
+  fi
+done
+
+# If any URLs failed, print them and exit with an error.
+if [ -n "$failed_urls" ]; then
+  echo "The following URLs did not return a 200 status code:"
+  echo "$failed_urls"
+  exit 1
+else
+  echo "All URLs returned a 200 status code. Download links are valid for version $version."
+  exit 0
+fi