From 10ea0084620139f611ecb59680142caf395d662d Mon Sep 17 00:00:00 2001 From: Diego Augusto Molina Date: Thu, 17 Oct 2024 17:20:20 -0300 Subject: [PATCH] [v11.0.x] CI: Additional changes for +security versions (#94927) CI: Additional changes for +security versions (#94854) * Build: Fix docker manifest create not using correct IMAGE_TAG * Support publishing security versions of NPM packages --------- Co-authored-by: Andreas Christou Co-authored-by: Kevin Minehart Co-authored-by: Diego Augusto Molina (cherry picked from commit 8f7352e862c62d411cda17b8523eea5c55a523c2) Co-authored-by: Josh Hunt --- .drone.yml | 72 +++++++++++---------- pkg/build/cmd/grafanacom.go | 3 + pkg/build/cmd/npm.go | 16 +++++ pkg/build/packaging/artifacts.go | 10 +++ pkg/build/versions/version.go | 19 ++++-- scripts/drone/pipelines/publish_images.star | 4 +- scripts/drone/steps/lib.star | 22 ++++--- scripts/drone/variables.star | 2 +- 8 files changed, 95 insertions(+), 53 deletions(-) diff --git a/.drone.yml b/.drone.yml index cf1a62b9867..f63d4619758 100644 --- a/.drone.yml +++ b/.drone.yml @@ -495,7 +495,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -902,7 +902,7 @@ steps: name: clone-enterprise - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1819,7 +1819,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2323,7 +2323,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2527,7 +2527,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: windows-init @@ -2657,7 +2657,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2703,9 +2703,9 @@ steps: $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 # Create the grafana manifests - $$debug docker manifest create grafana/grafana:${TAG} grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 + $$debug docker manifest create grafana/grafana:$${IMAGE_TAG} grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 - $$debug docker manifest create grafana/grafana:${TAG}-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 + $$debug docker manifest create grafana/grafana:$${IMAGE_TAG}-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 # Push the grafana manifests $$debug docker manifest push grafana/grafana:$${IMAGE_TAG} @@ -2789,7 +2789,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2835,9 +2835,9 @@ steps: $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 # Create the grafana manifests - $$debug docker manifest create grafana/grafana:${TAG} grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 + $$debug docker manifest create grafana/grafana:$${IMAGE_TAG} grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 - $$debug docker manifest create grafana/grafana:${TAG}-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 + $$debug docker manifest create grafana/grafana:$${IMAGE_TAG}-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 # Push the grafana manifests $$debug docker manifest push grafana/grafana:$${IMAGE_TAG} @@ -3100,6 +3100,7 @@ platform: services: [] steps: - commands: + - export version=$(echo ${TAG} | sed -e "s/+security-/-/g") - 'echo "Step 1: Updating package lists..."' - apt-get update >/dev/null 2>&1 - 'echo "Step 2: Installing prerequisites..."' @@ -3115,7 +3116,7 @@ steps: - 'echo "Step 5: Installing Grafana..."' - for i in $(seq 1 60); do - ' if apt-get update >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get - install -yq grafana=${TAG} >/dev/null 2>&1; then' + install -yq grafana=$version >/dev/null 2>&1; then' - ' echo "Command succeeded on attempt $i"' - ' break' - ' else' @@ -3129,10 +3130,10 @@ steps: - ' fi' - done - 'echo "Step 6: Verifying Grafana installation..."' - - 'if dpkg -s grafana | grep -q "Version: ${TAG}"; then' - - ' echo "Successfully verified Grafana version ${TAG}"' + - 'if dpkg -s grafana | grep -q "Version: $version"; then' + - ' echo "Successfully verified Grafana version $version"' - else - - ' echo "Failed to verify Grafana version ${TAG}"' + - ' echo "Failed to verify Grafana version $version"' - ' exit 1' - fi - echo "Verification complete." @@ -3160,11 +3161,12 @@ steps: sslcacert=/etc/pki/tls/certs/ca-bundle.crt ' > /etc/yum.repos.d/grafana.repo - 'echo "Step 5: Checking RPM repository..."' - - dnf list available grafana-${TAG} + - export version=$(echo "${TAG}" | sed -e "s/+security-/^security_/g") + - dnf list available grafana-$version - if [ $? -eq 0 ]; then - ' echo "Grafana package found in repository. Installing from repo..."' - for i in $(seq 1 60); do - - ' if dnf install -y --nogpgcheck grafana-${TAG} >/dev/null 2>&1; then' + - ' if dnf install -y --nogpgcheck grafana-$version >/dev/null 2>&1; then' - ' echo "Command succeeded on attempt $i"' - ' break' - ' else' @@ -3181,16 +3183,16 @@ steps: - ' rpm --import https://rpm.grafana.com/gpg.key' - ' rpm -qa gpg-pubkey* | xargs rpm -qi | grep -i grafana' - else - - ' echo "Grafana package version ${TAG} not found in repository."' + - ' echo "Grafana package version $version not found in repository."' - ' dnf repolist' - ' dnf list available grafana*' - ' exit 1' - fi - 'echo "Step 6: Verifying Grafana installation..."' - - if rpm -q grafana | grep -q "${TAG}"; then - - ' echo "Successfully verified Grafana version ${TAG}"' + - if rpm -q grafana | grep -q "$verison"; then + - ' echo "Successfully verified Grafana version $version"' - else - - ' echo "Failed to verify Grafana version ${TAG}"' + - ' echo "Failed to verify Grafana version $version"' - ' exit 1' - fi - echo "Verification complete." @@ -3277,6 +3279,7 @@ steps: from_secret: packages_service_account target_bucket: grafana-packages - commands: + - export version=$(echo ${TAG} | sed -e "s/+security-/-/g") - 'echo "Step 1: Updating package lists..."' - apt-get update >/dev/null 2>&1 - 'echo "Step 2: Installing prerequisites..."' @@ -3292,7 +3295,7 @@ steps: - 'echo "Step 5: Installing Grafana..."' - for i in $(seq 1 60); do - ' if apt-get update >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get - install -yq grafana=${TAG} >/dev/null 2>&1; then' + install -yq grafana=$version >/dev/null 2>&1; then' - ' echo "Command succeeded on attempt $i"' - ' break' - ' else' @@ -3306,10 +3309,10 @@ steps: - ' fi' - done - 'echo "Step 6: Verifying Grafana installation..."' - - 'if dpkg -s grafana | grep -q "Version: ${TAG}"; then' - - ' echo "Successfully verified Grafana version ${TAG}"' + - 'if dpkg -s grafana | grep -q "Version: $version"; then' + - ' echo "Successfully verified Grafana version $version"' - else - - ' echo "Failed to verify Grafana version ${TAG}"' + - ' echo "Failed to verify Grafana version $version"' - ' exit 1' - fi - echo "Verification complete." @@ -3338,11 +3341,12 @@ steps: sslcacert=/etc/pki/tls/certs/ca-bundle.crt ' > /etc/yum.repos.d/grafana.repo - 'echo "Step 5: Checking RPM repository..."' - - dnf list available grafana-${TAG} + - export version=$(echo "${TAG}" | sed -e "s/+security-/^security_/g") + - dnf list available grafana-$version - if [ $? -eq 0 ]; then - ' echo "Grafana package found in repository. Installing from repo..."' - for i in $(seq 1 60); do - - ' if dnf install -y --nogpgcheck grafana-${TAG} >/dev/null 2>&1; then' + - ' if dnf install -y --nogpgcheck grafana-$version >/dev/null 2>&1; then' - ' echo "Command succeeded on attempt $i"' - ' break' - ' else' @@ -3359,16 +3363,16 @@ steps: - ' rpm --import https://rpm.grafana.com/gpg.key' - ' rpm -qa gpg-pubkey* | xargs rpm -qi | grep -i grafana' - else - - ' echo "Grafana package version ${TAG} not found in repository."' + - ' echo "Grafana package version $version not found in repository."' - ' dnf repolist' - ' dnf list available grafana*' - ' exit 1' - fi - 'echo "Step 6: Verifying Grafana installation..."' - - if rpm -q grafana | grep -q "${TAG}"; then - - ' echo "Successfully verified Grafana version ${TAG}"' + - if rpm -q grafana | grep -q "$verison"; then + - ' echo "Successfully verified Grafana version $version"' - else - - ' echo "Failed to verify Grafana version ${TAG}"' + - ' echo "Failed to verify Grafana version $version"' - ' exit 1' - fi - echo "Verification complete." @@ -3623,7 +3627,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: windows-init @@ -4424,7 +4428,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -5215,6 +5219,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: d2767ab8ff45f3ebf6c8ab787d7138a8997fb1329ba185e9b2ac6b806e418454 +hmac: b590c18e0ab745e48f2b56eb5004c9c351fca29b164f338c7ebf6a71054b82e1 ... diff --git a/pkg/build/cmd/grafanacom.go b/pkg/build/cmd/grafanacom.go index 305aa1195e8..ab53217f7a4 100644 --- a/pkg/build/cmd/grafanacom.go +++ b/pkg/build/cmd/grafanacom.go @@ -145,6 +145,9 @@ func Builds(baseURL *url.URL, grafana, version string, packages []packaging.Buil if arch == "aarch64" { arch = "arm64" } + if arch == "x86_64" { + arch = "amd64" + } } if v.Distro == "deb" { diff --git a/pkg/build/cmd/npm.go b/pkg/build/cmd/npm.go index ba35679f57a..4660b22b636 100644 --- a/pkg/build/cmd/npm.go +++ b/pkg/build/cmd/npm.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "log" "os" "strings" @@ -23,6 +24,11 @@ func NpmRetrieveAction(c *cli.Context) error { return fmt.Errorf("no tag version specified, exitting") } + if strings.Contains(tag, "security") { + log.Printf("skipping npm publish because version '%s' has 'security'", tag) + return nil + } + prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET")) if prereleaseBucket == "" { return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1) @@ -48,6 +54,11 @@ func NpmStoreAction(c *cli.Context) error { return fmt.Errorf("no tag version specified, exiting") } + if strings.Contains(tag, "security") { + log.Printf("skipping npm publish because version '%s' has 'security'", tag) + return nil + } + prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET")) if prereleaseBucket == "" { return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1) @@ -73,6 +84,11 @@ func NpmReleaseAction(c *cli.Context) error { return fmt.Errorf("no tag version specified, exitting") } + if strings.Contains(tag, "security") { + log.Printf("skipping npm publish because version '%s' has 'security'", tag) + return nil + } + err := npm.PublishNpmPackages(c.Context, tag) if err != nil { return err diff --git a/pkg/build/packaging/artifacts.go b/pkg/build/packaging/artifacts.go index 1dc60f18a2b..b1740b68c5f 100644 --- a/pkg/build/packaging/artifacts.go +++ b/pkg/build/packaging/artifacts.go @@ -113,6 +113,16 @@ var ARMArtifacts = []BuildArtifact{ Arch: "armv7", Ext: "tar.gz", }, + { + Distro: "linux", + Arch: "arm64", + Ext: "tar.gz", + }, + { + Distro: "linux", + Arch: "amd64", + Ext: "tar.gz", + }, } func join(a []BuildArtifact, b ...[]BuildArtifact) []BuildArtifact { diff --git a/pkg/build/versions/version.go b/pkg/build/versions/version.go index 5197559b95a..240c750bf68 100644 --- a/pkg/build/versions/version.go +++ b/pkg/build/versions/version.go @@ -13,15 +13,17 @@ import ( ) var ( - reGrafanaTag = regexp.MustCompile(`^v(\d+\.\d+\.\d+$)`) - reGrafanaTagPreview = regexp.MustCompile(`^v(\d+\.\d+\.\d+-preview)`) - reGrafanaTagCustom = regexp.MustCompile(`^v(\d+\.\d+\.\d+-\w+)`) + reGrafanaTag = regexp.MustCompile(`^v(\d+\.\d+\.\d+$)`) + reGrafanaTagPreview = regexp.MustCompile(`^v(\d+\.\d+\.\d+-preview)`) + reGrafanaTagCustom = regexp.MustCompile(`^v(\d+\.\d+\.\d+-\w+)`) + reGrafanaTagSecurity = regexp.MustCompile(`^v(\d+\.\d+\.\d+\+\w+\-\d+)`) ) const ( - Latest = "latest" - Next = "next" - Test = "test" + Latest = "latest" + Next = "next" + Test = "test" + Security = "security" ) type Version struct { @@ -152,6 +154,11 @@ func GetVersion(tag string) (*Version, error) { Version: reGrafanaTagCustom.FindStringSubmatch(tag)[1], Channel: Test, } + case reGrafanaTagSecurity.MatchString(tag): + version = Version{ + Version: reGrafanaTagSecurity.FindStringSubmatch(tag)[1], + Channel: Security, + } default: return nil, fmt.Errorf("%s not a supported Grafana version, exitting", tag) } diff --git a/scripts/drone/pipelines/publish_images.star b/scripts/drone/pipelines/publish_images.star index 9ed847a22fb..a947d05d0b4 100644 --- a/scripts/drone/pipelines/publish_images.star +++ b/scripts/drone/pipelines/publish_images.star @@ -45,12 +45,12 @@ def publish_image_public_step(): $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 # Create the grafana manifests - $$debug docker manifest create grafana/grafana:${TAG} \ + $$debug docker manifest create grafana/grafana:$${IMAGE_TAG} \ grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 \ grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 \ grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 - $$debug docker manifest create grafana/grafana:${TAG}-ubuntu \ + $$debug docker manifest create grafana/grafana:$${IMAGE_TAG}-ubuntu \ grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 \ grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 \ grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 7ac9df866e7..cfb5e2ec09e 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -1241,13 +1241,14 @@ def retry_command(command, attempts = 60, delay = 30): ] def verify_linux_DEB_packages_step(depends_on = []): - install_command = "apt-get update >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get install -yq grafana=${TAG} >/dev/null 2>&1" + install_command = "apt-get update >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get install -yq grafana=$version >/dev/null 2>&1" return { "name": "verify-linux-DEB-packages", "image": images["ubuntu"], "environment": {}, "commands": [ + 'export version=$(echo ${TAG} | sed -e "s/+security-/-/g")', 'echo "Step 1: Updating package lists..."', "apt-get update >/dev/null 2>&1", 'echo "Step 2: Installing prerequisites..."', @@ -1261,10 +1262,10 @@ def verify_linux_DEB_packages_step(depends_on = []): # The packages take a bit of time to propogate within the repo. This retry will check their availability within 10 minutes. ] + retry_command(install_command) + [ 'echo "Step 6: Verifying Grafana installation..."', - 'if dpkg -s grafana | grep -q "Version: ${TAG}"; then', - ' echo "Successfully verified Grafana version ${TAG}"', + 'if dpkg -s grafana | grep -q "Version: $version"; then', + ' echo "Successfully verified Grafana version $version"', "else", - ' echo "Failed to verify Grafana version ${TAG}"', + ' echo "Failed to verify Grafana version $version"', " exit 1", "fi", 'echo "Verification complete."', @@ -1285,7 +1286,7 @@ def verify_linux_RPM_packages_step(depends_on = []): "sslcacert=/etc/pki/tls/certs/ca-bundle.crt\n" ) - install_command = "dnf install -y --nogpgcheck grafana-${TAG} >/dev/null 2>&1" + install_command = "dnf install -y --nogpgcheck grafana-$version >/dev/null 2>&1" return { "name": "verify-linux-RPM-packages", @@ -1301,7 +1302,8 @@ def verify_linux_RPM_packages_step(depends_on = []): 'echo "Step 4: Configuring Grafana repository..."', "echo -e '" + repo_config + "' > /etc/yum.repos.d/grafana.repo", 'echo "Step 5: Checking RPM repository..."', - "dnf list available grafana-${TAG}", + 'export version=$(echo "${TAG}" | sed -e "s/+security-/^security_/g")', + "dnf list available grafana-$version", "if [ $? -eq 0 ]; then", ' echo "Grafana package found in repository. Installing from repo..."', ] + retry_command(install_command) + [ @@ -1309,16 +1311,16 @@ def verify_linux_RPM_packages_step(depends_on = []): " rpm --import https://rpm.grafana.com/gpg.key", " rpm -qa gpg-pubkey* | xargs rpm -qi | grep -i grafana", "else", - ' echo "Grafana package version ${TAG} not found in repository."', + ' echo "Grafana package version $version not found in repository."', " dnf repolist", " dnf list available grafana*", " exit 1", "fi", 'echo "Step 6: Verifying Grafana installation..."', - 'if rpm -q grafana | grep -q "${TAG}"; then', - ' echo "Successfully verified Grafana version ${TAG}"', + 'if rpm -q grafana | grep -q "$verison"; then', + ' echo "Successfully verified Grafana version $version"', "else", - ' echo "Failed to verify Grafana version ${TAG}"', + ' echo "Failed to verify Grafana version $version"', " exit 1", "fi", 'echo "Verification complete."', diff --git a/scripts/drone/variables.star b/scripts/drone/variables.star index 653466ec77d..0171396effb 100644 --- a/scripts/drone/variables.star +++ b/scripts/drone/variables.star @@ -2,7 +2,7 @@ global variables """ -grabpl_version = "v3.0.53" +grabpl_version = "v3.0.56" golang_version = "1.22.7" # nodejs_version should match what's in ".nvmrc", but without the v prefix.