[v11.0.x] CI: Additional changes for +security versions (#94927)

CI: Additional changes for +security versions (#94854)

* Build: Fix docker manifest create not using correct IMAGE_TAG

* Support publishing security versions of NPM packages

---------

Co-authored-by: Andreas Christou <andreas.christou@grafana.com>
Co-authored-by: Kevin Minehart <kmineh0151@gmail.com>
Co-authored-by: Diego Augusto Molina <diegoaugustomolina@gmail.com>
(cherry picked from commit 8f7352e862)

Co-authored-by: Josh Hunt <joshhunt@users.noreply.github.com>
This commit is contained in:
Diego Augusto Molina
2024-10-17 17:20:20 -03:00
committed by GitHub
co-authored by Josh Hunt
parent f5a2695590
commit 10ea008462
8 changed files with 95 additions and 53 deletions
+38 -34
View File
@@ -495,7 +495,7 @@ steps:
name: identify-runner
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
@@ -902,7 +902,7 @@ steps:
name: clone-enterprise
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
@@ -1819,7 +1819,7 @@ steps:
name: identify-runner
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
@@ -2323,7 +2323,7 @@ services:
steps:
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
@@ -2527,7 +2527,7 @@ steps:
name: identify-runner
- commands:
- $$ProgressPreference = "SilentlyContinue"
- Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/windows/grabpl.exe
- Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/windows/grabpl.exe
-OutFile grabpl.exe
image: grafana/ci-wix:0.1.1
name: windows-init
@@ -2657,7 +2657,7 @@ steps:
name: identify-runner
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
@@ -2703,9 +2703,9 @@ steps:
$$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7
# Create the grafana manifests
$$debug docker manifest create grafana/grafana:${TAG} grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7
$$debug docker manifest create grafana/grafana:$${IMAGE_TAG} grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7
$$debug docker manifest create grafana/grafana:${TAG}-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7
$$debug docker manifest create grafana/grafana:$${IMAGE_TAG}-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7
# Push the grafana manifests
$$debug docker manifest push grafana/grafana:$${IMAGE_TAG}
@@ -2789,7 +2789,7 @@ steps:
name: identify-runner
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
@@ -2835,9 +2835,9 @@ steps:
$$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7
# Create the grafana manifests
$$debug docker manifest create grafana/grafana:${TAG} grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7
$$debug docker manifest create grafana/grafana:$${IMAGE_TAG} grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7
$$debug docker manifest create grafana/grafana:${TAG}-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7
$$debug docker manifest create grafana/grafana:$${IMAGE_TAG}-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7
# Push the grafana manifests
$$debug docker manifest push grafana/grafana:$${IMAGE_TAG}
@@ -3100,6 +3100,7 @@ platform:
services: []
steps:
- commands:
- export version=$(echo ${TAG} | sed -e "s/+security-/-/g")
- 'echo "Step 1: Updating package lists..."'
- apt-get update >/dev/null 2>&1
- 'echo "Step 2: Installing prerequisites..."'
@@ -3115,7 +3116,7 @@ steps:
- 'echo "Step 5: Installing Grafana..."'
- for i in $(seq 1 60); do
- ' if apt-get update >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get
install -yq grafana=${TAG} >/dev/null 2>&1; then'
install -yq grafana=$version >/dev/null 2>&1; then'
- ' echo "Command succeeded on attempt $i"'
- ' break'
- ' else'
@@ -3129,10 +3130,10 @@ steps:
- ' fi'
- done
- 'echo "Step 6: Verifying Grafana installation..."'
- 'if dpkg -s grafana | grep -q "Version: ${TAG}"; then'
- ' echo "Successfully verified Grafana version ${TAG}"'
- 'if dpkg -s grafana | grep -q "Version: $version"; then'
- ' echo "Successfully verified Grafana version $version"'
- else
- ' echo "Failed to verify Grafana version ${TAG}"'
- ' echo "Failed to verify Grafana version $version"'
- ' exit 1'
- fi
- echo "Verification complete."
@@ -3160,11 +3161,12 @@ steps:
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
' > /etc/yum.repos.d/grafana.repo
- 'echo "Step 5: Checking RPM repository..."'
- dnf list available grafana-${TAG}
- export version=$(echo "${TAG}" | sed -e "s/+security-/^security_/g")
- dnf list available grafana-$version
- if [ $? -eq 0 ]; then
- ' echo "Grafana package found in repository. Installing from repo..."'
- for i in $(seq 1 60); do
- ' if dnf install -y --nogpgcheck grafana-${TAG} >/dev/null 2>&1; then'
- ' if dnf install -y --nogpgcheck grafana-$version >/dev/null 2>&1; then'
- ' echo "Command succeeded on attempt $i"'
- ' break'
- ' else'
@@ -3181,16 +3183,16 @@ steps:
- ' rpm --import https://rpm.grafana.com/gpg.key'
- ' rpm -qa gpg-pubkey* | xargs rpm -qi | grep -i grafana'
- else
- ' echo "Grafana package version ${TAG} not found in repository."'
- ' echo "Grafana package version $version not found in repository."'
- ' dnf repolist'
- ' dnf list available grafana*'
- ' exit 1'
- fi
- 'echo "Step 6: Verifying Grafana installation..."'
- if rpm -q grafana | grep -q "${TAG}"; then
- ' echo "Successfully verified Grafana version ${TAG}"'
- if rpm -q grafana | grep -q "$verison"; then
- ' echo "Successfully verified Grafana version $version"'
- else
- ' echo "Failed to verify Grafana version ${TAG}"'
- ' echo "Failed to verify Grafana version $version"'
- ' exit 1'
- fi
- echo "Verification complete."
@@ -3277,6 +3279,7 @@ steps:
from_secret: packages_service_account
target_bucket: grafana-packages
- commands:
- export version=$(echo ${TAG} | sed -e "s/+security-/-/g")
- 'echo "Step 1: Updating package lists..."'
- apt-get update >/dev/null 2>&1
- 'echo "Step 2: Installing prerequisites..."'
@@ -3292,7 +3295,7 @@ steps:
- 'echo "Step 5: Installing Grafana..."'
- for i in $(seq 1 60); do
- ' if apt-get update >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get
install -yq grafana=${TAG} >/dev/null 2>&1; then'
install -yq grafana=$version >/dev/null 2>&1; then'
- ' echo "Command succeeded on attempt $i"'
- ' break'
- ' else'
@@ -3306,10 +3309,10 @@ steps:
- ' fi'
- done
- 'echo "Step 6: Verifying Grafana installation..."'
- 'if dpkg -s grafana | grep -q "Version: ${TAG}"; then'
- ' echo "Successfully verified Grafana version ${TAG}"'
- 'if dpkg -s grafana | grep -q "Version: $version"; then'
- ' echo "Successfully verified Grafana version $version"'
- else
- ' echo "Failed to verify Grafana version ${TAG}"'
- ' echo "Failed to verify Grafana version $version"'
- ' exit 1'
- fi
- echo "Verification complete."
@@ -3338,11 +3341,12 @@ steps:
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
' > /etc/yum.repos.d/grafana.repo
- 'echo "Step 5: Checking RPM repository..."'
- dnf list available grafana-${TAG}
- export version=$(echo "${TAG}" | sed -e "s/+security-/^security_/g")
- dnf list available grafana-$version
- if [ $? -eq 0 ]; then
- ' echo "Grafana package found in repository. Installing from repo..."'
- for i in $(seq 1 60); do
- ' if dnf install -y --nogpgcheck grafana-${TAG} >/dev/null 2>&1; then'
- ' if dnf install -y --nogpgcheck grafana-$version >/dev/null 2>&1; then'
- ' echo "Command succeeded on attempt $i"'
- ' break'
- ' else'
@@ -3359,16 +3363,16 @@ steps:
- ' rpm --import https://rpm.grafana.com/gpg.key'
- ' rpm -qa gpg-pubkey* | xargs rpm -qi | grep -i grafana'
- else
- ' echo "Grafana package version ${TAG} not found in repository."'
- ' echo "Grafana package version $version not found in repository."'
- ' dnf repolist'
- ' dnf list available grafana*'
- ' exit 1'
- fi
- 'echo "Step 6: Verifying Grafana installation..."'
- if rpm -q grafana | grep -q "${TAG}"; then
- ' echo "Successfully verified Grafana version ${TAG}"'
- if rpm -q grafana | grep -q "$verison"; then
- ' echo "Successfully verified Grafana version $version"'
- else
- ' echo "Failed to verify Grafana version ${TAG}"'
- ' echo "Failed to verify Grafana version $version"'
- ' exit 1'
- fi
- echo "Verification complete."
@@ -3623,7 +3627,7 @@ steps:
name: identify-runner
- commands:
- $$ProgressPreference = "SilentlyContinue"
- Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/windows/grabpl.exe
- Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/windows/grabpl.exe
-OutFile grabpl.exe
image: grafana/ci-wix:0.1.1
name: windows-init
@@ -4424,7 +4428,7 @@ services:
steps:
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.53/grabpl
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
@@ -5215,6 +5219,6 @@ kind: secret
name: gcr_credentials
---
kind: signature
hmac: d2767ab8ff45f3ebf6c8ab787d7138a8997fb1329ba185e9b2ac6b806e418454
hmac: b590c18e0ab745e48f2b56eb5004c9c351fca29b164f338c7ebf6a71054b82e1
...
+3
View File
@@ -145,6 +145,9 @@ func Builds(baseURL *url.URL, grafana, version string, packages []packaging.Buil
if arch == "aarch64" {
arch = "arm64"
}
if arch == "x86_64" {
arch = "amd64"
}
}
if v.Distro == "deb" {
+16
View File
@@ -2,6 +2,7 @@ package main
import (
"fmt"
"log"
"os"
"strings"
@@ -23,6 +24,11 @@ func NpmRetrieveAction(c *cli.Context) error {
return fmt.Errorf("no tag version specified, exitting")
}
if strings.Contains(tag, "security") {
log.Printf("skipping npm publish because version '%s' has 'security'", tag)
return nil
}
prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET"))
if prereleaseBucket == "" {
return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1)
@@ -48,6 +54,11 @@ func NpmStoreAction(c *cli.Context) error {
return fmt.Errorf("no tag version specified, exiting")
}
if strings.Contains(tag, "security") {
log.Printf("skipping npm publish because version '%s' has 'security'", tag)
return nil
}
prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET"))
if prereleaseBucket == "" {
return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1)
@@ -73,6 +84,11 @@ func NpmReleaseAction(c *cli.Context) error {
return fmt.Errorf("no tag version specified, exitting")
}
if strings.Contains(tag, "security") {
log.Printf("skipping npm publish because version '%s' has 'security'", tag)
return nil
}
err := npm.PublishNpmPackages(c.Context, tag)
if err != nil {
return err
+10
View File
@@ -113,6 +113,16 @@ var ARMArtifacts = []BuildArtifact{
Arch: "armv7",
Ext: "tar.gz",
},
{
Distro: "linux",
Arch: "arm64",
Ext: "tar.gz",
},
{
Distro: "linux",
Arch: "amd64",
Ext: "tar.gz",
},
}
func join(a []BuildArtifact, b ...[]BuildArtifact) []BuildArtifact {
+13 -6
View File
@@ -13,15 +13,17 @@ import (
)
var (
reGrafanaTag = regexp.MustCompile(`^v(\d+\.\d+\.\d+$)`)
reGrafanaTagPreview = regexp.MustCompile(`^v(\d+\.\d+\.\d+-preview)`)
reGrafanaTagCustom = regexp.MustCompile(`^v(\d+\.\d+\.\d+-\w+)`)
reGrafanaTag = regexp.MustCompile(`^v(\d+\.\d+\.\d+$)`)
reGrafanaTagPreview = regexp.MustCompile(`^v(\d+\.\d+\.\d+-preview)`)
reGrafanaTagCustom = regexp.MustCompile(`^v(\d+\.\d+\.\d+-\w+)`)
reGrafanaTagSecurity = regexp.MustCompile(`^v(\d+\.\d+\.\d+\+\w+\-\d+)`)
)
const (
Latest = "latest"
Next = "next"
Test = "test"
Latest = "latest"
Next = "next"
Test = "test"
Security = "security"
)
type Version struct {
@@ -152,6 +154,11 @@ func GetVersion(tag string) (*Version, error) {
Version: reGrafanaTagCustom.FindStringSubmatch(tag)[1],
Channel: Test,
}
case reGrafanaTagSecurity.MatchString(tag):
version = Version{
Version: reGrafanaTagSecurity.FindStringSubmatch(tag)[1],
Channel: Security,
}
default:
return nil, fmt.Errorf("%s not a supported Grafana version, exitting", tag)
}
+2 -2
View File
@@ -45,12 +45,12 @@ def publish_image_public_step():
$$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7
# Create the grafana manifests
$$debug docker manifest create grafana/grafana:${TAG} \
$$debug docker manifest create grafana/grafana:$${IMAGE_TAG} \
grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 \
grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 \
grafana/grafana-image-tags:$${IMAGE_TAG}-armv7
$$debug docker manifest create grafana/grafana:${TAG}-ubuntu \
$$debug docker manifest create grafana/grafana:$${IMAGE_TAG}-ubuntu \
grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 \
grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 \
grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7
+12 -10
View File
@@ -1241,13 +1241,14 @@ def retry_command(command, attempts = 60, delay = 30):
]
def verify_linux_DEB_packages_step(depends_on = []):
install_command = "apt-get update >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get install -yq grafana=${TAG} >/dev/null 2>&1"
install_command = "apt-get update >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get install -yq grafana=$version >/dev/null 2>&1"
return {
"name": "verify-linux-DEB-packages",
"image": images["ubuntu"],
"environment": {},
"commands": [
'export version=$(echo ${TAG} | sed -e "s/+security-/-/g")',
'echo "Step 1: Updating package lists..."',
"apt-get update >/dev/null 2>&1",
'echo "Step 2: Installing prerequisites..."',
@@ -1261,10 +1262,10 @@ def verify_linux_DEB_packages_step(depends_on = []):
# The packages take a bit of time to propogate within the repo. This retry will check their availability within 10 minutes.
] + retry_command(install_command) + [
'echo "Step 6: Verifying Grafana installation..."',
'if dpkg -s grafana | grep -q "Version: ${TAG}"; then',
' echo "Successfully verified Grafana version ${TAG}"',
'if dpkg -s grafana | grep -q "Version: $version"; then',
' echo "Successfully verified Grafana version $version"',
"else",
' echo "Failed to verify Grafana version ${TAG}"',
' echo "Failed to verify Grafana version $version"',
" exit 1",
"fi",
'echo "Verification complete."',
@@ -1285,7 +1286,7 @@ def verify_linux_RPM_packages_step(depends_on = []):
"sslcacert=/etc/pki/tls/certs/ca-bundle.crt\n"
)
install_command = "dnf install -y --nogpgcheck grafana-${TAG} >/dev/null 2>&1"
install_command = "dnf install -y --nogpgcheck grafana-$version >/dev/null 2>&1"
return {
"name": "verify-linux-RPM-packages",
@@ -1301,7 +1302,8 @@ def verify_linux_RPM_packages_step(depends_on = []):
'echo "Step 4: Configuring Grafana repository..."',
"echo -e '" + repo_config + "' > /etc/yum.repos.d/grafana.repo",
'echo "Step 5: Checking RPM repository..."',
"dnf list available grafana-${TAG}",
'export version=$(echo "${TAG}" | sed -e "s/+security-/^security_/g")',
"dnf list available grafana-$version",
"if [ $? -eq 0 ]; then",
' echo "Grafana package found in repository. Installing from repo..."',
] + retry_command(install_command) + [
@@ -1309,16 +1311,16 @@ def verify_linux_RPM_packages_step(depends_on = []):
" rpm --import https://rpm.grafana.com/gpg.key",
" rpm -qa gpg-pubkey* | xargs rpm -qi | grep -i grafana",
"else",
' echo "Grafana package version ${TAG} not found in repository."',
' echo "Grafana package version $version not found in repository."',
" dnf repolist",
" dnf list available grafana*",
" exit 1",
"fi",
'echo "Step 6: Verifying Grafana installation..."',
'if rpm -q grafana | grep -q "${TAG}"; then',
' echo "Successfully verified Grafana version ${TAG}"',
'if rpm -q grafana | grep -q "$verison"; then',
' echo "Successfully verified Grafana version $version"',
"else",
' echo "Failed to verify Grafana version ${TAG}"',
' echo "Failed to verify Grafana version $version"',
" exit 1",
"fi",
'echo "Verification complete."',
+1 -1
View File
@@ -2,7 +2,7 @@
global variables
"""
grabpl_version = "v3.0.53"
grabpl_version = "v3.0.56"
golang_version = "1.22.7"
# nodejs_version should match what's in ".nvmrc", but without the v prefix.