From fd6bd298c20187a78e5873e167c8042bb38ae59a Mon Sep 17 00:00:00 2001 From: Kevin Minehart <5140827+kminehart@users.noreply.github.com> Date: Thu, 18 Jul 2024 13:18:30 -0500 Subject: [PATCH] [v10.4.x] CI: replace publish image step with a much simpler one that supports LATEST (#90619) CI: replace publish image step with a much simpler one that supports LATEST (#90498) * replace publish image step with a much simpler one that supports LATEST (cherry picked from commit 6a2a6b0fbcc04b735734c3ffa0c2361756560c03) --- .drone.yml | 130 ++++++++++++++++++-- scripts/drone/pipelines/publish_images.star | 117 ++++++++++++++---- scripts/drone/utils/images.star | 1 + 3 files changed, 212 insertions(+), 36 deletions(-) diff --git a/.drone.yml b/.drone.yml index 2df913a16f9..f1e81e593a0 100644 --- a/.drone.yml +++ b/.drone.yml @@ -2593,8 +2593,26 @@ steps: - name: docker path: /var/run/docker.sock - commands: - - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana --version-tag - ${DRONE_TAG} + - "\n debug=\n if [[ -n $${DRY_RUN} ]]; then debug=echo; fi\n docker login + -u $${DOCKER_USER} -p $${DOCKER_PASSWORD}\n\n # Push the grafana-image-tags + images\n $debug docker push grafana/grafana-image-tags:$${TAG}-amd64\n $debug + docker push grafana/grafana-image-tags:$${TAG}-arm64\n $debug docker push grafana/grafana-image-tags:$${TAG}-armv7\n + \ $debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-amd64\n $debug + docker push grafana/grafana-image-tags:$${TAG}-ubuntu-arm64\n $debug docker + push grafana/grafana-image-tags:$${TAG}-ubuntu-armv7\n\n # Create the grafana + manifests\n $debug docker manifest create grafana/grafana:${TAG} grafana/grafana-image-tags:$${TAG}-amd64 + \ grafana/grafana-image-tags:$${TAG}-arm64 grafana/grafana-image-tags:$${TAG}-armv7\n\n + \ $debug docker manifest create grafana/grafana:${TAG}-ubuntu grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 + \ grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${TAG}-ubuntu-armv7\n\n + \ # Push the grafana manifests\n $debug docker manifest push grafana/grafana:$${TAG}\n + \ $debug docker manifest push grafana/grafana:$${TAG}-ubuntu\n\n # if LATEST + is set, then also create & push latest\n if [[ -n $${LATEST} ]]; then\n $debug + docker manifest create grafana/grafana:latest grafana/grafana-image-tags:$${TAG}-amd64 + \ grafana/grafana-image-tags:$${TAG}-arm64 grafana/grafana-image-tags:$${TAG}-armv7\n + \ $debug docker manifest create grafana/grafana:latest-ubuntu grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 + \ grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${TAG}-ubuntu-armv7\n\n + \ $debug docker manifest push grafana/grafana:latest\n $debug docker + manifest push grafana/grafana:latest-ubuntu\n\n fi\n " depends_on: - fetch-images environment: @@ -2602,15 +2620,7 @@ steps: from_secret: docker_password DOCKER_USER: from_secret: docker_username - GCP_KEY: - from_secret: gcp_grafanauploads - GITHUB_APP_ID: - from_secret: delivery-bot-app-id - GITHUB_APP_INSTALLATION_ID: - from_secret: delivery-bot-app-installation-id - GITHUB_APP_PRIVATE_KEY: - from_secret: delivery-bot-app-private-key - image: google/cloud-sdk:431.0.0 + image: docker:27-cli name: publish-images-grafana volumes: - name: docker @@ -2652,6 +2662,100 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: oss +image_pull_secrets: +- gcr +- gar +kind: pipeline +name: manually-publish-docker-public +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.19.1 + name: identify-runner +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.50/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] + environment: + CGO_ENABLED: 0 + image: golang:1.21.10-alpine + name: compile-build-cmd +- commands: + - ./bin/build artifacts docker fetch --edition oss + depends_on: + - compile-build-cmd + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_grafanauploads + image: google/cloud-sdk:431.0.0 + name: fetch-images + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - "\n debug=\n if [[ -n $${DRY_RUN} ]]; then debug=echo; fi\n docker login + -u $${DOCKER_USER} -p $${DOCKER_PASSWORD}\n\n # Push the grafana-image-tags + images\n $debug docker push grafana/grafana-image-tags:$${TAG}-amd64\n $debug + docker push grafana/grafana-image-tags:$${TAG}-arm64\n $debug docker push grafana/grafana-image-tags:$${TAG}-armv7\n + \ $debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-amd64\n $debug + docker push grafana/grafana-image-tags:$${TAG}-ubuntu-arm64\n $debug docker + push grafana/grafana-image-tags:$${TAG}-ubuntu-armv7\n\n # Create the grafana + manifests\n $debug docker manifest create grafana/grafana:${TAG} grafana/grafana-image-tags:$${TAG}-amd64 + \ grafana/grafana-image-tags:$${TAG}-arm64 grafana/grafana-image-tags:$${TAG}-armv7\n\n + \ $debug docker manifest create grafana/grafana:${TAG}-ubuntu grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 + \ grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${TAG}-ubuntu-armv7\n\n + \ # Push the grafana manifests\n $debug docker manifest push grafana/grafana:$${TAG}\n + \ $debug docker manifest push grafana/grafana:$${TAG}-ubuntu\n\n # if LATEST + is set, then also create & push latest\n if [[ -n $${LATEST} ]]; then\n $debug + docker manifest create grafana/grafana:latest grafana/grafana-image-tags:$${TAG}-amd64 + \ grafana/grafana-image-tags:$${TAG}-arm64 grafana/grafana-image-tags:$${TAG}-armv7\n + \ $debug docker manifest create grafana/grafana:latest-ubuntu grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 + \ grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${TAG}-ubuntu-armv7\n\n + \ $debug docker manifest push grafana/grafana:latest\n $debug docker + manifest push grafana/grafana:latest-ubuntu\n\n fi\n " + depends_on: + - fetch-images + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + image: docker:27-cli + name: publish-images-grafana + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + event: + - promote + target: + - publish-docker-public +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: [] image_pull_secrets: - gcr - gar @@ -4853,6 +4957,7 @@ steps: - name: config path: /root/.docker/ - commands: + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM docker:27-cli - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine/git:2.40.1 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM golang:1.21.10-alpine - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM node:20.9.0-alpine @@ -4888,6 +4993,7 @@ steps: - name: config path: /root/.docker/ - commands: + - trivy --exit-code 1 --severity HIGH,CRITICAL docker:27-cli - trivy --exit-code 1 --severity HIGH,CRITICAL alpine/git:2.40.1 - trivy --exit-code 1 --severity HIGH,CRITICAL golang:1.21.10-alpine - trivy --exit-code 1 --severity HIGH,CRITICAL node:20.9.0-alpine @@ -5144,6 +5250,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: 9769007f8bab9d525f5ce496355d9f599b4a6be072a485ca7aef6d06d6aa5358 +hmac: b49df1dd4738902619ac6c78b56b6a6d125fb59f92e255fef37a78ea3cc15c4b ... diff --git a/scripts/drone/pipelines/publish_images.star b/scripts/drone/pipelines/publish_images.star index 4f099eba389..060e14598fb 100644 --- a/scripts/drone/pipelines/publish_images.star +++ b/scripts/drone/pipelines/publish_images.star @@ -10,31 +10,80 @@ load( "identify_runner_step", "publish_images_step", ) +load( + "scripts/drone/utils/images.star", + "images", +) load( "scripts/drone/utils/utils.star", "pipeline", ) +load( + "scripts/drone/vault.star", + "from_secret", +) -def publish_image_steps(docker_repo): - """Generates the steps used for publising Docker images using grabpl. - - Args: - docker_repo: the Docker image name. - It is combined with the 'grafana/' library prefix. +def publish_image_public_step(): + """Returns a step which publishes images Returns: - List of Drone steps. + A drone step which publishes Docker images for a public release. """ - steps = [ - identify_runner_step(), - download_grabpl_step(), - compile_build_cmd(), - fetch_images_step(), - publish_images_step("release", docker_repo), - publish_images_step("release", "grafana-oss"), - ] + command = """ + debug= + if [[ -n $${DRY_RUN} ]]; then debug=echo; fi + docker login -u $${DOCKER_USER} -p $${DOCKER_PASSWORD} - return steps + # Push the grafana-image-tags images + $debug docker push grafana/grafana-image-tags:$${TAG}-amd64 + $debug docker push grafana/grafana-image-tags:$${TAG}-arm64 + $debug docker push grafana/grafana-image-tags:$${TAG}-armv7 + $debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 + $debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 + $debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + + # Create the grafana manifests + $debug docker manifest create grafana/grafana:${TAG} \ + grafana/grafana-image-tags:$${TAG}-amd64 \ + grafana/grafana-image-tags:$${TAG}-arm64 \ + grafana/grafana-image-tags:$${TAG}-armv7 + + $debug docker manifest create grafana/grafana:${TAG}-ubuntu \ + grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 \ + grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 \ + grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + + # Push the grafana manifests + $debug docker manifest push grafana/grafana:$${TAG} + $debug docker manifest push grafana/grafana:$${TAG}-ubuntu + + # if LATEST is set, then also create & push latest + if [[ -n $${LATEST} ]]; then + $debug docker manifest create grafana/grafana:latest \ + grafana/grafana-image-tags:$${TAG}-amd64 \ + grafana/grafana-image-tags:$${TAG}-arm64 \ + grafana/grafana-image-tags:$${TAG}-armv7 + $debug docker manifest create grafana/grafana:latest-ubuntu \ + grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 \ + grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 \ + grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + + $debug docker manifest push grafana/grafana:latest + $debug docker manifest push grafana/grafana:latest-ubuntu + + fi + """ + return { + "environment": { + "DOCKER_USER": from_secret("docker_username"), + "DOCKER_PASSWORD": from_secret("docker_password"), + }, + "name": "publish-images-grafana", + "image": images["docker"], + "depends_on": ["fetch-images"], + "commands": [command], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + } def publish_image_pipelines_public(): """Generates the pipeline used for publising public Docker images. @@ -42,16 +91,36 @@ def publish_image_pipelines_public(): Returns: Drone pipeline """ - mode = "public" - trigger = { - "event": ["promote"], - "target": [mode], - } return [ pipeline( - name = "publish-docker-{}".format(mode), - trigger = trigger, - steps = publish_image_steps(docker_repo = "grafana"), + name = "publish-docker-public", + trigger = { + "event": ["promote"], + "target": ["public"], + }, + steps = [ + identify_runner_step(), + download_grabpl_step(), + compile_build_cmd(), + fetch_images_step(), + publish_image_public_step(), + publish_images_step("release", "grafana-oss"), + ], + environment = {"EDITION": "oss"}, + ), + pipeline( + name = "manually-publish-docker-public", + trigger = { + "event": ["promote"], + "target": ["publish-docker-public"], + }, + steps = [ + identify_runner_step(), + download_grabpl_step(), + compile_build_cmd(), + fetch_images_step(), + publish_image_public_step(), + ], environment = {"EDITION": "oss"}, ), ] diff --git a/scripts/drone/utils/images.star b/scripts/drone/utils/images.star index ee664b8cbb9..da31287bad2 100644 --- a/scripts/drone/utils/images.star +++ b/scripts/drone/utils/images.star @@ -9,6 +9,7 @@ load( ) images = { + "docker": "docker:27-cli", "git": "alpine/git:2.40.1", "go": "golang:{}-alpine".format(golang_version), "node": "node:{}-alpine".format(nodejs_version),