From 8a5d4c4c6e323529bbfea3a2774a38b91062eeeb Mon Sep 17 00:00:00 2001 From: Kevin Minehart Date: Tue, 31 Oct 2023 20:52:09 +0000 Subject: [PATCH] CI: Update RGM steps to use the artifacts command (#77470) * update rgm steps to use artifacts subcmd * format-drone * make drone --- .drone.yml | 102 ++++++++++++++++------------- scripts/drone/pipelines/build.star | 15 +++-- scripts/drone/rgm.star | 2 + scripts/drone/steps/rgm.star | 44 +++++++------ scripts/drone/utils/images.star | 2 +- 5 files changed, 96 insertions(+), 69 deletions(-) diff --git a/.drone.yml b/.drone.yml index aa50476196c..4fe665bf568 100644 --- a/.drone.yml +++ b/.drone.yml @@ -17,7 +17,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd @@ -67,7 +67,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd @@ -117,7 +117,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - yarn install --immutable @@ -217,7 +217,7 @@ steps: name: clone-enterprise - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - yarn install --immutable @@ -306,7 +306,7 @@ steps: name: clone-enterprise - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -390,7 +390,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd @@ -485,7 +485,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - mkdir -p bin @@ -556,9 +556,9 @@ steps: token: from_secret: drone_token - commands: - - /src/grafana-build package --distro=linux/amd64,linux/arm64 --go-version=1.20.10 - --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD - > packages.txt + - /src/grafana-build artifacts -a targz:grafana:linux/amd64 -a targz:grafana:linux/arm64 + --go-version=1.20.10 --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER + --grafana-dir=$$PWD > packages.txt depends_on: - yarn-install image: grafana/grafana-build:main @@ -579,7 +579,7 @@ steps: GF_APP_MODE: development GF_SERVER_HTTP_PORT: "3001" GF_SERVER_ROUTER_LOGGING: "1" - image: alpine:3.18.3 + image: alpine:3.18.4 name: grafana-server - commands: - ./bin/build e2e-tests --port 3001 --suite dashboards-suite @@ -699,13 +699,15 @@ steps: name: test-a11y-frontend - commands: - docker run --privileged --rm tonistiigi/binfmt --install all - - /src/grafana-build docker $(cat packages.txt | grep tar.gz | grep -v docker | - grep -v sha256 | awk '{print "--package=" $0}') --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.18.3 - --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --ubuntu-tag-format='{{ - .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > docker.txt + - /src/grafana-build artifacts -a docker:grafana:linux/amd64 -a docker:grafana:linux/amd64:ubuntu + -a docker:grafana:linux/arm64 -a docker:grafana:linux/arm64:ubuntu --yarn-cache=$$YARN_CACHE_FOLDER + --build-id=$$DRONE_BUILD_NUMBER --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.18.4 + --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --grafana-dir=$$PWD + --ubuntu-tag-format='{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > + docker.txt - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i depends_on: - - rgm-package + - yarn-install image: grafana/grafana-build:main name: rgm-build-docker pull: always @@ -844,7 +846,7 @@ steps: name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -1030,7 +1032,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - yarn install --immutable @@ -1385,7 +1387,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - yarn install --immutable @@ -1460,7 +1462,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - yarn install --immutable @@ -1517,7 +1519,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - yarn install --immutable @@ -1584,7 +1586,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -1663,7 +1665,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd @@ -1737,7 +1739,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - mkdir -p bin @@ -1807,9 +1809,9 @@ steps: image: node:20.9.0-alpine name: build-frontend-packages - commands: - - /src/grafana-build package --distro=linux/amd64,linux/arm64 --go-version=1.20.10 - --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD - > packages.txt + - /src/grafana-build artifacts -a targz:grafana:linux/amd64 -a targz:grafana:linux/arm64 + --go-version=1.20.10 --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER + --grafana-dir=$$PWD > packages.txt depends_on: - update-package-json-version image: grafana/grafana-build:main @@ -1830,7 +1832,7 @@ steps: GF_APP_MODE: development GF_SERVER_HTTP_PORT: "3001" GF_SERVER_ROUTER_LOGGING: "1" - image: alpine:3.18.3 + image: alpine:3.18.4 name: grafana-server - commands: - ./bin/build e2e-tests --port 3001 --suite dashboards-suite @@ -1986,13 +1988,15 @@ steps: - grafana/grafana - commands: - docker run --privileged --rm tonistiigi/binfmt --install all - - /src/grafana-build docker $(cat packages.txt | grep tar.gz | grep -v docker | - grep -v sha256 | awk '{print "--package=" $0}') --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.18.3 - --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --ubuntu-tag-format='{{ - .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > docker.txt + - /src/grafana-build artifacts -a docker:grafana:linux/amd64 -a docker:grafana:linux/amd64:ubuntu + -a docker:grafana:linux/arm64 -a docker:grafana:linux/arm64:ubuntu --yarn-cache=$$YARN_CACHE_FOLDER + --build-id=$$DRONE_BUILD_NUMBER --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.18.4 + --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --grafana-dir=$$PWD + --ubuntu-tag-format='{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > + docker.txt - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i depends_on: - - rgm-package + - yarn-install image: grafana/grafana-build:main name: rgm-build-docker pull: always @@ -2193,7 +2197,7 @@ steps: name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -2506,7 +2510,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - mkdir -p bin @@ -2838,6 +2842,7 @@ steps: environment: _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: from_secret: dagger_token + ALPINE_BASE: alpine:3.18.4 CDN_DESTINATION: from_secret: rgm_cdn_destination DESTINATION: @@ -2865,6 +2870,7 @@ steps: from_secret: npm_token STORYBOOK_DESTINATION: from_secret: rgm_storybook_destination + UBUNTU_BASE: ubuntu:22.04 image: grafana/grafana-build:main name: rgm-build pull: always @@ -2951,7 +2957,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - yarn install --immutable @@ -3006,7 +3012,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -3087,6 +3093,7 @@ steps: environment: _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: from_secret: dagger_token + ALPINE_BASE: alpine:3.18.4 CDN_DESTINATION: from_secret: rgm_cdn_destination DESTINATION: @@ -3114,6 +3121,7 @@ steps: from_secret: npm_token STORYBOOK_DESTINATION: from_secret: rgm_storybook_destination + UBUNTU_BASE: ubuntu:22.04 image: grafana/grafana-build:main name: rgm-build pull: always @@ -3265,6 +3273,7 @@ steps: environment: _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: from_secret: dagger_token + ALPINE_BASE: alpine:3.18.4 CDN_DESTINATION: from_secret: rgm_cdn_destination DESTINATION: @@ -3292,6 +3301,7 @@ steps: from_secret: npm_token STORYBOOK_DESTINATION: from_secret: rgm_storybook_destination + UBUNTU_BASE: ubuntu:22.04 image: grafana/grafana-build:main name: rgm-build pull: always @@ -3363,7 +3373,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - yarn install --immutable @@ -3416,7 +3426,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -3495,6 +3505,7 @@ steps: environment: _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: from_secret: dagger_token + ALPINE_BASE: alpine:3.18.4 CDN_DESTINATION: from_secret: rgm_cdn_destination DESTINATION: @@ -3522,6 +3533,7 @@ steps: from_secret: npm_token STORYBOOK_DESTINATION: from_secret: rgm_storybook_destination + UBUNTU_BASE: ubuntu:22.04 image: grafana/grafana-build:main name: rgm-build pull: always @@ -3639,6 +3651,7 @@ steps: environment: _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: from_secret: dagger_token + ALPINE_BASE: alpine:3.18.4 CDN_DESTINATION: from_secret: rgm_cdn_destination DESTINATION: @@ -3666,6 +3679,7 @@ steps: from_secret: npm_token STORYBOOK_DESTINATION: from_secret: rgm_storybook_destination + UBUNTU_BASE: ubuntu:22.04 image: grafana/grafana-build:main name: rgm-publish pull: always @@ -3846,7 +3860,7 @@ steps: name: grabpl - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 + image: alpine:3.18.4 name: identify-runner - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -4068,7 +4082,7 @@ steps: - commands: - if [ -z "${BUILD_CONTAINER_VERSION}" ]; then echo Missing BUILD_CONTAINER_VERSION; false; fi - image: alpine:3.18.3 + image: alpine:3.18.4 name: validate-version - commands: - printenv GCP_KEY > /tmp/key.json @@ -4392,7 +4406,7 @@ steps: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM node:20.9.0-alpine - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM google/cloud-sdk:431.0.0 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana-ci-deploy:1.3.3 - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine:3.18.3 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine:3.18.4 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM ubuntu:22.04 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM byrnedo/alpine-curl:0.1.8 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM plugins/slack @@ -4426,7 +4440,7 @@ steps: - trivy --exit-code 1 --severity HIGH,CRITICAL node:20.9.0-alpine - trivy --exit-code 1 --severity HIGH,CRITICAL google/cloud-sdk:431.0.0 - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana-ci-deploy:1.3.3 - - trivy --exit-code 1 --severity HIGH,CRITICAL alpine:3.18.3 + - trivy --exit-code 1 --severity HIGH,CRITICAL alpine:3.18.4 - trivy --exit-code 1 --severity HIGH,CRITICAL ubuntu:22.04 - trivy --exit-code 1 --severity HIGH,CRITICAL byrnedo/alpine-curl:0.1.8 - trivy --exit-code 1 --severity HIGH,CRITICAL plugins/slack @@ -4670,6 +4684,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: 920b57d70a96a29872ce77dc83ca26e2d141ba37627eee58c9d3beebf9ec4055 +hmac: 88aa054b297d39ae9757ee3a13d16b51491931fabf7c78ba0d0c125a4088da33 ... diff --git a/scripts/drone/pipelines/build.star b/scripts/drone/pipelines/build.star index ee60b7b5103..f4d5800242c 100644 --- a/scripts/drone/pipelines/build.star +++ b/scripts/drone/pipelines/build.star @@ -28,8 +28,8 @@ load( ) load( "scripts/drone/steps/rgm.star", + "rgm_artifacts_step", "rgm_build_docker_step", - "rgm_package_step", ) load( "scripts/drone/utils/images.star", @@ -70,14 +70,21 @@ def build_e2e(trigger, ver_mode): [ build_frontend_package_step(), enterprise_downstream_step(ver_mode = ver_mode), - rgm_package_step(distros = "linux/amd64,linux/arm64", file = "packages.txt"), + rgm_artifacts_step(artifacts = ["targz:grafana:linux/amd64", "targz:grafana:linux/arm64"], file = "packages.txt"), ], ) else: build_steps.extend([ update_package_json_version(), build_frontend_package_step(depends_on = ["update-package-json-version"]), - rgm_package_step(depends_on = ["update-package-json-version"], distros = "linux/amd64,linux/arm64", file = "packages.txt"), + rgm_artifacts_step( + artifacts = [ + "targz:grafana:linux/amd64", + "targz:grafana:linux/arm64", + ], + depends_on = ["update-package-json-version"], + file = "packages.txt", + ), ]) build_steps.extend( @@ -104,7 +111,6 @@ def build_e2e(trigger, ver_mode): store_storybook_step(trigger = trigger_oss, ver_mode = ver_mode), frontend_metrics_step(trigger = trigger_oss), rgm_build_docker_step( - "packages.txt", images["ubuntu"], images["alpine"], tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}", @@ -135,7 +141,6 @@ def build_e2e(trigger, ver_mode): build_steps.extend( [ rgm_build_docker_step( - "packages.txt", images["ubuntu"], images["alpine"], tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}", diff --git a/scripts/drone/rgm.star b/scripts/drone/rgm.star index 411d4a4c969..b411178cee8 100644 --- a/scripts/drone/rgm.star +++ b/scripts/drone/rgm.star @@ -137,6 +137,8 @@ def rgm_run(name, script): """ env = { "GO_VERSION": golang_version, + "ALPINE_BASE": images["alpine"], + "UBUNTU_BASE": images["ubuntu"], } rgm_run_step = { "name": name, diff --git a/scripts/drone/steps/rgm.star b/scripts/drone/steps/rgm.star index 6d86ac48b34..ba12e69a6c8 100644 --- a/scripts/drone/steps/rgm.star +++ b/scripts/drone/steps/rgm.star @@ -8,15 +8,25 @@ load( "golang_version", ) -# rgm_package_step will create a tar.gz for use in e2e tests or other PR testing related activities.. -def rgm_package_step(distros = "linux/amd64,linux/arm64", file = "packages.txt", depends_on = ["yarn-install"]): +def artifacts_cmd(artifacts = []): + cmd = "/src/grafana-build artifacts " + + for artifact in artifacts: + cmd += "-a {} ".format(artifact) + + return cmd + +# rgm_artifacts_step will create artifacts using the '/src/build artifacts' command. +def rgm_artifacts_step(name = "rgm-package", artifacts = ["targz:grafana:linux/amd64", "targz:grafana:linux/arm64"], file = "packages.txt", depends_on = ["yarn-install"]): + cmd = artifacts_cmd(artifacts = artifacts) + return { - "name": "rgm-package", + "name": name, "image": "grafana/grafana-build:main", "pull": "always", "depends_on": depends_on, "commands": [ - "/src/grafana-build package --distro={} ".format(distros) + + cmd + "--go-version={} ".format(golang_version) + "--yarn-cache=$$YARN_CACHE_FOLDER " + "--build-id=$$DRONE_BUILD_NUMBER " + @@ -28,31 +38,27 @@ def rgm_package_step(distros = "linux/amd64,linux/arm64", file = "packages.txt", # rgm_build_backend will create compile the grafana backend for various platforms. It's preferred to use # 'rgm_package_step' if you creating a "usable" artifact. This should really only be used to verify that the code is # compilable. -def rgm_build_backend_step(distros = "linux/amd64,linux/arm64"): - return { - "name": "rgm-package", - "image": "grafana/grafana-build:main", - "pull": "always", - "commands": [ - "/src/grafana-build build " + - "--go-version={} ".format(golang_version) + - "--distro={} --grafana-dir=$$PWD".format(distros), - ], - "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], - } +def rgm_build_backend_step(artifacts = ["backend:grafana:linux/amd64", "backend:grafana:linux/arm64"]): + return rgm_artifacts_step(name = "rgm-build-backend", artifacts = artifacts, depends_on = []) -def rgm_build_docker_step(packages, ubuntu, alpine, depends_on = ["rgm-package"], file = "docker.txt", tag_format = "{{ .version }}-{{ .arch }}", ubuntu_tag_format = "{{ .version }}-ubuntu-{{ .arch }}"): +def rgm_build_docker_step(ubuntu, alpine, depends_on = ["yarn-install"], file = "docker.txt", tag_format = "{{ .version }}-{{ .arch }}", ubuntu_tag_format = "{{ .version }}-ubuntu-{{ .arch }}"): return { "name": "rgm-build-docker", "image": "grafana/grafana-build:main", "pull": "always", "commands": [ "docker run --privileged --rm tonistiigi/binfmt --install all", - "/src/grafana-build docker " + - "$(cat {} | grep tar.gz | grep -v docker | grep -v sha256 | awk '{{print \"--package=\" $0}}') ".format(packages) + + "/src/grafana-build artifacts " + + "-a docker:grafana:linux/amd64 " + + "-a docker:grafana:linux/amd64:ubuntu " + + "-a docker:grafana:linux/arm64 " + + "-a docker:grafana:linux/arm64:ubuntu " + + "--yarn-cache=$$YARN_CACHE_FOLDER " + + "--build-id=$$DRONE_BUILD_NUMBER " + "--ubuntu-base={} ".format(ubuntu) + "--alpine-base={} ".format(alpine) + "--tag-format='{}' ".format(tag_format) + + "--grafana-dir=$$PWD " + "--ubuntu-tag-format='{}' > {}".format(ubuntu_tag_format, file), "find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i", ], diff --git a/scripts/drone/utils/images.star b/scripts/drone/utils/images.star index 454130fbc4a..c16df446e17 100644 --- a/scripts/drone/utils/images.star +++ b/scripts/drone/utils/images.star @@ -14,7 +14,7 @@ images = { "node": "node:{}-alpine".format(nodejs_version), "cloudsdk": "google/cloud-sdk:431.0.0", "publish": "grafana/grafana-ci-deploy:1.3.3", - "alpine": "alpine:3.18.3", + "alpine": "alpine:3.18.4", "ubuntu": "ubuntu:22.04", "curl": "byrnedo/alpine-curl:0.1.8", "plugins_slack": "plugins/slack",