diff --git a/.drone.yml b/.drone.yml index b9f6d8665da..4e33bb330af 100644 --- a/.drone.yml +++ b/.drone.yml @@ -569,13 +569,6 @@ steps: depends_on: [] image: golang:1.23.1-alpine name: verify-gen-jsonnet -- commands: - - apk add --update make - - make gen-go - depends_on: - - verify-gen-cue - image: golang:1.23.1-alpine - name: wire-install - commands: - yarn install --immutable || yarn install --immutable depends_on: [] @@ -606,9 +599,16 @@ steps: token: from_secret: drone_token - commands: + - docker run --privileged --rm tonistiigi/binfmt --install all - /src/grafana-build artifacts -a targz:grafana:linux/amd64 -a targz:grafana:linux/arm64 - -a targz:grafana:linux/arm/v7 --go-version=1.23.1 --yarn-cache=$$YARN_CACHE_FOLDER - --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD > packages.txt + -a targz:grafana:linux/arm/v7 -a docker:grafana:linux/amd64 -a docker:grafana:linux/amd64:ubuntu + -a docker:grafana:linux/arm64 -a docker:grafana:linux/arm64:ubuntu -a docker:grafana:linux/arm/v7 + -a docker:grafana:linux/arm/v7:ubuntu --go-version=1.23.1 --yarn-cache=$$YARN_CACHE_FOLDER + --build-id=$$DRONE_BUILD_NUMBER --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.20.3 + --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --ubuntu-tag-format='{{ + .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' --verify='false' --grafana-dir=$$PWD + > packages.txt + - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i depends_on: - yarn-install environment: @@ -620,6 +620,27 @@ steps: volumes: - name: docker path: /var/run/docker.sock +- commands: + - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana + depends_on: + - rgm-package + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GITHUB_APP_ID: + from_secret: delivery-bot-app-id + GITHUB_APP_INSTALLATION_ID: + from_secret: delivery-bot-app-installation-id + GITHUB_APP_PRIVATE_KEY: + from_secret: delivery-bot-app-private-key + failure: ignore + image: google/cloud-sdk:431.0.0 + name: publish-images-grafana + volumes: + - name: docker + path: /var/run/docker.sock - commands: - yarn e2e:plugin:build depends_on: @@ -855,47 +876,6 @@ steps: failure: always image: grafana/docker-puppeteer:1.1.0 name: test-a11y-frontend -- commands: - - docker run --privileged --rm tonistiigi/binfmt --install all - - /src/grafana-build artifacts -a docker:grafana:linux/amd64 -a docker:grafana:linux/amd64:ubuntu - -a docker:grafana:linux/arm64 -a docker:grafana:linux/arm64:ubuntu -a docker:grafana:linux/arm/v7 - -a docker:grafana:linux/arm/v7:ubuntu --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER - --go-version=1.23.1 --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.20.3 --tag-format='{{ - .version_base }}-{{ .buildID }}-{{ .arch }}' --grafana-dir=$$PWD --ubuntu-tag-format='{{ - .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > docker.txt - - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i - depends_on: - - yarn-install - environment: - _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: - from_secret: dagger_token - image: grafana/grafana-build:main - name: rgm-build-docker - pull: always - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana - depends_on: - - rgm-build-docker - environment: - DOCKER_PASSWORD: - from_secret: docker_password - DOCKER_USER: - from_secret: docker_username - GITHUB_APP_ID: - from_secret: delivery-bot-app-id - GITHUB_APP_INSTALLATION_ID: - from_secret: delivery-bot-app-installation-id - GITHUB_APP_PRIVATE_KEY: - from_secret: delivery-bot-app-private-key - failure: ignore - image: google/cloud-sdk:431.0.0 - name: publish-images-grafana - volumes: - - name: docker - path: /var/run/docker.sock trigger: event: - pull_request @@ -1987,13 +1967,6 @@ steps: depends_on: [] image: golang:1.23.1-alpine name: verify-gen-jsonnet -- commands: - - apk add --update make - - make gen-go - depends_on: - - verify-gen-cue - image: golang:1.23.1-alpine - name: wire-install - commands: - yarn install --immutable || yarn install --immutable depends_on: [] @@ -2023,9 +1996,16 @@ steps: image: node:20.9.0-alpine name: build-frontend-packages - commands: + - docker run --privileged --rm tonistiigi/binfmt --install all - /src/grafana-build artifacts -a targz:grafana:linux/amd64 -a targz:grafana:linux/arm64 - -a targz:grafana:linux/arm/v7 --go-version=1.23.1 --yarn-cache=$$YARN_CACHE_FOLDER - --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD > packages.txt + -a targz:grafana:linux/arm/v7 -a docker:grafana:linux/amd64 -a docker:grafana:linux/amd64:ubuntu + -a docker:grafana:linux/arm64 -a docker:grafana:linux/arm64:ubuntu -a docker:grafana:linux/arm/v7 + -a docker:grafana:linux/arm/v7:ubuntu --go-version=1.23.1 --yarn-cache=$$YARN_CACHE_FOLDER + --build-id=$$DRONE_BUILD_NUMBER --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.20.3 + --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --ubuntu-tag-format='{{ + .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' --verify='false' --grafana-dir=$$PWD + > packages.txt + - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i depends_on: - update-package-json-version environment: @@ -2037,6 +2017,31 @@ steps: volumes: - name: docker path: /var/run/docker.sock +- commands: + - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana + depends_on: + - rgm-package + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_grafanauploads + GITHUB_APP_ID: + from_secret: delivery-bot-app-id + GITHUB_APP_INSTALLATION_ID: + from_secret: delivery-bot-app-installation-id + GITHUB_APP_PRIVATE_KEY: + from_secret: delivery-bot-app-private-key + image: google/cloud-sdk:431.0.0 + name: publish-images-grafana + volumes: + - name: docker + path: /var/run/docker.sock + when: + repo: + - grafana/grafana - commands: - yarn e2e:plugin:build depends_on: @@ -2308,55 +2313,10 @@ steps: when: repo: - grafana/grafana -- commands: - - docker run --privileged --rm tonistiigi/binfmt --install all - - /src/grafana-build artifacts -a docker:grafana:linux/amd64 -a docker:grafana:linux/amd64:ubuntu - -a docker:grafana:linux/arm64 -a docker:grafana:linux/arm64:ubuntu -a docker:grafana:linux/arm/v7 - -a docker:grafana:linux/arm/v7:ubuntu --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER - --go-version=1.23.1 --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.20.3 --tag-format='{{ - .version_base }}-{{ .buildID }}-{{ .arch }}' --grafana-dir=$$PWD --ubuntu-tag-format='{{ - .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > docker.txt - - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i - depends_on: - - update-package-json-version - environment: - _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: - from_secret: dagger_token - image: grafana/grafana-build:main - name: rgm-build-docker - pull: always - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana - depends_on: - - rgm-build-docker - environment: - DOCKER_PASSWORD: - from_secret: docker_password - DOCKER_USER: - from_secret: docker_username - GCP_KEY: - from_secret: gcp_grafanauploads - GITHUB_APP_ID: - from_secret: delivery-bot-app-id - GITHUB_APP_INSTALLATION_ID: - from_secret: delivery-bot-app-installation-id - GITHUB_APP_PRIVATE_KEY: - from_secret: delivery-bot-app-private-key - image: google/cloud-sdk:431.0.0 - name: publish-images-grafana - volumes: - - name: docker - path: /var/run/docker.sock - when: - repo: - - grafana/grafana - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana-oss depends_on: - - rgm-build-docker + - rgm-package environment: DOCKER_PASSWORD: from_secret: docker_password @@ -2401,10 +2361,7 @@ steps: - commands: - ./bin/build upload-packages --edition oss depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite + - rgm-package environment: GCP_KEY: from_secret: gcp_grafanauploads_base64 @@ -2418,7 +2375,7 @@ steps: - commands: - ./bin/build upload-cdn --edition oss depends_on: - - grafana-server + - rgm-package environment: GCP_KEY: from_secret: gcp_grafanauploads @@ -5778,6 +5735,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: 53143f19d4dd87f1a7b76cb3226c353084f85ee7b832d259ca8d318b13d1ed37 +hmac: 42b13a1314819f896e75c417f887d038d6e8ea5897f084b4a9dcb63ff308f434 ... diff --git a/scripts/drone/pipelines/build.star b/scripts/drone/pipelines/build.star index b330fc61fdc..828f3fdae7f 100644 --- a/scripts/drone/pipelines/build.star +++ b/scripts/drone/pipelines/build.star @@ -27,13 +27,11 @@ load( "upload_packages_step", "verify_gen_cue_step", "verify_gen_jsonnet_step", - "wire_install_step", "yarn_install_step", ) load( "scripts/drone/steps/rgm.star", "rgm_artifacts_step", - "rgm_build_docker_step", ) load( "scripts/drone/utils/images.star", @@ -44,6 +42,7 @@ load( "pipeline", ) +# This function isn't actually unused but I don't know why the linter thinks it is... # @unused def build_e2e(trigger, ver_mode): """Perform e2e building, testing, and publishing. @@ -63,37 +62,58 @@ def build_e2e(trigger, ver_mode): compile_build_cmd(), verify_gen_cue_step(), verify_gen_jsonnet_step(), - wire_install_step(), yarn_install_step(), ] build_steps = [] + create_packages = rgm_artifacts_step( + alpine = images["alpine"], + artifacts = [ + "targz:grafana:linux/amd64", + "targz:grafana:linux/arm64", + "targz:grafana:linux/arm/v7", + "docker:grafana:linux/amd64", + "docker:grafana:linux/amd64:ubuntu", + "docker:grafana:linux/arm64", + "docker:grafana:linux/arm64:ubuntu", + "docker:grafana:linux/arm/v7", + "docker:grafana:linux/arm/v7:ubuntu", + ], + file = "packages.txt", + tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}", + ubuntu = images["ubuntu"], + ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}", + ) + + publish_docker = publish_images_step( + depends_on = [create_packages["name"]], + docker_repo = "grafana", + trigger = trigger_oss, + ver_mode = ver_mode, + ) + if ver_mode == "pr": build_steps.extend( [ build_frontend_package_step(), enterprise_downstream_step(ver_mode = ver_mode), - rgm_artifacts_step(artifacts = ["targz:grafana:linux/amd64", "targz:grafana:linux/arm64", "targz:grafana:linux/arm/v7"], file = "packages.txt"), ], ) else: + # The only other event or "ver_mode" where this is used is 'main' + update_package_json = update_package_json_version() + create_packages["depends_on"] = [update_package_json["name"]] + build_steps.extend([ - update_package_json_version(), + update_package_json, build_frontend_package_step(depends_on = ["update-package-json-version"]), - rgm_artifacts_step( - artifacts = [ - "targz:grafana:linux/amd64", - "targz:grafana:linux/arm64", - "targz:grafana:linux/arm/v7", - ], - depends_on = ["update-package-json-version"], - file = "packages.txt", - ), ]) build_steps.extend( [ + create_packages, + publish_docker, build_test_plugins_step(), grafana_server_step(), e2e_tests_step("dashboards-suite"), @@ -123,45 +143,20 @@ def build_e2e(trigger, ver_mode): [ store_storybook_step(trigger = trigger_oss, ver_mode = ver_mode), frontend_metrics_step(trigger = trigger_oss), - rgm_build_docker_step( - images["ubuntu"], - images["alpine"], - depends_on = ["update-package-json-version"], - tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}", - ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}", - ), - publish_images_step( - docker_repo = "grafana", - trigger = trigger_oss, - ver_mode = ver_mode, - ), publish_images_step( + depends_on = [create_packages["name"]], docker_repo = "grafana-oss", trigger = trigger_oss, ver_mode = ver_mode, ), release_canary_npm_packages_step(trigger = trigger_oss), upload_packages_step( + depends_on = [create_packages["name"]], trigger = trigger_oss, ver_mode = ver_mode, ), upload_cdn_step( - trigger = trigger_oss, - ver_mode = ver_mode, - ), - ], - ) - elif ver_mode == "pr": - build_steps.extend( - [ - rgm_build_docker_step( - images["ubuntu"], - images["alpine"], - tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}", - ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}", - ), - publish_images_step( - docker_repo = "grafana", + depends_on = [create_packages["name"]], trigger = trigger_oss, ver_mode = ver_mode, ), diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 6941b49bc01..08400e9cd16 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -405,13 +405,14 @@ def playwright_e2e_report_post_link(): ], } -def upload_cdn_step(ver_mode, trigger = None): +def upload_cdn_step(ver_mode, trigger = None, depends_on = ["grafana-server"]): """Uploads CDN assets using the Grafana build tool. Args: ver_mode: only uses the step trigger when ver_mode == 'release-branch' or 'main' trigger: a Drone trigger for the step. Defaults to None. + depends_on: drone steps that this step depends on Returns: Drone step. @@ -420,9 +421,7 @@ def upload_cdn_step(ver_mode, trigger = None): step = { "name": "upload-cdn-assets", "image": images["publish"], - "depends_on": [ - "grafana-server", - ], + "depends_on": depends_on, "environment": { "GCP_KEY": from_secret(gcp_grafanauploads), "PRERELEASE_BUCKET": from_secret(prerelease_bucket), @@ -929,7 +928,7 @@ def fetch_images_step(): "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], } -def publish_images_step(ver_mode, docker_repo, trigger = None): +def publish_images_step(ver_mode, docker_repo, trigger = None, depends_on = ["rgm-build-docker"]): """Generates a step for publishing public Docker images with grabpl. Args: @@ -939,6 +938,7 @@ def publish_images_step(ver_mode, docker_repo, trigger = None): It is combined with the 'grafana/' library prefix. trigger: a Drone trigger for the pipeline. Defaults to None. + depends_on: drone steps that this step depends on Returns: Drone step. @@ -959,7 +959,7 @@ def publish_images_step(ver_mode, docker_repo, trigger = None): docker_repo, ) - deps = ["rgm-build-docker"] + deps = depends_on if ver_mode == "release": deps = ["fetch-images"] cmd += " --version-tag ${DRONE_TAG}" @@ -1149,7 +1149,12 @@ def release_canary_npm_packages_step(trigger = None): return step -def upload_packages_step(ver_mode, trigger = None): +def upload_packages_step(ver_mode, trigger = None, depends_on = [ + "end-to-end-tests-dashboards-suite", + "end-to-end-tests-panels-suite", + "end-to-end-tests-smoke-tests-suite", + "end-to-end-tests-various-suite", +]): """Upload packages to object storage. Args: @@ -1157,6 +1162,7 @@ def upload_packages_step(ver_mode, trigger = None): edition packages when executed. trigger: a Drone trigger for the step. Defaults to None. + depends_on: drone steps that this step depends on Returns: Drone step. @@ -1164,7 +1170,7 @@ def upload_packages_step(ver_mode, trigger = None): step = { "name": "upload-packages", "image": images["publish"], - "depends_on": end_to_end_tests_deps(), + "depends_on": depends_on, "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), diff --git a/scripts/drone/steps/rgm.star b/scripts/drone/steps/rgm.star index 7890bc3955e..fd96a60d156 100644 --- a/scripts/drone/steps/rgm.star +++ b/scripts/drone/steps/rgm.star @@ -3,6 +3,10 @@ Individual steps that use 'grafana-build' to replace existing individual steps. These aren't used in releases. """ +load( + "scripts/drone/utils/images.star", + "images", +) load( "scripts/drone/variables.star", "golang_version", @@ -22,7 +26,16 @@ def artifacts_cmd(artifacts = []): return cmd # rgm_artifacts_step will create artifacts using the '/src/build artifacts' command. -def rgm_artifacts_step(name = "rgm-package", artifacts = ["targz:grafana:linux/amd64", "targz:grafana:linux/arm64"], file = "packages.txt", depends_on = ["yarn-install"]): +def rgm_artifacts_step( + name = "rgm-package", + artifacts = ["targz:grafana:linux/amd64", "targz:grafana:linux/arm64"], + file = "packages.txt", + depends_on = ["yarn-install"], + tag_format = "{{ .version }}-{{ .arch }}", + ubuntu_tag_format = "{{ .version }}-ubuntu-{{ .arch }}", + verify = "false", + ubuntu = images["ubuntu"], + alpine = images["alpine"]): cmd = artifacts_cmd(artifacts = artifacts) return { @@ -34,11 +47,18 @@ def rgm_artifacts_step(name = "rgm-package", artifacts = ["targz:grafana:linux/a "_EXPERIMENTAL_DAGGER_CLOUD_TOKEN": from_secret(rgm_dagger_token), }, "commands": [ + "docker run --privileged --rm tonistiigi/binfmt --install all", cmd + "--go-version={} ".format(golang_version) + "--yarn-cache=$$YARN_CACHE_FOLDER " + "--build-id=$$DRONE_BUILD_NUMBER " + + "--ubuntu-base={} ".format(ubuntu) + + "--alpine-base={} ".format(alpine) + + "--tag-format='{}' ".format(tag_format) + + "--ubuntu-tag-format='{}' ".format(ubuntu_tag_format) + + "--verify='{}' ".format(verify) + "--grafana-dir=$$PWD > {}".format(file), + "find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i", ], "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], }