From 438200c50d467c52f1f042ccb1a81dc5c33e0cba Mon Sep 17 00:00:00 2001 From: Guilherme Caulada Date: Fri, 20 Oct 2023 10:04:18 -0300 Subject: [PATCH] [v10.0.x] CI: Update CI/CD tooling and pipelines from main (#76874) * CI: Update CI/CD tooling and pipelines from main (#76814) * CI: Update CI/CD tooling and pipelines from main * Update Makefile * Comment out validate_openapi_spec_step * Update broken frontend tests * Fix validate-npm-packages regex to work without suffix * Fix cypress image version (cherry picked from commit 03ecb1db39994bca1ba0a846e76719db33b5e883) * Comment out kindsysreport/codegen/report.go on Makefile * Remove modfile step, since modowners doesn't exist * Disable i18n step --- .drone.star | 20 +- .drone.yml | 2543 +++++++++-------- Makefile | 75 +- pkg/build/cmd.go | 12 +- pkg/build/cmd/grafanacom.go | 2 +- pkg/build/cmd/verifydrone.go | 6 +- pkg/build/config/genmetadata_test.go | 6 +- pkg/build/config/revision.go | 26 +- pkg/build/config/version.go | 2 +- pkg/build/config/versions.go | 37 +- pkg/build/docker/build.go | 2 +- pkg/build/env/fallback_test.go | 27 +- pkg/build/frontend/config_test.go | 29 +- pkg/build/gcloud/storage/gsutil.go | 22 +- pkg/build/git.go | 13 + pkg/build/grafana/build.go | 8 +- pkg/build/lerna/lerna.go | 2 +- pkg/build/npm/npm.go | 6 +- pkg/build/packaging/grafana.go | 2 +- scripts/drone/TAGS | 628 ---- scripts/drone/events/cron.star | 36 +- scripts/drone/events/main.star | 75 +- scripts/drone/events/pr.star | 47 +- scripts/drone/events/release.star | 260 +- scripts/drone/pipelines/benchmarks.star | 80 + scripts/drone/pipelines/build.star | 59 +- scripts/drone/pipelines/ci_images.star | 24 +- scripts/drone/pipelines/docs.star | 10 +- .../drone/pipelines/integration_tests.star | 32 +- scripts/drone/pipelines/lint_backend.star | 4 + scripts/drone/pipelines/lint_frontend.star | 4 + scripts/drone/pipelines/shellcheck.star | 16 +- scripts/drone/pipelines/test_backend.star | 2 - scripts/drone/pipelines/test_frontend.star | 2 - .../drone/pipelines/whats_new_checker.star | 10 +- scripts/drone/pipelines/windows.star | 22 +- scripts/drone/rgm.star | 290 +- scripts/drone/services/services.star | 36 +- scripts/drone/steps/lib.star | 795 ++---- scripts/drone/steps/lib_windows.star | 187 ++ scripts/drone/steps/rgm.star | 61 + scripts/drone/utils/images.star | 49 +- scripts/drone/utils/utils.star | 3 +- scripts/drone/utils/windows_images.star | 13 +- scripts/drone/variables.star | 9 + scripts/drone/vault.star | 23 +- scripts/drone/version.star | 16 - scripts/validate-npm-packages.sh | 2 +- 48 files changed, 2768 insertions(+), 2867 deletions(-) delete mode 100644 scripts/drone/TAGS create mode 100644 scripts/drone/pipelines/benchmarks.star create mode 100644 scripts/drone/steps/lib_windows.star create mode 100644 scripts/drone/steps/rgm.star create mode 100644 scripts/drone/variables.star delete mode 100644 scripts/drone/version.star diff --git a/.drone.star b/.drone.star index ce31860e844..a14146a7c14 100644 --- a/.drone.star +++ b/.drone.star @@ -7,8 +7,9 @@ This module returns a Drone configuration including pipelines and secrets. """ -load("scripts/drone/events/pr.star", "pr_pipelines") +load("scripts/drone/events/cron.star", "cronjobs") load("scripts/drone/events/main.star", "main_pipelines") +load("scripts/drone/events/pr.star", "pr_pipelines") load( "scripts/drone/events/release.star", "integration_test_pipelines", @@ -17,24 +18,22 @@ load( "publish_packages_pipeline", ) load( - "scripts/drone/rgm.star", - "rgm", + "scripts/drone/pipelines/ci_images.star", + "publish_ci_build_container_image_pipeline", + "publish_ci_windows_test_image_pipeline", ) load( "scripts/drone/pipelines/publish_images.star", "publish_image_pipelines_public", ) -load( - "scripts/drone/pipelines/ci_images.star", - "publish_ci_build_container_image_pipeline", - "publish_ci_windows_test_image_pipeline", -) load( "scripts/drone/pipelines/windows.star", "windows_test_backend", ) -load("scripts/drone/version.star", "version_branch_pipelines") -load("scripts/drone/events/cron.star", "cronjobs") +load( + "scripts/drone/rgm.star", + "rgm", +) load("scripts/drone/vault.star", "secrets") def main(_ctx): @@ -50,7 +49,6 @@ def main(_ctx): "event": ["promote"], "target": ["test-windows"], }, "oss", "testing")] + - version_branch_pipelines() + integration_test_pipelines() + publish_ci_windows_test_image_pipeline() + publish_ci_build_container_image_pipeline() + diff --git a/.drone.yml b/.drone.yml index a23ce6bddcd..2ab8f702f95 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,7 +24,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - ./bin/build verify-drone @@ -74,13 +74,14 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - - ./bin/build verify-starlark . + - go install github.com/bazelbuild/buildtools/buildifier@latest + - buildifier --lint=warn -mode=check -r . depends_on: - compile-build-cmd - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: lint-starlark trigger: event: @@ -118,24 +119,20 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: + - apk add --update git bash - yarn betterer ci depends_on: - yarn-install - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: betterer-frontend - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -153,7 +150,7 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - yarn run ci:test-frontend @@ -161,7 +158,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: test-frontend trigger: event: @@ -198,6 +195,7 @@ platform: services: [] steps: - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -215,7 +213,7 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - echo $DRONE_RUNNER_NAME @@ -224,18 +222,17 @@ steps: - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: - yarn run prettier:check - yarn run lint - - yarn run i18n:compile - yarn run typecheck depends_on: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: lint-frontend trigger: event: @@ -272,6 +269,7 @@ platform: services: [] steps: - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -289,55 +287,53 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: + - apk add --update build-base shared-mime-info shared-mime-info-lang - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: test-backend - commands: + - apk add --update build-base - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: test-backend-integration trigger: event: @@ -386,9 +382,10 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -406,21 +403,22 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: + - apk add --update make - make gen-go depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: - - apt-get update && apt-get install make + - apk add --update make build-base - make lint-go depends_on: - wire-install environment: CGO_ENABLED: "1" - image: golang:1.20.10 + image: golang:1.20.10-alpine name: lint-backend trigger: event: @@ -467,7 +465,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -476,36 +474,50 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install +- commands: + - apk add --update jq bash + - yarn packages:build + - yarn packages:pack + - ./scripts/validate-npm-packages.sh + depends_on: + - yarn-install + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: node:18.12.0-alpine + name: build-frontend-packages - failure: ignore image: grafana/drone-downstream name: trigger-enterprise-downstream @@ -520,66 +532,32 @@ steps: token: from_secret: drone_token - commands: - - ./bin/build build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - /src/grafana-build package --distro=linux/amd64,linux/arm64 --go-version=1.20.10 + --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD + > packages.txt depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.7.5 - name: build-backend -- commands: - - ./bin/build build-frontend --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 - name: build-frontend + image: grafana/grafana-build:main + name: rgm-package + pull: always + volumes: + - name: docker + path: /var/run/docker.sock - commands: - - ./bin/build build-frontend-packages --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - - yarn packages:pack - - ./scripts/validate-npm-packages.sh + - apk add --update tar bash + - mkdir grafana + - tar --strip-components=1 -xvf ./dist/*amd64.tar.gz -C grafana + - cp -r devenv scripts tools grafana && cd grafana && ./scripts/grafana-server/start-server depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 - name: build-frontend-packages -- commands: - - ./bin/build build-plugins --jobs 8 --edition oss - depends_on: - - compile-build-cmd - - yarn-install - environment: null - image: grafana/build-container:1.7.5 - name: build-plugins -- commands: - - . scripts/build/gpg-test-vars.sh && ./bin/build package --jobs 8 --edition oss - --build-id ${DRONE_BUILD_NUMBER} - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - environment: null - image: grafana/build-container:1.7.5 - name: package -- commands: - - ./scripts/grafana-server/start-server - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages + - rgm-package detach: true environment: - ARCH: linux-amd64 - PORT: 3001 - image: grafana/build-container:1.7.5 + GF_APP_MODE: development + GF_SERVER_HTTP_PORT: "3001" + GF_SERVER_ROUTER_LOGGING: "1" + image: alpine:3.18.3 name: grafana-server - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite dashboards-suite depends_on: - grafana-server @@ -588,7 +566,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-dashboards-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite depends_on: - grafana-server @@ -597,7 +574,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-smoke-tests-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite panels-suite depends_on: - grafana-server @@ -606,7 +582,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-panels-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite various-suite depends_on: - grafana-server @@ -630,7 +605,7 @@ steps: GITHUB_TOKEN: from_secret: github_token HOST: grafana-server - image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:3.0.0 name: end-to-end-tests-cloud-plugins-suite-azure when: paths: @@ -675,23 +650,16 @@ steps: - yarn storybook:build - ./bin/build verify-storybook depends_on: - - build-frontend + - rgm-package - build-frontend-packages environment: NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: build-storybook when: paths: include: - packages/grafana-ui/** -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.7.5 - name: copy-packages-for-docker - commands: - yarn wait-on http://$HOST:$PORT - pa11y-ci --config .pa11yci-pr.conf.js @@ -706,36 +674,24 @@ steps: image: grafana/docker-puppeteer:1.1.0 name: test-a11y-frontend - commands: - - ./bin/build build-docker --edition oss -archs amd64 + - docker run --privileged --rm tonistiigi/binfmt --install all + - /src/grafana-build docker $(cat packages.txt | grep tar.gz | grep -v docker | + grep -v sha256 | awk '{print "--package=" $0}') --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.18.3 + --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --ubuntu-tag-format='{{ + .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > docker.txt + - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build build-docker --edition oss --ubuntu -archs amd64 - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images-ubuntu + - rgm-package + image: grafana/grafana-build:main + name: rgm-build-docker + pull: always volumes: - name: docker path: /var/run/docker.sock - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana depends_on: - - build-docker-images - - build-docker-images-ubuntu + - rgm-build-docker environment: DOCKER_PASSWORD: from_secret: docker_password @@ -792,16 +748,35 @@ services: volumes: - name: postgres path: /var/lib/postgresql/data/pgdata -- environment: +- commands: + - docker-entrypoint.sh mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + environment: MYSQL_DATABASE: grafana_tests MYSQL_PASSWORD: password MYSQL_ROOT_PASSWORD: rootpass MYSQL_USER: grafana image: mysql:5.7.39 - name: mysql + name: mysql57 volumes: - - name: mysql + - name: mysql57 path: /var/lib/mysql +- commands: + - docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password + environment: + MYSQL_DATABASE: grafana_tests + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_USER: grafana + image: mysql:8.0.32 + name: mysql80 + volumes: + - name: mysql80 + path: /var/lib/mysql +- commands: + - /bin/mimir -target=backend + environment: {} + image: grafana/mimir:latest + name: mimir_backend - environment: {} image: redis:6.2.11-alpine name: redis @@ -810,6 +785,7 @@ services: name: memcached steps: - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -827,11 +803,11 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -840,7 +816,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME @@ -851,28 +827,34 @@ steps: with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: - - apt-get update - - apt-get install -yq postgresql-client - dockerize -wait tcp://postgres:5432 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-postgres +- commands: + - apk add --update build-base + - apk add --update postgresql-client - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find @@ -880,17 +862,21 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-postgres environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: postgres-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root + - dockerize -wait tcp://mysql57:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-5.7 +- commands: + - apk add --update build-base + - apk add --update mysql-client + - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql57 -P 3306 -u root -prootpass - go clean -testcache - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find @@ -898,31 +884,80 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-5.7 environment: GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql - image: grafana/build-container:1.7.5 - name: mysql-integration-tests + MYSQL_HOST: mysql57 + image: golang:1.20.10-alpine + name: mysql-5.7-integration-tests - commands: - - dockerize -wait tcp://redis:6379/0 -timeout 120s + - dockerize -wait tcp://mysql80:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-8.0 +- commands: + - apk add --update build-base + - apk add --update mysql-client + - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql80 -P 3306 -u root + -prootpass + - go clean -testcache + - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) + depends_on: + - wire-install + - wait-for-mysql-8.0 + environment: + GRAFANA_TEST_DB: mysql + MYSQL_HOST: mysql80 + image: golang:1.20.10-alpine + name: mysql-8.0-integration-tests +- commands: + - dockerize -wait tcp://redis:6379 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-redis +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-redis environment: REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: redis-integration-tests - commands: - dockerize -wait tcp://memcached:11211 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-memcached +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-memcached environment: MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: memcached-integration-tests +- commands: + - dockerize -wait tcp://mimir_backend:8080 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-remote-alertmanager +- commands: + - apk add --update build-base + - go clean -testcache + - go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/... + depends_on: + - wire-install + - wait-for-remote-alertmanager + environment: + AM_PASSWORD: test + AM_TENANT_ID: test + AM_URL: http://mimir_backend:8080 + image: golang:1.20.10-alpine + name: remote-alertmanager-integration-tests trigger: event: - pull_request @@ -946,7 +981,10 @@ volumes: - name: postgres temp: medium: memory -- name: mysql +- name: mysql57 + temp: + medium: memory +- name: mysql80 temp: medium: memory --- @@ -973,11 +1011,12 @@ steps: - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: + - pip3 install codespell - codespell -I .codespellignore docs/ - image: grafana/build-container:1.7.5 + image: python:3.8 name: codespell - commands: - yarn run prettier:checkDocs @@ -985,14 +1024,26 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: lint-docs - commands: - mkdir -p /hugo/content/docs/grafana/latest + - 'echo -e ''---\nredirectURL: /docs/grafana/latest/\ntype: redirect\nversioned: + true\n---\n'' > /hugo/content/docs/grafana/_index.md' - cp -r docs/sources/* /hugo/content/docs/grafana/latest/ - cd /hugo && make prod image: grafana/docs-base:dbd975af06 name: build-docs-website +- commands: + - '# It is required that code generated from Thema/CUE be committed and in sync + with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - apk add --update make + - CODEGEN_VERIFY=1 make gen-cue + depends_on: [] + image: golang:1.20.10-alpine + name: verify-gen-cue trigger: event: - pull_request @@ -1002,6 +1053,8 @@ trigger: - docs/** - packages/**/*.md - latest.json + repo: + - grafana/grafana type: docker volumes: - host: @@ -1029,13 +1082,12 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - - ./bin/build shellcheck - depends_on: - - compile-build-cmd - image: grafana/build-container:1.7.5 + - apt-get update -yq && apt-get install shellcheck + - shellcheck -e SC1071 -e SC2162 scripts/**/*.sh + image: ubuntu:22.04 name: shellcheck trigger: event: @@ -1061,6 +1113,182 @@ environment: image_pull_secrets: - dockerconfigjson kind: pipeline +name: pr-integration-benchmarks +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: +- environment: + PGDATA: /var/lib/postgresql/data/pgdata + POSTGRES_DB: grafanatest + POSTGRES_PASSWORD: grafanatest + POSTGRES_USER: grafanatest + image: postgres:12.3-alpine + name: postgres + volumes: + - name: postgres + path: /var/lib/postgresql/data/pgdata +- commands: + - docker-entrypoint.sh mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + environment: + MYSQL_DATABASE: grafana_tests + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_USER: grafana + image: mysql:5.7.39 + name: mysql57 + volumes: + - name: mysql57 + path: /var/lib/mysql +- commands: + - docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password + environment: + MYSQL_DATABASE: grafana_tests + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_USER: grafana + image: mysql:8.0.32 + name: mysql80 + volumes: + - name: mysql80 + path: /var/lib/mysql +- commands: + - /bin/mimir -target=backend + environment: {} + image: grafana/mimir:latest + name: mimir_backend +- environment: {} + image: redis:6.2.11-alpine + name: redis +- environment: {} + image: memcached:1.6.9-alpine + name: memcached +steps: +- commands: + - apk add --update curl jq bash + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + ../grafana-enterprise + - cd ../grafana-enterprise + - if git checkout ${DRONE_SOURCE_BRANCH}; then echo "checked out ${DRONE_SOURCE_BRANCH}"; + elif git checkout ${DRONE_TARGET_BRANCH}; then echo "git checkout ${DRONE_TARGET_BRANCH}"; + else git checkout main; fi + - cd ../ + - ln -s src grafana + - cd ./grafana-enterprise + - ./build.sh + environment: + GITHUB_TOKEN: + from_secret: github_token + failure: ignore + image: alpine/git:2.40.1 + name: clone-enterprise +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] + environment: + CGO_ENABLED: 0 + image: golang:1.20.10-alpine + name: compile-build-cmd +- commands: + - '# It is required that code generated from Thema/CUE be committed and in sync + with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - apk add --update make + - CODEGEN_VERIFY=1 make gen-cue + depends_on: + - clone-enterprise + image: golang:1.20.10-alpine + name: verify-gen-cue +- commands: + - '# It is required that generated jsonnet is committed and in sync with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - apk add --update make + - CODEGEN_VERIFY=1 make gen-jsonnet + depends_on: + - clone-enterprise + image: golang:1.20.10-alpine + name: verify-gen-jsonnet +- commands: + - apk add --update make + - make gen-go + depends_on: + - verify-gen-cue + image: golang:1.20.10-alpine + name: wire-install +- commands: + - apk add --update build-base + - if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi + - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} + depends_on: + - wire-install + image: golang:1.20.10-alpine + name: sqlite-benchmark-integration-tests +- commands: + - apk add --update build-base + - if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi + - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} + depends_on: + - wire-install + environment: + GRAFANA_TEST_DB: postgres + PGPASSWORD: grafanatest + POSTGRES_HOST: postgres + image: golang:1.20.10-alpine + name: postgres-benchmark-integration-tests +- commands: + - apk add --update build-base + - if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi + - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} + depends_on: + - wire-install + environment: + GRAFANA_TEST_DB: mysql + MYSQL_HOST: mysql57 + image: golang:1.20.10-alpine + name: mysql-5.7-benchmark-integration-tests +- commands: + - apk add --update build-base + - if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi + - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} + depends_on: + - wire-install + environment: + GRAFANA_TEST_DB: mysql + MYSQL_HOST: mysql80 + image: golang:1.20.10-alpine + name: mysql-8.0-benchmark-integration-tests +trigger: + event: + - promote + target: + - gobenchmarks +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +- name: postgres + temp: + medium: memory +- name: mysql57 + temp: + medium: memory +- name: mysql80 + temp: + medium: memory +--- +clone: + retries: 3 +depends_on: [] +environment: + EDITION: oss +image_pull_secrets: +- dockerconfigjson +kind: pipeline name: main-docs node: type: no-parallel @@ -1076,11 +1304,12 @@ steps: - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: + - pip3 install codespell - codespell -I .codespellignore docs/ - image: grafana/build-container:1.7.5 + image: python:3.8 name: codespell - commands: - yarn run prettier:checkDocs @@ -1088,14 +1317,26 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: lint-docs - commands: - mkdir -p /hugo/content/docs/grafana/latest + - 'echo -e ''---\nredirectURL: /docs/grafana/latest/\ntype: redirect\nversioned: + true\n---\n'' > /hugo/content/docs/grafana/_index.md' - cp -r docs/sources/* /hugo/content/docs/grafana/latest/ - cd /hugo && make prod image: grafana/docs-base:dbd975af06 name: build-docs-website +- commands: + - '# It is required that code generated from Thema/CUE be committed and in sync + with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - apk add --update make + - CODEGEN_VERIFY=1 make gen-cue + depends_on: [] + image: golang:1.20.10-alpine + name: verify-gen-cue trigger: branch: main event: @@ -1106,6 +1347,8 @@ trigger: - docs/** - packages/**/*.md - latest.json + repo: + - grafana/grafana type: docker volumes: - host: @@ -1132,22 +1375,17 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: + - apk add --update git bash - yarn betterer ci depends_on: - yarn-install - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: betterer-frontend - commands: - yarn run ci:test-frontend @@ -1155,7 +1393,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: test-frontend trigger: branch: main @@ -1166,6 +1404,8 @@ trigger: - '*.md' - docs/** - latest.json + repo: + - grafana/grafana type: docker volumes: - host: @@ -1195,18 +1435,17 @@ steps: - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: - yarn run prettier:check - yarn run lint - - yarn run i18n:compile - yarn run typecheck depends_on: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: lint-frontend trigger: branch: main @@ -1217,6 +1456,8 @@ trigger: - '*.md' - docs/** - latest.json + repo: + - grafana/grafana type: docker volumes: - host: @@ -1243,49 +1484,47 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: + - apk add --update build-base shared-mime-info shared-mime-info-lang - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: test-backend - commands: + - apk add --update build-base - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: test-backend-integration trigger: branch: main @@ -1296,6 +1535,8 @@ trigger: - '*.md' - docs/** - latest.json + repo: + - grafana/grafana type: docker volumes: - host: @@ -1327,21 +1568,22 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: + - apk add --update make - make gen-go depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: - - apt-get update && apt-get install make + - apk add --update make build-base - make lint-go depends_on: - wire-install environment: CGO_ENABLED: "1" - image: golang:1.20.10 + image: golang:1.20.10-alpine name: lint-backend - commands: - ./bin/build verify-drone @@ -1358,6 +1600,8 @@ trigger: - '*.md' - docs/** - latest.json + repo: + - grafana/grafana type: docker volumes: - host: @@ -1386,7 +1630,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1395,106 +1639,89 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: - - ./bin/build build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - apk add --update jq + - new_version=$(cat package.json | jq .version | sed s/pre/${DRONE_BUILD_NUMBER}/g) + - 'echo "New version: $new_version"' + - yarn run lerna version $new_version --exact --no-git-tag-version --no-push --force-publish + -y + - yarn install --mode=update-lockfile depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.7.5 - name: build-backend -- commands: - - ./bin/build build-frontend --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 - name: build-frontend + image: node:18.12.0-alpine + name: update-package-json-version - commands: - - ./bin/build build-frontend-packages --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} + - apk add --update jq bash + - yarn packages:build - yarn packages:pack - ./scripts/validate-npm-packages.sh depends_on: - - compile-build-cmd - yarn-install + - update-package-json-version environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: build-frontend-packages - commands: - - ./bin/build build-plugins --jobs 8 --edition oss + - /src/grafana-build package --distro=linux/amd64,linux/arm64 --go-version=1.20.10 + --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD + > packages.txt depends_on: - - compile-build-cmd - yarn-install - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.5 - name: build-plugins + image: grafana/grafana-build:main + name: rgm-package + pull: always + volumes: + - name: docker + path: /var/run/docker.sock - commands: - - ./bin/build package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --sign + - apk add --update tar bash + - mkdir grafana + - tar --strip-components=1 -xvf ./dist/*amd64.tar.gz -C grafana + - cp -r devenv scripts tools grafana && cd grafana && ./scripts/grafana-server/start-server depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - environment: - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.5 - name: package -- commands: - - ./scripts/grafana-server/start-server - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages + - rgm-package detach: true environment: - ARCH: linux-amd64 - PORT: 3001 - image: grafana/build-container:1.7.5 + GF_APP_MODE: development + GF_SERVER_HTTP_PORT: "3001" + GF_SERVER_ROUTER_LOGGING: "1" + image: alpine:3.18.3 name: grafana-server - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite dashboards-suite depends_on: - grafana-server @@ -1503,7 +1730,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-dashboards-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite depends_on: - grafana-server @@ -1512,7 +1738,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-smoke-tests-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite panels-suite depends_on: - grafana-server @@ -1521,7 +1746,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-panels-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite various-suite depends_on: - grafana-server @@ -1545,7 +1769,7 @@ steps: GITHUB_TOKEN: from_secret: github_token HOST: grafana-server - image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:3.0.0 name: end-to-end-tests-cloud-plugins-suite-azure when: paths: @@ -1590,23 +1814,16 @@ steps: - yarn storybook:build - ./bin/build verify-storybook depends_on: - - build-frontend + - rgm-package - build-frontend-packages environment: NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: build-storybook when: paths: include: - packages/grafana-ui/** -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.7.5 - name: copy-packages-for-docker - commands: - yarn wait-on http://$HOST:$PORT - pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json @@ -1642,49 +1859,39 @@ steps: repo: - grafana/grafana - commands: - - ./scripts/ci-frontend-metrics.sh | ./bin/build publish-metrics $${GRAFANA_MISC_STATS_API_KEY} + - apk add --update bash grep git + - ./scripts/ci-frontend-metrics.sh ./grafana/public/build | ./bin/build publish-metrics + $$GRAFANA_MISC_STATS_API_KEY depends_on: - test-a11y-frontend environment: GRAFANA_MISC_STATS_API_KEY: from_secret: grafana_misc_stats_api_key failure: ignore - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: publish-frontend-metrics when: repo: - grafana/grafana - commands: - - ./bin/build build-docker --edition oss + - docker run --privileged --rm tonistiigi/binfmt --install all + - /src/grafana-build docker $(cat packages.txt | grep tar.gz | grep -v docker | + grep -v sha256 | awk '{print "--package=" $0}') --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.18.3 + --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --ubuntu-tag-format='{{ + .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > docker.txt + - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build build-docker --edition oss --ubuntu - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images-ubuntu + - rgm-package + image: grafana/grafana-build:main + name: rgm-build-docker + pull: always volumes: - name: docker path: /var/run/docker.sock - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana depends_on: - - build-docker-images - - build-docker-images-ubuntu + - rgm-build-docker environment: DOCKER_PASSWORD: from_secret: docker_password @@ -1709,8 +1916,7 @@ steps: - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana-oss depends_on: - - build-docker-images - - build-docker-images-ubuntu + - rgm-build-docker environment: DOCKER_PASSWORD: from_secret: docker_password @@ -1733,16 +1939,18 @@ steps: repo: - grafana/grafana - commands: + - apk add --update bash - ./scripts/publish-npm-packages.sh --dist-tag 'canary' --registry 'https://registry.npmjs.org' depends_on: - end-to-end-tests-dashboards-suite - end-to-end-tests-panels-suite - end-to-end-tests-smoke-tests-suite - end-to-end-tests-various-suite + - build-frontend-packages environment: NPM_TOKEN: from_secret: npm_token - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: release-canary-npm-packages when: paths: @@ -1790,6 +1998,8 @@ trigger: - '*.md' - docs/** - latest.json + repo: + - grafana/grafana type: docker volumes: - host: @@ -1821,16 +2031,35 @@ services: volumes: - name: postgres path: /var/lib/postgresql/data/pgdata -- environment: +- commands: + - docker-entrypoint.sh mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + environment: MYSQL_DATABASE: grafana_tests MYSQL_PASSWORD: password MYSQL_ROOT_PASSWORD: rootpass MYSQL_USER: grafana image: mysql:5.7.39 - name: mysql + name: mysql57 volumes: - - name: mysql + - name: mysql57 path: /var/lib/mysql +- commands: + - docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password + environment: + MYSQL_DATABASE: grafana_tests + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_USER: grafana + image: mysql:8.0.32 + name: mysql80 + volumes: + - name: mysql80 + path: /var/lib/mysql +- commands: + - /bin/mimir -target=backend + environment: {} + image: grafana/mimir:latest + name: mimir_backend - environment: {} image: redis:6.2.11-alpine name: redis @@ -1840,7 +2069,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1849,7 +2078,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME @@ -1860,28 +2089,34 @@ steps: with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: - - apt-get update - - apt-get install -yq postgresql-client - dockerize -wait tcp://postgres:5432 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-postgres +- commands: + - apk add --update build-base + - apk add --update postgresql-client - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find @@ -1889,17 +2124,21 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-postgres environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: postgres-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root + - dockerize -wait tcp://mysql57:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-5.7 +- commands: + - apk add --update build-base + - apk add --update mysql-client + - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql57 -P 3306 -u root -prootpass - go clean -testcache - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find @@ -1907,31 +2146,80 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-5.7 environment: GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql - image: grafana/build-container:1.7.5 - name: mysql-integration-tests + MYSQL_HOST: mysql57 + image: golang:1.20.10-alpine + name: mysql-5.7-integration-tests - commands: - - dockerize -wait tcp://redis:6379/0 -timeout 120s + - dockerize -wait tcp://mysql80:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-8.0 +- commands: + - apk add --update build-base + - apk add --update mysql-client + - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql80 -P 3306 -u root + -prootpass + - go clean -testcache + - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) + depends_on: + - wire-install + - wait-for-mysql-8.0 + environment: + GRAFANA_TEST_DB: mysql + MYSQL_HOST: mysql80 + image: golang:1.20.10-alpine + name: mysql-8.0-integration-tests +- commands: + - dockerize -wait tcp://redis:6379 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-redis +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-redis environment: REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: redis-integration-tests - commands: - dockerize -wait tcp://memcached:11211 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-memcached +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-memcached environment: MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: memcached-integration-tests +- commands: + - dockerize -wait tcp://mimir_backend:8080 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-remote-alertmanager +- commands: + - apk add --update build-base + - go clean -testcache + - go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/... + depends_on: + - wire-install + - wait-for-remote-alertmanager + environment: + AM_PASSWORD: test + AM_TENANT_ID: test + AM_URL: http://mimir_backend:8080 + image: golang:1.20.10-alpine + name: remote-alertmanager-integration-tests trigger: branch: main event: @@ -1941,6 +2229,8 @@ trigger: - '*.md' - docs/** - latest.json + repo: + - grafana/grafana type: docker volumes: - host: @@ -1949,7 +2239,10 @@ volumes: - name: postgres temp: medium: memory -- name: mysql +- name: mysql57 + temp: + medium: memory +- name: mysql80 temp: medium: memory --- @@ -1978,33 +2271,10 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: windows-init -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - - .\grabpl.exe windows-installer --edition oss --build-id $$env:DRONE_BUILD_NUMBER - - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp "$$fname" gs://grafana-downloads/oss/main/ - - gsutil cp "$$fname.sha256" gs://grafana-downloads/oss/main/ - depends_on: - - windows-init - environment: - GCP_KEY: - from_secret: gcp_grafanauploads_base64 - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer trigger: branch: main event: @@ -2022,38 +2292,6 @@ volumes: path: //./pipe/docker_engine/ name: docker --- -clone: - retries: 3 -depends_on: [] -kind: pipeline -name: notify-drone-changes -platform: - arch: amd64 - os: linux -steps: -- image: plugins/slack - name: slack - settings: - channel: slack-webhooks-test - template: "`.drone.yml` and `starlark` files have been changed on the OSS repo, - by: {{build.author}}. \nBranch: \nCommit hash: " - webhook: - from_secret: drone-changes-webhook -trigger: - branch: main - event: - - push - paths: - exclude: - - exclude - include: - - .drone.yml - repo: - - grafana/grafana -type: docker ---- clone: retries: 3 depends_on: @@ -2133,6 +2371,8 @@ trigger: - '*.md' - docs/** - latest.json + repo: + - grafana/grafana status: - failure type: docker @@ -2159,7 +2399,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2168,7 +2408,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - ./bin/build artifacts docker fetch --edition oss @@ -2264,7 +2504,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - ./bin/build artifacts packages --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET} @@ -2333,12 +2573,12 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: - ./bin/build artifacts npm retrieve --tag ${DRONE_TAG} @@ -2362,7 +2602,7 @@ steps: NPM_TOKEN: from_secret: npm_token failure: ignore - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: release-npm-packages trigger: event: @@ -2398,7 +2638,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - depends_on: - compile-build-cmd @@ -2465,6 +2705,79 @@ volumes: path: /var/run/docker.sock name: docker --- +clone: + retries: 3 +depends_on: +- main-test-backend +- main-test-frontend +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-main-prerelease +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_publish_main.sh + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.20.10 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: grafana/grafana-build:main + name: rgm-build + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + branch: main + event: + - push + paths: + exclude: + - '*.md' + - docs/** + - packages/**/*.md + - latest.json + repo: + - grafana/grafana +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- clone: retries: 3 depends_on: [] @@ -2486,13 +2799,13 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - ./bin/build whatsnew-checker depends_on: - compile-build-cmd - image: golang:1.20.10 + image: golang:1.20.10-alpine name: whats-new-checker trigger: event: @@ -2529,22 +2842,17 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: + - apk add --update git bash - yarn betterer ci depends_on: - yarn-install - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: betterer-frontend - commands: - yarn run ci:test-frontend @@ -2552,7 +2860,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: test-frontend trigger: event: @@ -2589,49 +2897,47 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: + - apk add --update build-base shared-mime-info shared-mime-info-lang - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: test-backend - commands: + - apk add --update build-base - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: test-backend-integration trigger: event: @@ -2651,64 +2957,6 @@ volumes: clone: retries: 3 depends_on: -- main-test-backend -- main-test-frontend -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: rgm-main-prerelease -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - export GRAFANA_DIR=$$(pwd) - - cd /src && ./scripts/drone_publish_main.sh - environment: - _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: - from_secret: dagger_token - DESTINATION: - from_secret: destination - GCP_KEY_BASE64: - from_secret: gcp_key_base64 - GITHUB_TOKEN: - from_secret: github_token - GPG_PASSPHRASE: - from_secret: packages_gpg_passphrase - GPG_PRIVATE_KEY: - from_secret: packages_gpg_private_key - GPG_PUBLIC_KEY: - from_secret: packages_gpg_public_key - failure: ignore - image: grafana/grafana-build:main - name: rgm-build - volumes: - - name: docker - path: /var/run/docker.sock -trigger: - branch: main - event: - - push - paths: - exclude: - - '*.md' - - docs/** - - packages/**/*.md - - latest.json - repo: - - grafana/grafana -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: - release-test-backend - release-test-frontend image_pull_secrets: @@ -2728,20 +2976,36 @@ steps: environment: _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination DESTINATION: from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key GCP_KEY_BASE64: from_secret: gcp_key_base64 GITHUB_TOKEN: from_secret: github_token + GO_VERSION: 1.20.10 GPG_PASSPHRASE: from_secret: packages_gpg_passphrase GPG_PRIVATE_KEY: from_secret: packages_gpg_private_key GPG_PUBLIC_KEY: from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination image: grafana/grafana-build:main name: rgm-build + pull: always volumes: - name: docker path: /var/run/docker.sock @@ -2776,14 +3040,12 @@ services: [] steps: - commands: - echo $env:DRONE_RUNNER_NAME - failure: ignore image: mcr.microsoft.com/windows:1809 name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/windows/grabpl.exe -OutFile grabpl.exe - failure: ignore image: grafana/ci-wix:0.1.1 name: windows-init - commands: @@ -2797,7 +3059,7 @@ steps: - .\grabpl.exe windows-installer --target gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/grafana-${DRONE_TAG:1}.windows-amd64.zip --edition oss ${DRONE_TAG} - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp "$$fname" gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/ + - gsutil cp $$fname gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/ - gsutil cp "$$fname.sha256" gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/ depends_on: - windows-init @@ -2808,7 +3070,6 @@ steps: from_secret: github_token PRERELEASE_BUCKET: from_secret: prerelease_bucket - failure: ignore image: grafana/ci-wix:0.1.1 name: build-windows-installer trigger: @@ -2871,6 +3132,488 @@ volumes: path: /var/run/docker.sock name: docker --- +clone: + retries: 3 +depends_on: +- release-test-backend +- release-test-frontend +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-version-branch-prerelease +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_publish_tag_grafana.sh + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.20.10 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: grafana/grafana-build:main + name: rgm-build + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + ref: + - refs/heads/v[0-9]* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: +- rgm-version-branch-prerelease +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-prerelease-verify-prerelease-assets +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - apt-get update && apt-get install -yq gettext + - printenv GCP_KEY | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - ./scripts/list-release-artifacts.sh ${DRONE_TAG} | xargs -n1 gsutil stat >> /tmp/stat.log + - '! cat /tmp/stat.log | grep "No URLs matched"' + depends_on: + - clone + environment: + BUCKET: grafana-prerelease + GCP_KEY: + from_secret: gcp_key_base64 + image: google/cloud-sdk:431.0.0 + name: gsutil-stat +trigger: + ref: + - refs/heads/v[0-9]* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: [] +environment: + EDITION: oss +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: nightly-test-frontend +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.18.3 + name: identify-runner +- commands: + - yarn install --immutable + depends_on: [] + image: node:18.12.0-alpine + name: yarn-install +- commands: + - apk add --update git bash + - yarn betterer ci + depends_on: + - yarn-install + image: node:18.12.0-alpine + name: betterer-frontend +- commands: + - yarn run ci:test-frontend + depends_on: + - yarn-install + environment: + TEST_MAX_WORKERS: 50% + image: node:18.12.0-alpine + name: test-frontend +trigger: + cron: + include: + - nightly-release + event: + include: + - cron +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: [] +environment: + EDITION: oss +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: nightly-test-backend +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.18.3 + name: identify-runner +- commands: + - '# It is required that code generated from Thema/CUE be committed and in sync + with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - apk add --update make + - CODEGEN_VERIFY=1 make gen-cue + depends_on: [] + image: golang:1.20.10-alpine + name: verify-gen-cue +- commands: + - '# It is required that generated jsonnet is committed and in sync with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - apk add --update make + - CODEGEN_VERIFY=1 make gen-jsonnet + depends_on: [] + image: golang:1.20.10-alpine + name: verify-gen-jsonnet +- commands: + - apk add --update make + - make gen-go + depends_on: + - verify-gen-cue + image: golang:1.20.10-alpine + name: wire-install +- commands: + - apk add --update build-base shared-mime-info shared-mime-info-lang + - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... + depends_on: + - wire-install + image: golang:1.20.10-alpine + name: test-backend +- commands: + - apk add --update build-base + - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) + depends_on: + - wire-install + image: golang:1.20.10-alpine + name: test-backend-integration +trigger: + cron: + include: + - nightly-release + event: + include: + - cron +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: +- nightly-test-backend +- nightly-test-frontend +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-nightly-build +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_build_nightly_grafana.sh + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.20.10 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: grafana/grafana-build:main + name: rgm-build + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - mkdir -p $${DESTINATION}/$${DRONE_BUILD_EVENT} + - printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - gcloud storage cp -r $${DRONE_WORKSPACE}/dist/* $${DESTINATION}/$${DRONE_BUILD_EVENT} + depends_on: + - rgm-build + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: google/cloud-sdk:alpine + name: rgm-copy +trigger: + cron: + include: + - nightly-release + event: + include: + - cron +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: +- rgm-nightly-build +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-nightly-publish +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p $${DRONE_WORKSPACE}/dist + - printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - gcloud storage cp -r $${DESTINATION}/$${DRONE_BUILD_EVENT}/*_$${DRONE_BUILD_NUMBER}_* + $${DRONE_WORKSPACE}/dist + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: google/cloud-sdk:alpine + name: rgm-copy +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_publish_nightly_grafana.sh + depends_on: + - rgm-copy + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.20.10 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: grafana/grafana-build:main + name: rgm-publish + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +- depends_on: + - rgm-publish + image: us.gcr.io/kubernetes-dev/package-publish:latest + name: publish-deb + privileged: true + settings: + access_key_id: + from_secret: packages_access_key_id + gpg_passphrase: + from_secret: packages_gpg_passphrase + gpg_private_key: + from_secret: packages_gpg_private_key + gpg_public_key: + from_secret: packages_gpg_public_key + package_path: file:///drone/src/dist/*.deb + secret_access_key: + from_secret: packages_secret_access_key + service_account_json: + from_secret: packages_service_account + target_bucket: grafana-packages +- depends_on: + - rgm-publish + image: us.gcr.io/kubernetes-dev/package-publish:latest + name: publish-rpm + privileged: true + settings: + access_key_id: + from_secret: packages_access_key_id + gpg_passphrase: + from_secret: packages_gpg_passphrase + gpg_private_key: + from_secret: packages_gpg_private_key + gpg_public_key: + from_secret: packages_gpg_public_key + package_path: file:///drone/src/dist/*.rpm + secret_access_key: + from_secret: packages_secret_access_key + service_account_json: + from_secret: packages_service_account + target_bucket: grafana-packages +trigger: + cron: + include: + - nightly-release + event: + include: + - cron +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- clone: disable: true depends_on: [] @@ -2897,20 +3640,20 @@ steps: - commands: [] depends_on: - clone - image: grafana/grafana-ci-windows-test:0.1.0 + image: golang:1.20.10-windowsservercore-1809 name: windows-init - commands: - go install github.com/google/wire/cmd/wire@v0.5.0 - wire gen -tags oss ./pkg/server depends_on: - windows-init - image: grafana/grafana-ci-windows-test:0.1.0 + image: golang:1.20.10-windowsservercore-1809 name: wire-install - commands: - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/grafana-ci-windows-test:0.1.0 + image: golang:1.20.10-windowsservercore-1809 name: test-backend trigger: event: @@ -2931,613 +3674,6 @@ environment: image_pull_secrets: - dockerconfigjson kind: pipeline -name: release-branch-build-e2e-publish -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 - name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: [] - image: grafana/build-container:1.7.5 - name: verify-gen-cue -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.7.5 - name: wire-install -- commands: - - yarn install --immutable - depends_on: [] - image: grafana/build-container:1.7.5 - name: yarn-install -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd -- commands: - - ./bin/build build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} - depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.7.5 - name: build-backend -- commands: - - ./bin/build build-frontend --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 - name: build-frontend -- commands: - - ./bin/build build-frontend-packages --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - - yarn packages:pack - - ./scripts/validate-npm-packages.sh - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 - name: build-frontend-packages -- commands: - - ./bin/build build-plugins --jobs 8 --edition oss - depends_on: - - compile-build-cmd - - yarn-install - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.5 - name: build-plugins -- commands: - - ./bin/build package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --sign - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - environment: - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.5 - name: package -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.7.5 - name: copy-packages-for-docker -- commands: - - ./bin/build build-docker --edition oss --shouldSave - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build build-docker --edition oss --shouldSave --ubuntu - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images-ubuntu - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./scripts/grafana-server/start-server - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - detach: true - environment: - ARCH: linux-amd64 - PORT: 3001 - image: grafana/build-container:1.7.5 - name: grafana-server -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite dashboards-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-dashboards-suite -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-smoke-tests-suite -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite panels-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-panels-suite -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite various-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-various-suite -- commands: - - apt-get update - - apt-get install -yq zip - - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ - - gsutil cp e2e/videos.zip gs://$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - export E2E_ARTIFACTS_VIDEO_ZIP=https://storage.googleapis.com/$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - 'echo "E2E Test artifacts uploaded to: $${E2E_ARTIFACTS_VIDEO_ZIP}"' - - 'curl -X POST https://api.github.com/repos/${DRONE_REPO}/statuses/${DRONE_COMMIT_SHA} - -H "Authorization: token $${GITHUB_TOKEN}" -d "{\"state\":\"success\",\"target_url\":\"$${E2E_ARTIFACTS_VIDEO_ZIP}\", - \"description\": \"Click on the details to download e2e recording videos\", \"context\": - \"e2e_artifacts\"}"' - depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - E2E_TEST_ARTIFACTS_BUCKET: releng-pipeline-artifacts-dev - GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY: - from_secret: gcp_upload_artifacts_key - GITHUB_TOKEN: - from_secret: github_token - failure: ignore - image: google/cloud-sdk:431.0.0 - name: e2e-tests-artifacts-upload - when: - status: - - success - - failure -- commands: - - yarn storybook:build - - ./bin/build verify-storybook - depends_on: - - build-frontend - - build-frontend-packages - environment: - NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.7.5 - name: build-storybook - when: - paths: - include: - - packages/grafana-ui/** -- commands: - - ./bin/build upload-cdn --edition oss - depends_on: - - grafana-server - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-cdn-assets - when: - repo: - - grafana/grafana -- commands: - - ./bin/build upload-packages --edition oss - depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - GCP_KEY: - from_secret: gcp_grafanauploads_base64 - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-packages - when: - repo: - - grafana/grafana -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql - temp: - medium: memory ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-test-frontend -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 - name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - yarn install --immutable - depends_on: [] - image: grafana/build-container:1.7.5 - name: yarn-install -- commands: - - yarn betterer ci - depends_on: - - yarn-install - image: grafana/build-container:1.7.5 - name: betterer-frontend -- commands: - - yarn run ci:test-frontend - depends_on: - - yarn-install - environment: - TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 - name: test-frontend -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-test-backend -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 - name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: [] - image: grafana/build-container:1.7.5 - name: verify-gen-cue -- commands: - - '# It is required that generated jsonnet is committed and in sync with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-jsonnet - depends_on: [] - image: grafana/build-container:1.7.5 - name: verify-gen-jsonnet -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.7.5 - name: wire-install -- commands: - - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... - depends_on: - - wire-install - image: grafana/build-container:1.7.5 - name: test-backend -- commands: - - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find - ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' - | grep -o '\(.*\)/' | sort -u) - depends_on: - - wire-install - image: grafana/build-container:1.7.5 - name: test-backend-integration -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-integration-tests -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: -- environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_DB: grafanatest - POSTGRES_PASSWORD: grafanatest - POSTGRES_USER: grafanatest - image: postgres:12.3-alpine - name: postgres - volumes: - - name: postgres - path: /var/lib/postgresql/data/pgdata -- environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:5.7.39 - name: mysql - volumes: - - name: mysql - path: /var/lib/mysql -- environment: {} - image: redis:6.2.11-alpine - name: redis -- environment: {} - image: memcached:1.6.9-alpine - name: memcached -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 - name: identify-runner -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: [] - image: grafana/build-container:1.7.5 - name: verify-gen-cue -- commands: - - '# It is required that generated jsonnet is committed and in sync with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-jsonnet - depends_on: [] - image: grafana/build-container:1.7.5 - name: verify-gen-jsonnet -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.7.5 - name: wire-install -- commands: - - apt-get update - - apt-get install -yq postgresql-client - - dockerize -wait tcp://postgres:5432 -timeout 120s - - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - - go clean -testcache - - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find - ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' - | grep -o '\(.*\)/' | sort -u) - depends_on: - - wire-install - environment: - GRAFANA_TEST_DB: postgres - PGPASSWORD: grafanatest - POSTGRES_HOST: postgres - image: grafana/build-container:1.7.5 - name: postgres-integration-tests -- commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root - -prootpass - - go clean -testcache - - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find - ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' - | grep -o '\(.*\)/' | sort -u) - depends_on: - - wire-install - environment: - GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql - image: grafana/build-container:1.7.5 - name: mysql-integration-tests -- commands: - - dockerize -wait tcp://redis:6379/0 -timeout 120s - - go clean -testcache - - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... - depends_on: - - wire-install - environment: - REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.7.5 - name: redis-integration-tests -- commands: - - dockerize -wait tcp://memcached:11211 -timeout 120s - - go clean -testcache - - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... - depends_on: - - wire-install - environment: - MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.7.5 - name: memcached-integration-tests -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql - temp: - medium: memory ---- -clone: - retries: 3 -depends_on: -- release-branch-build-e2e-publish -- release-branch-test-frontend -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-windows -platform: - arch: amd64 - os: windows - version: "1809" -services: [] -steps: -- commands: - - echo $env:DRONE_RUNNER_NAME - image: mcr.microsoft.com/windows:1809 - name: identify-runner -- commands: - - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/windows/grabpl.exe - -OutFile grabpl.exe - image: grafana/ci-wix:0.1.1 - name: windows-init -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - depends_on: - - windows-init - environment: - GCP_KEY: - from_secret: gcp_grafanauploads_base64 - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: //./pipe/docker_engine/ - name: docker ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline name: integration-tests node: type: no-parallel @@ -3555,16 +3691,35 @@ services: volumes: - name: postgres path: /var/lib/postgresql/data/pgdata -- environment: +- commands: + - docker-entrypoint.sh mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + environment: MYSQL_DATABASE: grafana_tests MYSQL_PASSWORD: password MYSQL_ROOT_PASSWORD: rootpass MYSQL_USER: grafana image: mysql:5.7.39 - name: mysql + name: mysql57 volumes: - - name: mysql + - name: mysql57 path: /var/lib/mysql +- commands: + - docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password + environment: + MYSQL_DATABASE: grafana_tests + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_USER: grafana + image: mysql:8.0.32 + name: mysql80 + volumes: + - name: mysql80 + path: /var/lib/mysql +- commands: + - /bin/mimir -target=backend + environment: {} + image: grafana/mimir:latest + name: mimir_backend - environment: {} image: redis:6.2.11-alpine name: redis @@ -3574,7 +3729,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -3587,28 +3742,34 @@ steps: with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: - - apt-get update - - apt-get install -yq postgresql-client - dockerize -wait tcp://postgres:5432 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-postgres +- commands: + - apk add --update build-base + - apk add --update postgresql-client - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find @@ -3616,17 +3777,21 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-postgres environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: postgres-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root + - dockerize -wait tcp://mysql57:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-5.7 +- commands: + - apk add --update build-base + - apk add --update mysql-client + - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql57 -P 3306 -u root -prootpass - go clean -testcache - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find @@ -3634,31 +3799,80 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-5.7 environment: GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql - image: grafana/build-container:1.7.5 - name: mysql-integration-tests + MYSQL_HOST: mysql57 + image: golang:1.20.10-alpine + name: mysql-5.7-integration-tests - commands: - - dockerize -wait tcp://redis:6379/0 -timeout 120s + - dockerize -wait tcp://mysql80:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-8.0 +- commands: + - apk add --update build-base + - apk add --update mysql-client + - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql80 -P 3306 -u root + -prootpass + - go clean -testcache + - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) + depends_on: + - wire-install + - wait-for-mysql-8.0 + environment: + GRAFANA_TEST_DB: mysql + MYSQL_HOST: mysql80 + image: golang:1.20.10-alpine + name: mysql-8.0-integration-tests +- commands: + - dockerize -wait tcp://redis:6379 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-redis +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-redis environment: REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: redis-integration-tests - commands: - dockerize -wait tcp://memcached:11211 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-memcached +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-memcached environment: MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: memcached-integration-tests +- commands: + - dockerize -wait tcp://mimir_backend:8080 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-remote-alertmanager +- commands: + - apk add --update build-base + - go clean -testcache + - go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/... + depends_on: + - wire-install + - wait-for-remote-alertmanager + environment: + AM_PASSWORD: test + AM_TENANT_ID: test + AM_URL: http://mimir_backend:8080 + image: golang:1.20.10-alpine + name: remote-alertmanager-integration-tests trigger: event: - promote @@ -3671,7 +3885,10 @@ volumes: - name: postgres temp: medium: memory -- name: mysql +- name: mysql57 + temp: + medium: memory +- name: mysql80 temp: medium: memory --- @@ -3797,7 +4014,7 @@ steps: - name: config path: /root/.docker/ - commands: - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest + - trivy image --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -3805,15 +4022,22 @@ steps: volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - commands: - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest + - trivy image --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest depends_on: - authenticate-gcr + environment: + GOOGLE_APPLICATION_CREDENTIALS: + from_secret: gcr_credentials_json image: aquasec/trivy:0.21.0 name: scan-high-critical-vulnerabilities volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - image: plugins/slack name: slack-notify-failure settings: @@ -3831,6 +4055,8 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: config + temp: {} --- clone: retries: 3 @@ -3853,7 +4079,7 @@ steps: - name: config path: /root/.docker/ - commands: - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main + - trivy image --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -3861,15 +4087,22 @@ steps: volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - commands: - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main + - trivy image --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main depends_on: - authenticate-gcr + environment: + GOOGLE_APPLICATION_CREDENTIALS: + from_secret: gcr_credentials_json image: aquasec/trivy:0.21.0 name: scan-high-critical-vulnerabilities volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - image: plugins/slack name: slack-notify-failure settings: @@ -3887,6 +4120,8 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: config + temp: {} --- clone: retries: 3 @@ -3909,7 +4144,7 @@ steps: - name: config path: /root/.docker/ - commands: - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest-ubuntu + - trivy image --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest-ubuntu depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -3917,15 +4152,22 @@ steps: volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - commands: - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest-ubuntu + - trivy image --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest-ubuntu depends_on: - authenticate-gcr + environment: + GOOGLE_APPLICATION_CREDENTIALS: + from_secret: gcr_credentials_json image: aquasec/trivy:0.21.0 name: scan-high-critical-vulnerabilities volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - image: plugins/slack name: slack-notify-failure settings: @@ -3944,6 +4186,8 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: config + temp: {} --- clone: retries: 3 @@ -3966,7 +4210,7 @@ steps: - name: config path: /root/.docker/ - commands: - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main-ubuntu + - trivy image --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main-ubuntu depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -3974,15 +4218,22 @@ steps: volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - commands: - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main-ubuntu + - trivy image --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main-ubuntu depends_on: - authenticate-gcr + environment: + GOOGLE_APPLICATION_CREDENTIALS: + from_secret: gcr_credentials_json image: aquasec/trivy:0.21.0 name: scan-high-critical-vulnerabilities volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - image: plugins/slack name: slack-notify-failure settings: @@ -4001,6 +4252,8 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: config + temp: {} --- clone: retries: 3 @@ -4023,14 +4276,18 @@ steps: - name: config path: /root/.docker/ - commands: + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine/git:2.40.1 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM golang:1.20.10-alpine + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM node:18.12.0-alpine - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM google/cloud-sdk:431.0.0 - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/build-container:1.7.5 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana-ci-deploy:1.3.3 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine:3.18.3 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM ubuntu:22.04 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM byrnedo/alpine-curl:0.1.8 - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM golang:1.20.10 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM plugins/slack + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM python:3.8 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM postgres:12.3-alpine + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/mimir:latest - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:5.7.39 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:8.0.32 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM redis:6.2.11-alpine @@ -4041,7 +4298,8 @@ steps: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docker-puppeteer:1.1.0 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docs-base:dbd975af06 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM jwilder/dockerize:0.6.1 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM koalaman/shellcheck:stable depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -4049,15 +4307,21 @@ steps: volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - commands: + - trivy --exit-code 1 --severity HIGH,CRITICAL alpine/git:2.40.1 + - trivy --exit-code 1 --severity HIGH,CRITICAL golang:1.20.10-alpine + - trivy --exit-code 1 --severity HIGH,CRITICAL node:18.12.0-alpine - trivy --exit-code 1 --severity HIGH,CRITICAL google/cloud-sdk:431.0.0 - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/build-container:1.7.5 - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana-ci-deploy:1.3.3 - trivy --exit-code 1 --severity HIGH,CRITICAL alpine:3.18.3 + - trivy --exit-code 1 --severity HIGH,CRITICAL ubuntu:22.04 - trivy --exit-code 1 --severity HIGH,CRITICAL byrnedo/alpine-curl:0.1.8 - - trivy --exit-code 1 --severity HIGH,CRITICAL golang:1.20.10 - trivy --exit-code 1 --severity HIGH,CRITICAL plugins/slack + - trivy --exit-code 1 --severity HIGH,CRITICAL python:3.8 - trivy --exit-code 1 --severity HIGH,CRITICAL postgres:12.3-alpine + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/mimir:latest - trivy --exit-code 1 --severity HIGH,CRITICAL mysql:5.7.39 - trivy --exit-code 1 --severity HIGH,CRITICAL mysql:8.0.32 - trivy --exit-code 1 --severity HIGH,CRITICAL redis:6.2.11-alpine @@ -4068,14 +4332,20 @@ steps: - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docker-puppeteer:1.1.0 - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docs-base:dbd975af06 - trivy --exit-code 1 --severity HIGH,CRITICAL cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - - trivy --exit-code 1 --severity HIGH,CRITICAL us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + - trivy --exit-code 1 --severity HIGH,CRITICAL jwilder/dockerize:0.6.1 + - trivy --exit-code 1 --severity HIGH,CRITICAL koalaman/shellcheck:stable depends_on: - authenticate-gcr + environment: + GOOGLE_APPLICATION_CREDENTIALS: + from_secret: gcr_credentials_json image: aquasec/trivy:0.21.0 name: scan-high-critical-vulnerabilities volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - image: plugins/slack name: slack-notify-failure settings: @@ -4093,41 +4363,8 @@ volumes: - host: path: /var/run/docker.sock name: docker ---- -clone: - retries: 3 -kind: pipeline -name: grafana-com-nightly -platform: - arch: amd64 - os: linux -steps: -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd -- commands: - - ./bin/build publish grafana-com --edition oss - depends_on: - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - GRAFANA_COM_API_KEY: - from_secret: grafana_api_key - image: grafana/grafana-ci-deploy:1.3.3 - name: post-to-grafana-com -trigger: - cron: grafana-com-nightly - event: cron -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker +- name: config + temp: {} --- get: name: credentials.json @@ -4255,12 +4492,6 @@ get: kind: secret name: packages_secret_access_key --- -get: - name: bucket - path: infra/data/ci/grafana-release-eng/security-bucket -kind: secret -name: security_dest_bucket ---- get: name: static_asset_editions path: infra/data/ci/grafana-release-eng/artifact-publishing @@ -4279,6 +4510,24 @@ get: kind: secret name: destination --- +get: + name: storybook_destination + path: infra/data/ci/grafana-release-eng/rgm +kind: secret +name: rgm_storybook_destination +--- +get: + name: cdn_destination + path: infra/data/ci/grafana-release-eng/rgm +kind: secret +name: rgm_cdn_destination +--- +get: + name: downloads_destination + path: infra/data/ci/grafana-release-eng/rgm +kind: secret +name: rgm_downloads_destination +--- get: name: dagger_token path: infra/data/ci/grafana-release-eng/rgm @@ -4310,6 +4559,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: 0808b6f03f013b661d4971f2602907805ecbf24a61f6bd2f33d688aa1ff675e8 +hmac: 4c68ef8aedb7202b86f077dbccc04504b81eaa1729276238d90a06672198487e ... diff --git a/Makefile b/Makefile index 83140ce0eef..6653f7d240c 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,6 @@ GO = go GO_FILES ?= ./pkg/... SH_FILES ?= $(shell find ./scripts -name *.sh) GO_BUILD_FLAGS += $(if $(GO_BUILD_DEV),-dev) -GO_BUILD_FLAGS += $(if $(GO_BUILD_DEV),-dev) GO_BUILD_FLAGS += $(if $(GO_BUILD_TAGS),-build-tags=$(GO_BUILD_TAGS)) targets := $(shell echo '$(sources)' | tr "," " ") @@ -37,36 +36,68 @@ node_modules: package.json yarn.lock ## Install node modules. ##@ Swagger SPEC_TARGET = public/api-spec.json -MERGED_SPEC_TARGET := public/api-merged.json +ENTERPRISE_SPEC_TARGET = public/api-enterprise-spec.json +MERGED_SPEC_TARGET = public/api-merged.json NGALERT_SPEC_TARGET = pkg/services/ngalert/api/tooling/api.json $(NGALERT_SPEC_TARGET): +$(MAKE) -C pkg/services/ngalert/api/tooling api.json -$(MERGED_SPEC_TARGET): $(SPEC_TARGET) $(NGALERT_SPEC_TARGET) $(SWAGGER) ## Merge generated and ngalert API specs +$(MERGED_SPEC_TARGET): swagger-oss-gen swagger-enterprise-gen $(NGALERT_SPEC_TARGET) $(SWAGGER) ## Merge generated and ngalert API specs # known conflicts DsPermissionType, AddApiKeyCommand, Json, Duration (identical models referenced by both specs) - $(SWAGGER) mixin $(SPEC_TARGET) $(NGALERT_SPEC_TARGET) --ignore-conflicts -o $(MERGED_SPEC_TARGET) + $(SWAGGER) mixin $(SPEC_TARGET) $(ENTERPRISE_SPEC_TARGET) $(NGALERT_SPEC_TARGET) --ignore-conflicts -o $(MERGED_SPEC_TARGET) -$(SPEC_TARGET): $(SWAGGER) ## Generate API Swagger specification +swagger-oss-gen: $(SWAGGER) ## Generate API Swagger specification + @echo "re-generating swagger for OSS" + rm -f $(SPEC_TARGET) SWAGGER_GENERATE_EXTENSION=false $(SWAGGER) generate spec -m -w pkg/server -o $(SPEC_TARGET) \ -x "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" \ -x "github.com/prometheus/alertmanager" \ -i pkg/api/swagger_tags.json \ - --exclude-tag=alpha - go run pkg/services/ngalert/api/tooling/cmd/clean-swagger/main.go -if $@ -of $@ + --exclude-tag=alpha \ + --exclude-tag=enterprise -swagger-api-spec: gen-go $(SPEC_TARGET) $(MERGED_SPEC_TARGET) validate-api-spec +# this file only exists if enterprise is enabled +ENTERPRISE_EXT_FILE = pkg/extensions/ext.go +ifeq ("$(wildcard $(ENTERPRISE_EXT_FILE))","") ## if enterprise is not enabled +swagger-enterprise-gen: + @echo "skipping re-generating swagger for enterprise: not enabled" +else +swagger-enterprise-gen: $(SWAGGER) ## Generate API Swagger specification + @echo "re-generating swagger for enterprise" + rm -f $(ENTERPRISE_SPEC_TARGET) + SWAGGER_GENERATE_EXTENSION=false $(SWAGGER) generate spec -m -w pkg/server -o $(ENTERPRISE_SPEC_TARGET) \ + -x "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" \ + -x "github.com/prometheus/alertmanager" \ + -i pkg/api/swagger_tags.json \ + --exclude-tag=alpha \ + --include-tag=enterprise +endif -validate-api-spec: $(MERGED_SPEC_TARGET) $(SWAGGER) ## Validate API spec +swagger-gen: gen-go $(MERGED_SPEC_TARGET) swagger-validate + +swagger-validate: $(MERGED_SPEC_TARGET) $(SWAGGER) ## Validate API spec $(SWAGGER) validate $(<) -clean-api-spec: +swagger-clean: rm -f $(SPEC_TARGET) $(MERGED_SPEC_TARGET) $(OAPI_SPEC_TARGET) +.PHONY: cleanup-old-git-hooks +cleanup-old-git-hooks: + ./scripts/cleanup-husky.sh + +.PHONY: lefthook-install +lefthook-install: cleanup-old-git-hooks $(LEFTHOOK) # install lefthook for pre-commit hooks + $(LEFTHOOK) install -f + +.PHONY: lefthook-uninstall +lefthook-uninstall: $(LEFTHOOK) + $(LEFTHOOK) uninstall + ##@ OpenAPI 3 OAPI_SPEC_TARGET = public/openapi3.json -openapi3-gen: swagger-api-spec ## Generates OpenApi 3 specs from the Swagger 2 already generated +openapi3-gen: swagger-gen ## Generates OpenApi 3 specs from the Swagger 2 already generated $(GO) run scripts/openapi3/openapi3conv.go $(MERGED_SPEC_TARGET) $(OAPI_SPEC_TARGET) ##@ Building @@ -75,9 +106,9 @@ gen-cue: ## Do all CUE/Thema code generation go generate ./pkg/plugins/plugindef go generate ./kinds/gen.go go generate ./public/app/plugins/gen.go - go generate ./pkg/kindsys/report.go +# go generate ./pkg/kindsysreport/codegen/report.go -gen-go: $(WIRE) gen-cue +gen-go: $(WIRE) @echo "generate go files" $(WIRE) gen -tags $(WIRE_TAGS) ./pkg/server @@ -133,6 +164,13 @@ test-go-integration: ## Run integration tests for backend with flags. @echo "test backend integration tests" $(GO) test -count=1 -run "^TestIntegration" -covermode=atomic -timeout=5m $(GO_INTEGRATION_TESTS) +.PHONY: test-go-integration-alertmanager +test-go-integration-alertmanager: ## Run integration tests for the remote alertmanager (config taken from the mimir_backend block). + @echo "test remote alertmanager integration tests" + $(GO) clean -testcache + AM_URL=http://localhost:8080 AM_TENANT_ID=test AM_PASSWORD=test \ + $(GO) test -count=1 -run "^TestIntegrationRemoteAlertmanager" -covermode=atomic -timeout=5m ./pkg/services/ngalert/notifier/... + .PHONY: test-go-integration-postgres test-go-integration-postgres: devenv-postgres ## Run integration tests for postgres backend with flags. @echo "test backend integration postgres tests" @@ -191,7 +229,7 @@ build-docker-full: ## Build Docker image for development. --build-arg BINGO=false \ --build-arg GO_BUILD_TAGS=$(GO_BUILD_TAGS) \ --build-arg WIRE_TAGS=$(WIRE_TAGS) \ - --build-arg COMMIT_SHA=$$(git rev-parse --short HEAD) \ + --build-arg COMMIT_SHA=$$(git rev-parse HEAD) \ --build-arg BUILD_BRANCH=$$(git rev-parse --abbrev-ref HEAD) \ --tag grafana/grafana$(TAG_SUFFIX):dev \ $(DOCKER_BUILD_ARGS) @@ -204,9 +242,9 @@ build-docker-full-ubuntu: ## Build Docker image based on Ubuntu for development. --build-arg BINGO=false \ --build-arg GO_BUILD_TAGS=$(GO_BUILD_TAGS) \ --build-arg WIRE_TAGS=$(WIRE_TAGS) \ - --build-arg COMMIT_SHA=$$(git rev-parse --short HEAD) \ + --build-arg COMMIT_SHA=$$(git rev-parse HEAD) \ --build-arg BUILD_BRANCH=$$(git rev-parse --abbrev-ref HEAD) \ - --build-arg BASE_IMAGE=ubuntu:20.04 \ + --build-arg BASE_IMAGE=ubuntu:22.04 \ --build-arg GO_IMAGE=golang:1.20.10 \ --tag grafana/grafana$(TAG_SUFFIX):dev-ubuntu \ $(DOCKER_BUILD_ARGS) @@ -252,6 +290,9 @@ devenv-mysql: protobuf: ## Compile protobuf definitions bash scripts/protobuf-check.sh bash pkg/plugins/backendplugin/pluginextensionv2/generate.sh + bash pkg/plugins/backendplugin/secretsmanagerplugin/generate.sh + bash pkg/services/store/entity/generate.sh + bash pkg/infra/grn/generate.sh clean: ## Clean up intermediate build artifacts. @echo "cleaning" @@ -277,7 +318,7 @@ scripts/drone/TAGS: $(shell find scripts/drone -name '*.star') etags --lang none --regex="/def \(\w+\)[^:]+:/\1/" --regex="/\s*\(\w+\) =/\1/" $^ -o $@ format-drone: - buildifier -r scripts/drone + buildifier --lint=fix -r scripts/drone help: ## Display this help. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/pkg/build/cmd.go b/pkg/build/cmd.go index 7248c91080d..fa98316e9c2 100644 --- a/pkg/build/cmd.go +++ b/pkg/build/cmd.go @@ -222,15 +222,25 @@ func ldflags(opts BuildOpts) (string, error) { commitSha = v } + var enterpriseCommitSha string + if opts.enterprise { + enterpriseCommitSha = getGitEnterpriseSha() + if v := os.Getenv("ENTERPRISE_COMMIT_SHA"); v != "" { + enterpriseCommitSha = v + } + } + buildBranch := getGitBranch() if v := os.Getenv("BUILD_BRANCH"); v != "" { buildBranch = v } - var b bytes.Buffer b.WriteString("-w") b.WriteString(fmt.Sprintf(" -X main.version=%s", opts.version)) b.WriteString(fmt.Sprintf(" -X main.commit=%s", commitSha)) + if enterpriseCommitSha != "" { + b.WriteString(fmt.Sprintf(" -X main.enterpriseCommit=%s", enterpriseCommitSha)) + } b.WriteString(fmt.Sprintf(" -X main.buildstamp=%d", buildStamp)) b.WriteString(fmt.Sprintf(" -X main.buildBranch=%s", buildBranch)) if v := os.Getenv("LDFLAGS"); v != "" { diff --git a/pkg/build/cmd/grafanacom.go b/pkg/build/cmd/grafanacom.go index e56b3367200..9d44b9298c9 100644 --- a/pkg/build/cmd/grafanacom.go +++ b/pkg/build/cmd/grafanacom.go @@ -232,7 +232,7 @@ func getSHA256(u string) ([]byte, error) { return sha256, nil } -func postRequest(cfg packaging.PublishConfig, pth string, obj interface{}, descr string) error { +func postRequest(cfg packaging.PublishConfig, pth string, obj any, descr string) error { var sfx string switch cfg.Edition { case config.EditionOSS: diff --git a/pkg/build/cmd/verifydrone.go b/pkg/build/cmd/verifydrone.go index 4794efa4500..14596377110 100644 --- a/pkg/build/cmd/verifydrone.go +++ b/pkg/build/cmd/verifydrone.go @@ -68,7 +68,7 @@ func VerifyDrone(c *cli.Context) error { return nil } -func readConfig(fpath string) ([]map[string]interface{}, error) { +func readConfig(fpath string) ([]map[string]any, error) { //nolint:gosec f, err := os.Open(fpath) if err != nil { @@ -82,9 +82,9 @@ func readConfig(fpath string) ([]map[string]interface{}, error) { // The YAML stream may contain multiple pipeline configurations, read them all dec := yaml.NewDecoder(f) - var c []map[string]interface{} + var c []map[string]any for { - var m map[string]interface{} + var m map[string]any if err := dec.Decode(&m); err != nil { if errors.Is(err, io.EOF) { break diff --git a/pkg/build/config/genmetadata_test.go b/pkg/build/config/genmetadata_test.go index 8810d4409c4..d8eeb06bb6a 100644 --- a/pkg/build/config/genmetadata_test.go +++ b/pkg/build/config/genmetadata_test.go @@ -72,10 +72,8 @@ func setUpEnv(t *testing.T, envMap map[string]string) { t.Helper() os.Clearenv() - err := os.Setenv("DRONE_COMMIT", "abcd12345") - require.NoError(t, err) + t.Setenv("DRONE_COMMIT", "abcd12345") for k, v := range envMap { - err := os.Setenv(k, v) - require.NoError(t, err) + t.Setenv(k, v) } } diff --git a/pkg/build/config/revision.go b/pkg/build/config/revision.go index aa104916652..70be21cdf18 100644 --- a/pkg/build/config/revision.go +++ b/pkg/build/config/revision.go @@ -3,6 +3,7 @@ package config import ( "context" "fmt" + "log" "strconv" "time" @@ -10,9 +11,10 @@ import ( ) type Revision struct { - Timestamp int64 - SHA256 string - Branch string + Timestamp int64 + SHA256 string + EnterpriseCommit string + Branch string } func GrafanaTimestamp(ctx context.Context, dir string) (int64, error) { @@ -42,14 +44,26 @@ func GrafanaRevision(ctx context.Context, grafanaDir string) (Revision, error) { return Revision{}, err } + enterpriseCommit, err := executil.OutputAt(ctx, grafanaDir, "git", "-C", "../grafana-enterprise", "rev-parse", "--short", "HEAD") + if err != nil { + enterpriseCommit, err = executil.OutputAt(ctx, grafanaDir, "git", "-C", "..", "rev-parse", "--short", "HEAD") + if err != nil { + enterpriseCommit, err = executil.OutputAt(ctx, grafanaDir, "git", "-C", "/tmp/grafana-enterprise", "rev-parse", "--short", "HEAD") + if err != nil { + log.Println("Could not get enterprise commit. Error:", err) + } + } + } + branch, err := executil.OutputAt(ctx, grafanaDir, "git", "rev-parse", "--abbrev-ref", "HEAD") if err != nil { return Revision{}, err } return Revision{ - SHA256: sha, - Branch: branch, - Timestamp: stamp, + SHA256: sha, + EnterpriseCommit: enterpriseCommit, + Branch: branch, + Timestamp: stamp, }, nil } diff --git a/pkg/build/config/version.go b/pkg/build/config/version.go index 5eb96ab63f3..05b99cf4678 100644 --- a/pkg/build/config/version.go +++ b/pkg/build/config/version.go @@ -94,7 +94,7 @@ func GetPackageJSONVersion(grafanaDir string) (string, error) { if err != nil { return "", fmt.Errorf("failed to read %q: %w", pkgJSONPath, err) } - pkgObj := map[string]interface{}{} + pkgObj := map[string]any{} if err := json.Unmarshal(pkgJSONB, &pkgObj); err != nil { return "", fmt.Errorf("failed decoding %q: %w", pkgJSONPath, err) } diff --git a/pkg/build/config/versions.go b/pkg/build/config/versions.go index aa618e90358..8decc72593c 100644 --- a/pkg/build/config/versions.go +++ b/pkg/build/config/versions.go @@ -1,7 +1,5 @@ package config -const PublicBucket = "grafana-downloads" - var Versions = VersionMap{ PullRequestMode: { Variants: []Variant{ @@ -9,8 +7,9 @@ var Versions = VersionMap{ VariantLinuxAmd64Musl, VariantDarwinAmd64, VariantWindowsAmd64, - VariantArm64, - VariantArm64Musl, + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // VariantArm64, + // VariantArm64Musl, }, PluginSignature: PluginSignature{ Sign: false, @@ -29,9 +28,10 @@ var Versions = VersionMap{ }, MainMode: { Variants: []Variant{ - VariantArmV6, - VariantArmV7, - VariantArmV7Musl, + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // VariantArmV6, + // VariantArmV7, + // VariantArmV7Musl, VariantArm64, VariantArm64Musl, VariantDarwinAmd64, @@ -48,7 +48,8 @@ var Versions = VersionMap{ Architectures: []Architecture{ ArchAMD64, ArchARM64, - ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable. + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable. }, Distribution: []Distribution{ Alpine, @@ -64,9 +65,10 @@ var Versions = VersionMap{ }, DownstreamMode: { Variants: []Variant{ - VariantArmV6, - VariantArmV7, - VariantArmV7Musl, + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // VariantArmV6, + //VariantArmV7, + // VariantArmV7Musl, VariantArm64, VariantArm64Musl, VariantDarwinAmd64, @@ -83,7 +85,8 @@ var Versions = VersionMap{ Architectures: []Architecture{ ArchAMD64, ArchARM64, - ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable. + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable. }, Distribution: []Distribution{ Alpine, @@ -171,9 +174,10 @@ var Versions = VersionMap{ }, Enterprise2Mode: { Variants: []Variant{ - VariantArmV6, - VariantArmV7, - VariantArmV7Musl, + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // VariantArmV6, + // VariantArmV7, + // VariantArmV7Musl, VariantArm64, VariantArm64Musl, VariantDarwinAmd64, @@ -190,7 +194,8 @@ var Versions = VersionMap{ Architectures: []Architecture{ ArchAMD64, ArchARM64, - ArchARMv7, + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // ArchARMv7, }, Distribution: []Distribution{ Alpine, diff --git a/pkg/build/docker/build.go b/pkg/build/docker/build.go index 64016257b2a..29b2263e62b 100644 --- a/pkg/build/docker/build.go +++ b/pkg/build/docker/build.go @@ -74,7 +74,7 @@ func BuildImage(version string, arch config.Architecture, grafanaDir string, use tagSuffix := "" if useUbuntu { libc = "" - baseImage = fmt.Sprintf("%subuntu:20.04", baseArch) + baseImage = fmt.Sprintf("%subuntu:22.04", baseArch) tagSuffix = "-ubuntu" } diff --git a/pkg/build/env/fallback_test.go b/pkg/build/env/fallback_test.go index d14e13ee774..8871bb68a72 100644 --- a/pkg/build/env/fallback_test.go +++ b/pkg/build/env/fallback_test.go @@ -15,6 +15,11 @@ const ( flag2 = "flag2" ) +type flagObj struct { + name string + value string +} + func TestRequireListWithEnvFallback(t *testing.T) { var app = cli.NewApp() tests := []struct { @@ -73,7 +78,7 @@ func TestRequireStringWithEnvFallback(t *testing.T) { }{ { testName: "string present in the context", - ctx: cli.NewContext(app, setFlags(t, flag1, flag2, flag.NewFlagSet("test", flag.ContinueOnError)), nil), + ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("test", flag.ContinueOnError), flagObj{name: flag1, value: "a"}), nil), name: flag1, envName: "", expected: "a", @@ -81,7 +86,7 @@ func TestRequireStringWithEnvFallback(t *testing.T) { }, { testName: "string present in env", - ctx: cli.NewContext(app, setFlags(t, "", "", flag.NewFlagSet("test", flag.ContinueOnError)), nil), + ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("test", flag.ContinueOnError)), nil), name: flag1, envName: setEnv(t, flag1, "a"), expected: "a", @@ -89,7 +94,7 @@ func TestRequireStringWithEnvFallback(t *testing.T) { }, { testName: "string absent from both context and env", - ctx: cli.NewContext(app, setFlags(t, "", flag2, flag.NewFlagSet("test", flag.ContinueOnError)), nil), + ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("test", flag.ContinueOnError), flagObj{name: flag2, value: "b"}), nil), name: flag1, envName: "", expected: "", @@ -120,13 +125,12 @@ func applyFlagSet(t *testing.T, aFlag, aValue string) *flag.FlagSet { return set } -func setFlags(t *testing.T, flag1, flag2 string, flagSet *flag.FlagSet) *flag.FlagSet { +func setFlags(t *testing.T, flagSet *flag.FlagSet, flags ...flagObj) *flag.FlagSet { t.Helper() - if flag1 != "" { - flagSet.StringVar(&flag1, "flag1", "a", "") - } - if flag2 != "" { - flagSet.StringVar(&flag2, "flag2", "b", "") + for _, f := range flags { + if f.name != "" { + flagSet.StringVar(&f.name, f.name, f.value, "") + } } return flagSet } @@ -135,9 +139,6 @@ func setEnv(t *testing.T, key, value string) string { t.Helper() os.Clearenv() - err := os.Setenv(key, value) - if err != nil { - require.NoError(t, err) - } + t.Setenv(key, value) return key } diff --git a/pkg/build/frontend/config_test.go b/pkg/build/frontend/config_test.go index f10892e1cef..21be441cc7a 100644 --- a/pkg/build/frontend/config_test.go +++ b/pkg/build/frontend/config_test.go @@ -21,6 +21,11 @@ type packageJson struct { Version string `json:"version"` } +type flagObj struct { + name string + value string +} + var app = cli.NewApp() func TestGetConfig(t *testing.T) { @@ -32,35 +37,35 @@ func TestGetConfig(t *testing.T) { wantErr bool }{ { - ctx: cli.NewContext(app, setFlags(t, jobs, githubToken, "", flag.NewFlagSet("flagSet", flag.ContinueOnError)), nil), + ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil), name: "package.json matches tag", packageJsonVersion: "10.0.0", metadata: config.Metadata{GrafanaVersion: "10.0.0", ReleaseMode: config.ReleaseMode{Mode: config.TagMode}}, wantErr: false, }, { - ctx: cli.NewContext(app, setFlags(t, jobs, githubToken, "", flag.NewFlagSet("flagSet", flag.ContinueOnError)), nil), + ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil), name: "custom tag, package.json doesn't match", packageJsonVersion: "10.0.0", metadata: config.Metadata{GrafanaVersion: "10.0.0-abcd123pre", ReleaseMode: config.ReleaseMode{Mode: config.TagMode}}, wantErr: false, }, { - ctx: cli.NewContext(app, setFlags(t, jobs, githubToken, "", flag.NewFlagSet("flagSet", flag.ContinueOnError)), nil), + ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil), name: "package.json doesn't match tag", packageJsonVersion: "10.1.0", metadata: config.Metadata{GrafanaVersion: "10.0.0", ReleaseMode: config.ReleaseMode{Mode: config.TagMode}}, wantErr: true, }, { - ctx: cli.NewContext(app, setFlags(t, jobs, githubToken, "", flag.NewFlagSet("flagSet", flag.ContinueOnError)), nil), + ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil), name: "test tag event, check should be skipped", packageJsonVersion: "10.1.0", metadata: config.Metadata{GrafanaVersion: "10.1.0-test", ReleaseMode: config.ReleaseMode{Mode: config.TagMode, IsTest: true}}, wantErr: false, }, { - ctx: cli.NewContext(app, setFlags(t, jobs, githubToken, buildID, flag.NewFlagSet("flagSet", flag.ContinueOnError)), nil), + ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}, flagObj{name: buildID, value: "12345"}), nil), name: "non-tag event", packageJsonVersion: "10.1.0-pre", metadata: config.Metadata{GrafanaVersion: "10.1.0-12345pre", ReleaseMode: config.ReleaseMode{Mode: config.PullRequestMode}}, @@ -85,16 +90,12 @@ func TestGetConfig(t *testing.T) { } } -func setFlags(t *testing.T, flag1, flag2, flag3 string, flagSet *flag.FlagSet) *flag.FlagSet { +func setFlags(t *testing.T, flagSet *flag.FlagSet, flags ...flagObj) *flag.FlagSet { t.Helper() - if flag1 != "" { - flagSet.StringVar(&flag1, jobs, "2", "") - } - if flag2 != "" { - flagSet.StringVar(&flag2, githubToken, "token", "") - } - if flag3 != "" { - flagSet.StringVar(&flag3, buildID, "12345", "") + for _, f := range flags { + if f.name != "" { + flagSet.StringVar(&f.name, f.name, f.value, "") + } } return flagSet } diff --git a/pkg/build/gcloud/storage/gsutil.go b/pkg/build/gcloud/storage/gsutil.go index 066f9d86dcd..711728f4fb0 100644 --- a/pkg/build/gcloud/storage/gsutil.go +++ b/pkg/build/gcloud/storage/gsutil.go @@ -389,11 +389,17 @@ func GetLatestMainBuild(ctx context.Context, bucket *storage.BucketHandle, path return "", ErrorNilBucket } - it := bucket.Objects(ctx, &storage.Query{ + query := &storage.Query{ Prefix: path, - }) + } + err := query.SetAttrSelection([]string{"Name", "Generation"}) + if err != nil { + return "", fmt.Errorf("failed to set attribute selector, err: %q", err) + } + it := bucket.Objects(ctx, query) var files []string + var oldGeneration int64 for { attrs, err := it.Next() if errors.Is(err, iterator.Done) { @@ -402,13 +408,17 @@ func GetLatestMainBuild(ctx context.Context, bucket *storage.BucketHandle, path if err != nil { return "", fmt.Errorf("failed to iterate through bucket, err: %w", err) } - - files = append(files, attrs.Name) + if attrs.Generation >= oldGeneration { + files = append([]string{attrs.Name}, files...) + oldGeneration = attrs.Generation + } else { + files = append(files, attrs.Name) + } } var latestVersion string - for i := len(files) - 1; i >= 0; i-- { - captureVersion := regexp.MustCompile(`(\d+\.\d+\.\d+-\d+pre)`) + for i := 0; i < len(files); i++ { + captureVersion := regexp.MustCompile(`(\d+\.\d+\.\d+-\d+)`) if captureVersion.MatchString(files[i]) { latestVersion = captureVersion.FindString(files[i]) break diff --git a/pkg/build/git.go b/pkg/build/git.go index 6c86d74e004..f955d5b07c1 100644 --- a/pkg/build/git.go +++ b/pkg/build/git.go @@ -15,3 +15,16 @@ func getGitSha() string { } return string(v) } + +func getGitEnterpriseSha() string { + // supporting the old way of dev setup + v, err := runError("git", "-C", "../grafana-enterprise", "rev-parse", "--short", "HEAD") + if err != nil { + // supporting the new way of dev setup + v, err = runError("git", "-C", "..", "rev-parse", "--short", "HEAD") + if err != nil { + return "" + } + } + return string(v) +} diff --git a/pkg/build/grafana/build.go b/pkg/build/grafana/build.go index 0596d094875..586b2d22839 100644 --- a/pkg/build/grafana/build.go +++ b/pkg/build/grafana/build.go @@ -23,13 +23,19 @@ const ( ) func GrafanaLDFlags(version string, r config.Revision) []string { - return []string{ + cmd := []string{ "-w", fmt.Sprintf("-X main.version=%s", version), fmt.Sprintf("-X main.commit=%s", r.SHA256), fmt.Sprintf("-X main.buildstamp=%d", r.Timestamp), fmt.Sprintf("-X main.buildBranch=%s", r.Branch), } + + if r.EnterpriseCommit != "" { + cmd = append(cmd, fmt.Sprintf("-X main.enterpriseCommit=%s", r.EnterpriseCommit)) + } + + return cmd } // BinaryFolder returns the path to where the Grafana binary is build given the provided arguments. diff --git a/pkg/build/lerna/lerna.go b/pkg/build/lerna/lerna.go index de04f5f0e88..3bd67a87b44 100644 --- a/pkg/build/lerna/lerna.go +++ b/pkg/build/lerna/lerna.go @@ -47,7 +47,7 @@ func GetLernaVersion(grafanaDir string) (string, error) { if err != nil { return "", fmt.Errorf("failed to read %q: %w", lernaJSONPath, err) } - pkgObj := map[string]interface{}{} + pkgObj := map[string]any{} if err := json.Unmarshal(lernaJSONB, &pkgObj); err != nil { return "", fmt.Errorf("failed decoding %q: %w", lernaJSONPath, err) } diff --git a/pkg/build/npm/npm.go b/pkg/build/npm/npm.go index c2b93aef46b..5b1a895ec49 100644 --- a/pkg/build/npm/npm.go +++ b/pkg/build/npm/npm.go @@ -21,11 +21,11 @@ const NpmArtifactDir = "./npm-artifacts" var packages = []string{ "@grafana/ui", "@grafana/data", - "@grafana/toolkit", "@grafana/runtime", "@grafana/e2e", "@grafana/e2e-selectors", "@grafana/schema", + "@grafana/flamegraph", } // PublishNpmPackages will publish local NPM packages to NPM registry. @@ -117,11 +117,11 @@ func FetchNpmPackages(ctx context.Context, tag, bucketName string) error { // Latest and next is 9.1.6. // 9.2.0-beta1 is released, the latest should stay on 9.1.6, next should point to 9.2.0-beta1 // No move of dist-tags -// 9.1.7 is relased, the latest should point to 9.1.7, next should stay to 9.2.0-beta1 +// 9.1.7 is released, the latest should point to 9.1.7, next should stay to 9.2.0-beta1 // No move of dist-tags // Next week 9.2.0-beta2 is released, the latest should point to 9.1.7, next should point to 9.2.0-beta2 // No move of dist-tags -// In two weeks 9.2.0 stable is relased, the latest and next should point to 9.2.0. +// In two weeks 9.2.0 stable is released, the latest and next should point to 9.2.0. // The next dist-tag is moved to point to 9.2.0. // // 3. Releasing an older stable than the current stable diff --git a/pkg/build/packaging/grafana.go b/pkg/build/packaging/grafana.go index 88fef5c6793..58f4dbf7209 100644 --- a/pkg/build/packaging/grafana.go +++ b/pkg/build/packaging/grafana.go @@ -533,7 +533,7 @@ func copyPlugins(ctx context.Context, v config.Variant, grafanaDir, tmpDir strin if err != nil { return fmt.Errorf("failed to read %q: %w", filepath.Join(srcDir, "plugin.json"), err) } - var plugJSON map[string]interface{} + var plugJSON map[string]any if err := json.Unmarshal(jsonB, &plugJSON); err != nil { return err } diff --git a/scripts/drone/TAGS b/scripts/drone/TAGS deleted file mode 100644 index 1c9f9da89a5..00000000000 --- a/scripts/drone/TAGS +++ /dev/null @@ -1,628 +0,0 @@ - -events/release.star,6652 -ver_mode =ver_mode64,1602 -release_trigger =release_trigger65,1623 -def store_npm_packages_step():store_npm_packages_step74,1752 -def retrieve_npm_packages_step():retrieve_npm_packages_step90,2193 -def release_npm_packages_step():release_npm_packages_step107,2663 -def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):oss_pipelines123,3076 - environment =environment135,3492 - edition =edition136,3529 - services =services137,3549 - volumes =volumes138,3609 - package_steps =package_steps139,3659 - publish_steps =publish_steps140,3682 - should_publish =should_publish141,3705 - should_upload =should_upload142,3748 - init_steps =init_steps143,3818 - build_steps =build_steps152,4033 - integration_test_steps =integration_test_steps159,4342 - build_storybook =build_storybook182,5254 - publish_step =publish_step190,5674 - store_npm_step =store_npm_step191,5758 - windows_package_steps =windows_package_steps196,5957 - windows_pipeline =windows_pipeline198,6044 - name =name199,6077 - edition =edition200,6127 - trigger =trigger201,6154 - steps =steps202,6181 - platform =platform203,6256 - depends_on =depends_on204,6286 - environment =environment207,6393 - pipelines =pipelines209,6434 - name =name211,6470 - edition =edition212,6550 - trigger =trigger213,6581 - services =services214,6612 - steps =steps215,6639 - environment =environment216,6717 - volumes =volumes217,6756 - name =name225,6970 - edition =edition226,7038 - trigger =trigger227,7073 - services =services228,7108 - steps =steps229,7145 - environment =environment230,7329 - volumes =volumes231,7372 - deps =deps234,7433 -def enterprise_pipelines(ver_mode = ver_mode, trigger = release_trigger):enterprise_pipelines247,7856 - environment =environment259,8284 - edition =edition260,8328 - services =services261,8355 - volumes =volumes262,8415 - package_steps =package_steps263,8465 - publish_steps =publish_steps264,8488 - should_publish =should_publish265,8511 - should_upload =should_upload266,8554 - include_enterprise =include_enterprise267,8624 - edition2 =edition2268,8673 - init_steps =init_steps269,8702 - build_steps =build_steps277,8909 - integration_test_steps =integration_test_steps284,9218 - build_storybook =build_storybook312,10299 - publish_step =publish_step324,10892 - store_npm_step =store_npm_step325,10976 - windows_package_steps =windows_package_steps330,11175 - step =step333,11284 - deps_on_clone_enterprise_step =deps_on_clone_enterprise_step337,11418 - windows_pipeline =windows_pipeline347,11746 - name =name348,11779 - edition =edition349,11836 - trigger =trigger350,11863 - steps =steps351,11890 - platform =platform352,11965 - depends_on =depends_on353,11995 - environment =environment356,12109 - pipelines =pipelines358,12150 - name =name360,12186 - edition =edition361,12273 - trigger =trigger362,12304 - services =services363,12335 - steps =steps364,12362 - environment =environment365,12440 - volumes =volumes366,12479 - name =name374,12711 - edition =edition375,12786 - trigger =trigger376,12821 - services =services377,12856 - steps =steps378,12893 - environment =environment379,13213 - volumes =volumes380,13256 - deps =deps383,13317 -def enterprise2_pipelines(prefix = "", ver_mode = ver_mode, trigger = release_trigger):enterprise2_pipelines397,13769 - environment =environment412,14364 - edition =edition415,14424 - volumes =volumes416,14451 - package_steps =package_steps417,14501 - publish_steps =publish_steps418,14524 - should_publish =should_publish419,14547 - should_upload =should_upload420,14590 - include_enterprise =include_enterprise421,14660 - edition2 =edition2422,14709 - init_steps =init_steps423,14738 - build_steps =build_steps431,14945 - fetch_images =fetch_images442,15355 - upload_cdn =upload_cdn444,15497 - step =step458,16187 - deps_on_clone_enterprise_step =deps_on_clone_enterprise_step462,16321 - pipelines =pipelines472,16608 - name =name474,16644 - edition =edition475,16742 - trigger =trigger476,16773 - services =services477,16804 - steps =steps478,16831 - volumes =volumes479,16909 - environment =environment480,16940 -def publish_artifacts_step(mode):publish_artifacts_step486,17019 - security =security487,17053 - security =security489,17098 -def publish_artifacts_pipelines(mode):publish_artifacts_pipelines501,17538 - trigger =trigger502,17577 - steps =steps506,17655 - name =name512,17768 - trigger =trigger513,17820 - steps =steps514,17847 - edition =edition515,17870 - environment =environment516,17895 -def publish_packages_pipeline():publish_packages_pipeline519,17945 - trigger =trigger526,18162 - oss_steps =oss_steps530,18244 - enterprise_steps =enterprise_steps538,18560 - deps =deps545,18903 - name =name552,19062 - trigger =trigger553,19101 - steps =steps554,19128 - edition =edition555,19155 - depends_on =depends_on556,19180 - environment =environment557,19207 - name =name559,19266 - trigger =trigger560,19312 - steps =steps561,19339 - edition =edition562,19373 - depends_on =depends_on563,19398 - environment =environment564,19425 -def publish_npm_pipelines(mode):publish_npm_pipelines567,19482 - trigger =trigger568,19515 - steps =steps572,19593 - name =name580,19772 - trigger =trigger581,19827 - steps =steps582,19854 - edition =edition583,19877 - environment =environment584,19902 -def artifacts_page_pipeline():artifacts_page_pipeline587,19952 - trigger =trigger588,19983 - name =name593,20087 - trigger =trigger594,20128 - steps =steps595,20155 - edition =edition596,20220 - environment =environment597,20245 -def get_e2e_suffix():get_e2e_suffix600,20295 - -events/cron.star,1016 -aquasec_trivy_image =aquasec_trivy_image8,209 -def cronjobs(edition):cronjobs10,255 - grafana_com_nightly_pipeline =grafana_com_nightly_pipeline11,278 - cronName =cronName12,332 - name =name13,374 - steps =steps14,412 -def cron_job_pipeline(cronName, name, steps):cron_job_pipeline24,773 -def scan_docker_image_pipeline(edition, tag):scan_docker_image_pipeline43,1175 - edition =edition55,1530 - edition =edition57,1579 - docker_image =docker_image59,1608 - cronName =cronName62,1695 - name =name63,1725 - steps =steps64,1775 -def scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image):scan_docker_image_unkown_low_medium_vulnerabilities_step71,2047 -def scan_docker_image_high_critical_vulnerabilities_step(docker_image):scan_docker_image_high_critical_vulnerabilities_step80,2353 -def slack_job_failed_step(channel, image):slack_job_failed_step89,2646 -def post_to_grafana_com_step():post_to_grafana_com_step103,3069 - -events/main.star,633 -ver_mode =ver_mode49,966 -trigger =trigger50,984 -def main_pipelines(edition):main_pipelines62,1168 - drone_change_trigger =drone_change_trigger63,1197 - pipelines =pipelines79,1513 - name =name89,1951 - slack_channel =slack_channel90,1994 - trigger =trigger91,2045 - template =template92,2089 - secret =secret93,2135 - name =name97,2276 - slack_channel =slack_channel98,2310 - trigger =trigger99,2366 - depends_on =depends_on100,2425 - template =template101,2563 - secret =secret102,2604 - -events/pr.star,252 -ver_mode =ver_mode48,997 -trigger =trigger49,1013 -def pr_pipelines(edition):pr_pipelines62,1198 -def get_pr_trigger(include_paths = None, exclude_paths = None):get_pr_trigger76,2396 - paths_ex =paths_ex91,3080 - paths_in =paths_in92,3115 - -services/services.star,225 -def integration_test_services_volumes():integration_test_services_volumes5,79 -def integration_test_services(edition):integration_test_services14,292 - services =services15,332 -def ldap_service():ldap_service59,1616 - -utils/utils.star,561 -failure_template =failure_template11,191 -drone_change_template =drone_change_template12,509 - services =services19,932 - platform =platform20,955 - depends_on =depends_on21,983 - environment =environment22,1008 - volumes =volumes23,1036 - platform_conf =platform_conf50,2166 - platform_conf =platform_conf62,2534 - pipeline =pipeline70,2713 -def notify_pipeline(name, slack_channel, trigger, depends_on = [], template = None, secret = None):notify_pipeline105,3545 - trigger =trigger106,3645 - -pipelines/trigger_downstream.star,440 -trigger =trigger14,249 -def enterprise_downstream_pipeline(edition, ver_mode):enterprise_downstream_pipeline26,433 - environment =environment27,488 - steps =steps28,527 - deps =deps29,587 - name =name31,672 - edition =edition32,714 - trigger =trigger33,741 - services =services34,768 - steps =steps35,791 - depends_on =depends_on36,814 - environment =environment37,841 - -pipelines/verify_starlark.star,323 -def verify_starlark(trigger, ver_mode):verify_starlark17,305 - environment =environment18,345 - steps =steps19,382 - name =name26,546 - edition =edition27,600 - trigger =trigger28,625 - services =services29,652 - steps =steps30,675 - environment =environment31,698 - -pipelines/build.star,508 -def build_e2e(trigger, ver_mode, edition):build_e2e39,936 - environment =environment50,1096 - variants =variants51,1135 - init_steps =init_steps52,1219 - build_steps =build_steps61,1491 - publish_suffix =publish_suffix107,4049 - publish_suffix =publish_suffix109,4100 - name =name112,4158 - edition =edition113,4224 - environment =environment114,4249 - services =services115,4284 - steps =steps116,4307 - trigger =trigger117,4349 - -pipelines/shellcheck.star,386 -trigger =trigger15,235 -def shellcheck_step():shellcheck_step31,483 -def shellcheck_pipeline():shellcheck_pipeline43,725 - environment =environment44,752 - steps =steps45,789 - name =name50,886 - edition =edition51,918 - trigger =trigger52,943 - services =services53,970 - steps =steps54,993 - environment =environment55,1016 - -pipelines/verify_drone.star,317 -def verify_drone(trigger, ver_mode):verify_drone17,293 - environment =environment18,330 - steps =steps19,367 - name =name26,528 - edition =edition27,579 - trigger =trigger28,604 - services =services29,631 - steps =steps30,654 - environment =environment31,677 - -pipelines/test_backend.star,474 -def test_backend(trigger, ver_mode, edition = "oss"):test_backend23,463 - environment =environment35,882 - init_steps =init_steps36,921 - test_steps =test_steps46,1291 - pipeline_name =pipeline_name51,1387 - pipeline_name =pipeline_name53,1492 - name =name55,1584 - edition =edition56,1614 - trigger =trigger57,1641 - services =services58,1668 - steps =steps59,1691 - environment =environment60,1732 - -pipelines/lint_frontend.star,415 -def lint_frontend_pipeline(trigger, ver_mode):lint_frontend_pipeline16,260 - environment =environment26,546 - yarn_step =yarn_step27,583 - init_steps =init_steps29,660 - test_steps =test_steps33,736 - name =name37,812 - edition =edition38,864 - trigger =trigger39,889 - services =services40,916 - steps =steps41,939 - environment =environment42,980 - -pipelines/docs.star,494 -docs_paths =docs_paths19,383 -def docs_pipelines(edition, ver_mode, trigger):docs_pipelines28,511 - environment =environment29,559 - steps =steps30,598 - name =name40,815 - edition =edition41,858 - trigger =trigger42,885 - services =services43,912 - steps =steps44,935 - environment =environment45,958 -def lint_docs():lint_docs48,1000 -def trigger_docs_main():trigger_docs_main63,1328 -def trigger_docs_pr():trigger_docs_pr72,1478 - -pipelines/test_frontend.star,476 -def test_frontend(trigger, ver_mode, edition = "oss"):test_frontend20,374 - environment =environment32,794 - init_steps =init_steps33,833 - test_steps =test_steps41,1102 - pipeline_name =pipeline_name45,1205 - pipeline_name =pipeline_name47,1311 - name =name49,1404 - edition =edition50,1434 - trigger =trigger51,1461 - services =services52,1488 - steps =steps53,1511 - environment =environment54,1552 - -pipelines/integration_tests.star,483 -def integration_tests(trigger, ver_mode, edition):integration_tests26,542 - environment =environment37,900 - services =services38,939 - volumes =volumes39,989 - init_steps =init_steps40,1039 - test_steps =test_steps48,1282 - name =name54,1412 - edition =edition55,1468 - trigger =trigger56,1493 - services =services57,1520 - steps =steps58,1549 - environment =environment59,1590 - volumes =volumes60,1625 - -pipelines/windows.star,954 -def windows(trigger, edition, ver_mode):windows17,339 - environment =environment29,798 - init_cmds =init_cmds30,837 - steps =steps38,1205 - bucket =bucket49,1497 - ver_part =ver_part51,1590 - dir =dir52,1628 - dir =dir54,1670 - bucket =bucket55,1695 - build_no =build_no56,1736 - ver_part =ver_part57,1780 - installer_commands =installer_commands58,1842 - committish =committish100,3763 - committish =committish102,3846 - committish =committish104,3906 - download_grabpl_step_cmds =download_grabpl_step_cmds107,4057 - clone_cmds =clone_cmds113,4363 - name =name146,5711 - edition =edition147,5742 - trigger =trigger148,5769 - steps =steps149,5830 - depends_on =depends_on150,5889 - platform =platform151,6007 - environment =environment152,6037 - -pipelines/lint_backend.star,418 -def lint_backend_pipeline(trigger, ver_mode):lint_backend_pipeline18,306 - environment =environment28,590 - wire_step =wire_step29,627 - init_steps =init_steps31,704 - test_steps =test_steps36,809 - name =name43,959 - edition =edition44,1010 - trigger =trigger45,1035 - services =services46,1062 - steps =steps47,1085 - environment =environment48,1126 - -pipelines/publish_images.star,998 -def publish_image_steps(edition, mode, docker_repo):publish_image_steps17,303 - additional_docker_repo =additional_docker_repo31,922 - additional_docker_repo =additional_docker_repo33,979 - steps =steps34,1034 -def publish_image_pipelines_public():publish_image_pipelines_public45,1369 - mode =mode51,1521 - trigger =trigger52,1541 - name =name57,1641 - trigger =trigger58,1694 - steps =steps59,1721 - edition =edition60,1813 - environment =environment61,1835 - name =name63,1894 - trigger =trigger64,1954 - steps =steps65,1981 - edition =edition66,2091 - environment =environment67,2113 -def publish_image_pipelines_security():publish_image_pipelines_security70,2170 - mode =mode71,2210 - trigger =trigger72,2232 - name =name77,2332 - trigger =trigger78,2392 - steps =steps79,2419 - edition =edition80,2529 - environment =environment81,2551 - -steps/lib.star,8579 -grabpl_version =grabpl_version7,181 -build_image =build_image8,208 -publish_image =publish_image9,254 -deploy_docker_image =deploy_docker_image10,304 -alpine_image =alpine_image11,380 -curl_image =curl_image12,411 -windows_image =windows_image13,452 -wix_image =wix_image14,501 -go_image =go_image15,536 -disable_tests =disable_tests17,564 -trigger_oss =trigger_oss18,586 -def slack_step(channel, template, secret):slack_step24,653 -def yarn_install_step(edition = "oss"):yarn_install_step35,918 - deps =deps36,958 - deps =deps38,1004 -def wire_install_step():wire_install_step48,1222 -def identify_runner_step(platform = "linux"):identify_runner_step60,1454 -def clone_enterprise_step(ver_mode):clone_enterprise_step78,1916 - committish =committish87,2193 - committish =committish89,2268 - committish =committish91,2317 -def init_enterprise_step(ver_mode):init_enterprise_step105,2747 - source_commit =source_commit115,3098 - source_commit =source_commit117,3151 - environment =environment118,3191 - token =token121,3280 - environment =environment123,3369 - token =token126,3458 - environment =environment128,3518 - token =token129,3543 -def download_grabpl_step(platform = "linux"):download_grabpl_step148,4147 -def lint_drone_step():lint_drone_step173,4973 -def lint_starlark_step():lint_starlark_step185,5216 -def enterprise_downstream_step(edition, ver_mode):enterprise_downstream_step206,6000 - repo =repo219,6482 - step =step225,6623 -def lint_backend_step():lint_backend_step247,7248 -def benchmark_ldap_step():benchmark_ldap_step265,7713 -def build_storybook_step(edition, ver_mode):build_storybook_step278,8087 -def store_storybook_step(edition, ver_mode, trigger = None):store_storybook_step300,8743 - commands =commands314,9202 - commands =commands323,9521 - step =step325,9593 - when_cond =when_cond338,10125 - step =step346,10330 -def e2e_tests_artifacts(edition):e2e_tests_artifacts349,10391 -def upload_cdn_step(edition, ver_mode, trigger = None):upload_cdn_step386,12378 - deps =deps397,12763 - step =step407,12970 - step =step420,13423 -def build_backend_step(edition, ver_mode, variants = None):build_backend_step423,13482 - variants_str =variants_str437,14070 - variants_str =variants_str439,14109 - cmds =cmds443,14256 - build_no =build_no449,14418 - cmds =cmds450,14461 -def build_frontend_step(edition, ver_mode):build_frontend_step468,14906 - build_no =build_no478,15246 - cmds =cmds482,15356 - cmds =cmds487,15505 -def build_frontend_package_step(edition, ver_mode):build_frontend_package_step505,15960 - build_no =build_no515,16312 - cmds =cmds519,16422 - cmds =cmds524,16580 -def build_plugins_step(edition, ver_mode):build_plugins_step542,17053 - env =env544,17121 - env =env548,17220 -def test_backend_step():test_backend_step563,17607 -def test_backend_integration_step():test_backend_integration_step575,17880 -def betterer_frontend_step(edition = "oss"):betterer_frontend_step587,18187 - deps =deps596,18427 -def test_frontend_step(edition = "oss"):test_frontend_step609,18728 - deps =deps618,18962 -def lint_frontend_step():lint_frontend_step634,19343 -def test_a11y_frontend_step(ver_mode, edition, port = 3001):test_a11y_frontend_step652,19793 - commands =commands664,20279 - failure =failure667,20345 - failure =failure672,20483 -def frontend_metrics_step(edition, trigger = None):frontend_metrics_step693,21146 - step =step706,21507 - step =step721,22007 -def codespell_step():codespell_step724,22066 -def package_step(edition, ver_mode, variants = None):package_step736,22468 - deps =deps750,23006 - variants_str =variants_str757,23167 - variants_str =variants_str759,23206 - sign_args =sign_args762,23332 - env =env763,23362 - test_args =test_args769,23628 - sign_args =sign_args771,23661 - env =env772,23684 - test_args =test_args773,23703 - cmds =cmds777,23829 - build_no =build_no784,24036 - cmds =cmds785,24079 -def grafana_server_step(edition, port = 3001):grafana_server_step798,24459 - package_file_pfx =package_file_pfx808,24729 - package_file_pfx =package_file_pfx810,24788 - package_file_pfx =package_file_pfx812,24889 - environment =environment814,24938 -def e2e_tests_step(suite, edition, port = 3001, tries = None):e2e_tests_step837,25554 - cmd =cmd838,25617 -def cloud_plugins_e2e_tests_step(suite, edition, cloud, trigger = None):cloud_plugins_e2e_tests_step856,26186 - environment =environment869,26649 - when =when870,26670 - when =when872,26700 - environment =environment874,26748 - when =when882,27129 - branch =branch888,27345 - step =step889,27401 - step =step901,27822 -def build_docs_website_step():build_docs_website_step904,27874 -def copy_packages_for_docker_step(edition = None):copy_packages_for_docker_step916,28272 -def build_docker_images_step(edition, archs = None, ubuntu = False, publish = False):build_docker_images_step929,28622 - cmd =cmd943,29193 - ubuntu_sfx =ubuntu_sfx947,29307 - ubuntu_sfx =ubuntu_sfx949,29342 - environment =environment955,29468 -def fetch_images_step(edition):fetch_images_step979,30079 -def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None):publish_images_step997,30745 - name =name1013,31562 - docker_repo =docker_repo1014,31585 - mode =mode1016,31663 - mode =mode1018,31709 - environment =environment1020,31728 - cmd =cmd1026,31912 - deps =deps1029,32041 - deps =deps1032,32147 - name =name1035,32250 - docker_repo =docker_repo1036,32273 - cmd =cmd1038,32459 - step =step1040,32565 - step =step1052,32929 -def postgres_integration_tests_step():postgres_integration_tests_step1056,32989 - cmds =cmds1057,33028 -def mysql_integration_tests_step():mysql_integration_tests_step1079,33850 - cmds =cmds1080,33886 -def redis_integration_tests_step():redis_integration_tests_step1100,34629 -def memcached_integration_tests_step():memcached_integration_tests_step1114,35026 -def release_canary_npm_packages_step(edition, trigger = None):release_canary_npm_packages_step1128,35435 - step =step1141,35805 - step =step1153,36143 -def enterprise2_suffix(edition):enterprise2_suffix1156,36202 -def upload_packages_step(edition, ver_mode, trigger = None):upload_packages_step1161,36320 - deps =deps1176,36816 - step =step1184,37036 - step =step1195,37471 -def publish_grafanacom_step(edition, ver_mode):publish_grafanacom_step1198,37530 - cmd =cmd1211,38044 - build_no =build_no1215,38188 - cmd =cmd1216,38231 -def publish_linux_packages_step(edition, package_manager = "deb"):publish_linux_packages_step1239,38866 -def get_windows_steps(edition, ver_mode):get_windows_steps1261,39989 - init_cmds =init_cmds1270,40281 - steps =steps1278,40649 - bucket =bucket1289,40941 - ver_part =ver_part1291,41034 - dir =dir1292,41072 - dir =dir1294,41114 - bucket =bucket1295,41139 - build_no =build_no1296,41180 - ver_part =ver_part1297,41224 - installer_commands =installer_commands1298,41286 - committish =committish1340,43207 - committish =committish1342,43290 - committish =committish1344,43350 - download_grabpl_step_cmds =download_grabpl_step_cmds1347,43501 - clone_cmds =clone_cmds1353,43807 -def verify_gen_cue_step(edition):verify_gen_cue_step1387,45152 - deps =deps1388,45186 -def verify_gen_jsonnet_step(edition):verify_gen_jsonnet_step1402,45694 - deps =deps1403,45732 -def trigger_test_release():trigger_test_release1417,46236 -def artifacts_page_step():artifacts_page_step1451,47731 -def end_to_end_tests_deps():end_to_end_tests_deps1466,48058 -def compile_build_cmd(edition = "oss"):compile_build_cmd1476,48321 - dependencies =dependencies1477,48361 - dependencies =dependencies1479,48432 -def get_trigger_storybook(ver_mode):get_trigger_storybook1492,48780 - trigger_storybook =trigger_storybook1500,49031 - trigger_storybook =trigger_storybook1502,49088 - trigger_storybook =trigger_storybook1506,49168 - -vault.star,444 -pull_secret =pull_secret4,87 -github_token =github_token5,120 -drone_token =drone_token6,150 -prerelease_bucket =prerelease_bucket7,178 -gcp_upload_artifacts_key =gcp_upload_artifacts_key8,218 -azure_sp_app_id =azure_sp_app_id9,272 -azure_sp_app_pw =azure_sp_app_pw10,308 -azure_tenant =azure_tenant11,344 -def from_secret(secret):from_secret13,375 -def vault_secret(name, path, key):vault_secret18,451 -def secrets():secrets28,633 - -version.star,116 -ver_mode =ver_mode12,197 -trigger =trigger13,225 -def version_branch_pipelines():version_branch_pipelines15,268 diff --git a/scripts/drone/events/cron.star b/scripts/drone/events/cron.star index 6715c175b17..6f9e524139b 100644 --- a/scripts/drone/events/cron.star +++ b/scripts/drone/events/cron.star @@ -2,15 +2,11 @@ This module provides functions for cronjob pipelines and steps used within. """ -load("scripts/drone/vault.star", "from_secret") -load( - "scripts/drone/steps/lib.star", - "compile_build_cmd", -) load( "scripts/drone/utils/images.star", "images", ) +load("scripts/drone/vault.star", "from_secret") aquasec_trivy_image = "aquasec/trivy:0.21.0" @@ -21,7 +17,6 @@ def cronjobs(): scan_docker_image_pipeline("latest-ubuntu"), scan_docker_image_pipeline("main-ubuntu"), scan_build_test_publish_docker_image_pipeline(), - grafana_com_nightly_pipeline(), ] def authenticate_gcr_step(): @@ -59,6 +54,10 @@ def cron_job_pipeline(cronName, name, steps): "path": "/var/run/docker.sock", }, }, + { + "name": "config", + "temp": {}, + }, ], } @@ -117,13 +116,13 @@ def scan_docker_image_unknown_low_medium_vulnerabilities_step(docker_image): for key in images: cmds = cmds + ["trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + images[key]] else: - cmds = ["trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image] + cmds = ["trivy image --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image] return { "name": "scan-unknown-low-medium-vulnerabilities", "image": aquasec_trivy_image, "commands": cmds, "depends_on": ["authenticate-gcr"], - "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}, {"name": "config", "path": "/root/.docker/"}], } def scan_docker_image_high_critical_vulnerabilities_step(docker_image): @@ -141,19 +140,22 @@ def scan_docker_image_high_critical_vulnerabilities_step(docker_image): for key in images: cmds = cmds + ["trivy --exit-code 1 --severity HIGH,CRITICAL " + images[key]] else: - cmds = ["trivy --exit-code 1 --severity HIGH,CRITICAL " + docker_image] + cmds = ["trivy image --exit-code 1 --severity HIGH,CRITICAL " + docker_image] return { "name": "scan-high-critical-vulnerabilities", "image": aquasec_trivy_image, "commands": cmds, "depends_on": ["authenticate-gcr"], - "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + "environment": { + "GOOGLE_APPLICATION_CREDENTIALS": from_secret("gcr_credentials_json"), + }, + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}, {"name": "config", "path": "/root/.docker/"}], } def slack_job_failed_step(channel, image): return { "name": "slack-notify-failure", - "image": images["plugins_slack_image"], + "image": images["plugins_slack"], "settings": { "webhook": from_secret("slack_webhook_backend"), "channel": channel, @@ -167,7 +169,7 @@ def slack_job_failed_step(channel, image): def post_to_grafana_com_step(): return { "name": "post-to-grafana-com", - "image": images["publish_image"], + "image": images["publish"], "environment": { "GRAFANA_COM_API_KEY": from_secret("grafana_api_key"), "GCP_KEY": from_secret("gcp_key"), @@ -175,13 +177,3 @@ def post_to_grafana_com_step(): "depends_on": ["compile-build-cmd"], "commands": ["./bin/build publish grafana-com --edition oss"], } - -def grafana_com_nightly_pipeline(): - return cron_job_pipeline( - cronName = "grafana-com-nightly", - name = "grafana-com-nightly", - steps = [ - compile_build_cmd(), - post_to_grafana_com_step(), - ], - ) diff --git a/scripts/drone/events/main.star b/scripts/drone/events/main.star index 7c592c53ae8..ee47392ee5c 100644 --- a/scripts/drone/events/main.star +++ b/scripts/drone/events/main.star @@ -3,40 +3,18 @@ This module returns all the pipelines used in the event of pushes to the main br """ load( - "scripts/drone/utils/utils.star", - "drone_change_template", - "failure_template", - "notify_pipeline", + "scripts/drone/pipelines/build.star", + "build_e2e", ) load( "scripts/drone/pipelines/docs.star", "docs_pipelines", "trigger_docs_main", ) -load( - "scripts/drone/pipelines/test_frontend.star", - "test_frontend", -) -load( - "scripts/drone/pipelines/test_backend.star", - "test_backend", -) load( "scripts/drone/pipelines/integration_tests.star", "integration_tests", ) -load( - "scripts/drone/pipelines/build.star", - "build_e2e", -) -load( - "scripts/drone/pipelines/windows.star", - "windows", -) -load( - "scripts/drone/pipelines/trigger_downstream.star", - "enterprise_downstream_pipeline", -) load( "scripts/drone/pipelines/lint_backend.star", "lint_backend_pipeline", @@ -45,6 +23,27 @@ load( "scripts/drone/pipelines/lint_frontend.star", "lint_frontend_pipeline", ) +load( + "scripts/drone/pipelines/test_backend.star", + "test_backend", +) +load( + "scripts/drone/pipelines/test_frontend.star", + "test_frontend", +) +load( + "scripts/drone/pipelines/trigger_downstream.star", + "enterprise_downstream_pipeline", +) +load( + "scripts/drone/pipelines/windows.star", + "windows", +) +load( + "scripts/drone/utils/utils.star", + "failure_template", + "notify_pipeline", +) ver_mode = "main" trigger = { @@ -59,27 +58,12 @@ trigger = { "latest.json", ], }, + "repo": [ + "grafana/grafana", + ], } def main_pipelines(): - drone_change_trigger = { - "event": [ - "push", - ], - "branch": "main", - "repo": [ - "grafana/grafana", - ], - "paths": { - "include": [ - ".drone.yml", - ], - "exclude": [ - "exclude", - ], - }, - } - pipelines = [ docs_pipelines(ver_mode, trigger_docs_main()), test_frontend(trigger, ver_mode), @@ -89,13 +73,6 @@ def main_pipelines(): build_e2e(trigger, ver_mode), integration_tests(trigger, prefix = ver_mode, ver_mode = ver_mode), windows(trigger, ver_mode = ver_mode), - notify_pipeline( - name = "notify-drone-changes", - slack_channel = "slack-webhooks-test", - trigger = drone_change_trigger, - template = drone_change_template, - secret = "drone-changes-webhook", - ), enterprise_downstream_pipeline(), notify_pipeline( name = "main-notify", diff --git a/scripts/drone/events/pr.star b/scripts/drone/events/pr.star index 14605cd7909..cbd425efcf6 100644 --- a/scripts/drone/events/pr.star +++ b/scripts/drone/events/pr.star @@ -4,37 +4,21 @@ It also includes a function generating a PR trigger from a list of included and """ load( - "scripts/drone/pipelines/test_frontend.star", - "test_frontend", -) -load( - "scripts/drone/pipelines/test_backend.star", - "test_backend", -) -load( - "scripts/drone/pipelines/integration_tests.star", - "integration_tests", + "scripts/drone/pipelines/benchmarks.star", + "integration_benchmarks", ) load( "scripts/drone/pipelines/build.star", "build_e2e", ) -load( - "scripts/drone/pipelines/verify_drone.star", - "verify_drone", -) -load( - "scripts/drone/pipelines/verify_starlark.star", - "verify_starlark", -) load( "scripts/drone/pipelines/docs.star", "docs_pipelines", "trigger_docs_pr", ) load( - "scripts/drone/pipelines/shellcheck.star", - "shellcheck_pipeline", + "scripts/drone/pipelines/integration_tests.star", + "integration_tests", ) load( "scripts/drone/pipelines/lint_backend.star", @@ -44,6 +28,26 @@ load( "scripts/drone/pipelines/lint_frontend.star", "lint_frontend_pipeline", ) +load( + "scripts/drone/pipelines/shellcheck.star", + "shellcheck_pipeline", +) +load( + "scripts/drone/pipelines/test_backend.star", + "test_backend", +) +load( + "scripts/drone/pipelines/test_frontend.star", + "test_frontend", +) +load( + "scripts/drone/pipelines/verify_drone.star", + "verify_drone", +) +load( + "scripts/drone/pipelines/verify_starlark.star", + "verify_starlark", +) ver_mode = "pr" trigger = { @@ -133,6 +137,9 @@ def pr_pipelines(): ), docs_pipelines(ver_mode, trigger_docs_pr()), shellcheck_pipeline(), + integration_benchmarks( + prefix = ver_mode, + ), ] def get_pr_trigger(include_paths = None, exclude_paths = None): diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index f93dc30e0b1..5e71f1eed04 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -2,55 +2,36 @@ This module returns all the pipelines used in the event of a release along with supporting functions. """ -load( - "scripts/drone/steps/lib.star", - "build_backend_step", - "build_docker_images_step", - "build_frontend_package_step", - "build_frontend_step", - "build_plugins_step", - "build_storybook_step", - "compile_build_cmd", - "copy_packages_for_docker_step", - "download_grabpl_step", - "e2e_tests_artifacts", - "e2e_tests_step", - "get_windows_steps", - "grafana_server_step", - "identify_runner_step", - "memcached_integration_tests_step", - "mysql_integration_tests_step", - "package_step", - "postgres_integration_tests_step", - "publish_grafanacom_step", - "publish_linux_packages_step", - "redis_integration_tests_step", - "store_storybook_step", - "trigger_oss", - "upload_cdn_step", - "upload_packages_step", - "verify_gen_cue_step", - "verify_gen_jsonnet_step", - "wire_install_step", - "yarn_install_step", -) load( "scripts/drone/services/services.star", "integration_test_services", "integration_test_services_volumes", ) +load( + "scripts/drone/steps/lib.star", + "compile_build_cmd", + "download_grabpl_step", + "identify_runner_step", + "memcached_integration_tests_steps", + "mysql_integration_tests_steps", + "postgres_integration_tests_steps", + "publish_grafanacom_step", + "publish_linux_packages_step", + "redis_integration_tests_steps", + "remote_alertmanager_integration_tests_steps", + "verify_gen_cue_step", + "verify_gen_jsonnet_step", + "wire_install_step", + "yarn_install_step", +) +load( + "scripts/drone/utils/images.star", + "images", +) load( "scripts/drone/utils/utils.star", "pipeline", ) -load( - "scripts/drone/pipelines/test_frontend.star", - "test_frontend", -) -load( - "scripts/drone/pipelines/test_backend.star", - "test_backend", -) load( "scripts/drone/vault.star", "from_secret", @@ -59,51 +40,13 @@ load( "prerelease_bucket", "rgm_gcp_key_base64", ) -load( - "scripts/drone/utils/images.star", - "images", -) -load( - "scripts/drone/pipelines/whats_new_checker.star", - "whats_new_checker_pipeline", -) ver_mode = "release" -release_trigger = { - "event": { - "exclude": [ - "promote", - ], - }, - "ref": { - "include": [ - "refs/tags/v*", - ], - "exclude": [ - "refs/tags/*-cloud*", - ], - }, -} - -def store_npm_packages_step(): - return { - "name": "store-npm-packages", - "image": images["build_image"], - "depends_on": [ - "compile-build-cmd", - "build-frontend-packages", - ], - "environment": { - "GCP_KEY": from_secret(gcp_grafanauploads_base64), - "PRERELEASE_BUCKET": from_secret(prerelease_bucket), - }, - "commands": ["./bin/build artifacts npm store --tag ${DRONE_TAG}"], - } def retrieve_npm_packages_step(): return { "name": "retrieve-npm-packages", - "image": images["publish_image"], + "image": images["publish"], "depends_on": [ "compile-build-cmd", "yarn-install", @@ -119,7 +62,7 @@ def retrieve_npm_packages_step(): def release_npm_packages_step(): return { "name": "release-npm-packages", - "image": images["build_image"], + "image": images["go"], "depends_on": [ "compile-build-cmd", "retrieve-npm-packages", @@ -131,145 +74,10 @@ def release_npm_packages_step(): "commands": ["./bin/build artifacts npm release --tag ${DRONE_TAG}"], } -def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger): - """Generates all pipelines used for Grafana OSS. - - Args: - ver_mode: controls which steps are included in the pipeline. - Defaults to 'release'. - trigger: controls which events can trigger the pipeline execution. - Defaults to tag events for tags with a 'v' prefix. - - Returns: - List of Drone pipelines. - """ - - environment = {"EDITION": "oss"} - - services = integration_test_services() - volumes = integration_test_services_volumes() - - init_steps = [ - identify_runner_step(), - download_grabpl_step(), - verify_gen_cue_step(), - wire_install_step(), - yarn_install_step(), - compile_build_cmd(), - ] - - build_steps = [ - build_backend_step(ver_mode = ver_mode), - build_frontend_step(ver_mode = ver_mode), - build_frontend_package_step(ver_mode = ver_mode), - build_plugins_step(ver_mode = ver_mode), - package_step(ver_mode = ver_mode), - copy_packages_for_docker_step(), - build_docker_images_step(publish = True), - build_docker_images_step( - publish = True, - ubuntu = True, - ), - grafana_server_step(), - e2e_tests_step("dashboards-suite", tries = 3), - e2e_tests_step("smoke-tests-suite", tries = 3), - e2e_tests_step("panels-suite", tries = 3), - e2e_tests_step("various-suite", tries = 3), - e2e_tests_artifacts(), - build_storybook_step(ver_mode = ver_mode), - ] - - publish_steps = [] - - if ver_mode in ( - "release", - "release-branch", - ): - publish_steps.extend( - [ - upload_cdn_step(ver_mode = ver_mode, trigger = trigger_oss), - upload_packages_step( - ver_mode = ver_mode, - trigger = trigger_oss, - ), - ], - ) - - if ver_mode in ("release",): - publish_steps.extend( - [ - store_storybook_step(ver_mode = ver_mode), - store_npm_packages_step(), - ], - ) - - integration_test_steps = [ - postgres_integration_tests_step(), - mysql_integration_tests_step(), - redis_integration_tests_step(), - memcached_integration_tests_step(), - ] - - pipelines = [] - - # We don't need to run integration tests at release time since they have - # been run multiple times before: - if ver_mode in ("release"): - pipelines.append(whats_new_checker_pipeline(release_trigger)) - integration_test_steps = [] - volumes = [] - - windows_pipeline_dependencies = [ - "{}-build-e2e-publish".format(ver_mode), - "{}-test-frontend".format(ver_mode), - ] - pipelines.extend([ - pipeline( - name = "{}-build-e2e-publish".format(ver_mode), - trigger = trigger, - services = [], - steps = init_steps + build_steps + publish_steps, - environment = environment, - volumes = volumes, - ), - test_frontend(trigger, ver_mode), - test_backend(trigger, ver_mode), - ]) - - if ver_mode not in ("release"): - pipelines.append(pipeline( - name = "{}-integration-tests".format(ver_mode), - trigger = trigger, - services = services, - steps = [ - download_grabpl_step(), - identify_runner_step(), - verify_gen_cue_step(), - verify_gen_jsonnet_step(), - wire_install_step(), - ] + - integration_test_steps, - environment = environment, - volumes = volumes, - )) - - windows_pipeline = pipeline( - name = "{}-windows".format(ver_mode), - trigger = trigger, - steps = get_windows_steps(ver_mode = ver_mode), - platform = "windows", - depends_on = windows_pipeline_dependencies, - environment = environment, - ) - - pipelines.append(windows_pipeline) - - return pipelines - def publish_artifacts_step(): return { "name": "publish-artifacts", - "image": images["publish_image"], + "image": images["publish"], "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -283,7 +91,7 @@ def publish_artifacts_step(): def publish_static_assets_step(): return { "name": "publish-static-assets", - "image": images["publish_image"], + "image": images["publish"], "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -298,7 +106,7 @@ def publish_static_assets_step(): def publish_storybook_step(): return { "name": "publish-storybook", - "image": images["publish_image"], + "image": images["publish"], "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -410,12 +218,12 @@ def integration_test_pipelines(): } pipelines = [] volumes = integration_test_services_volumes() - integration_test_steps = [ - postgres_integration_tests_step(), - mysql_integration_tests_step(), - redis_integration_tests_step(), - memcached_integration_tests_step(), - ] + integration_test_steps = postgres_integration_tests_steps() + \ + mysql_integration_tests_steps("mysql57", "5.7") + \ + mysql_integration_tests_steps("mysql80", "8.0") + \ + redis_integration_tests_steps() + \ + memcached_integration_tests_steps() + \ + remote_alertmanager_integration_tests_steps() pipelines.append(pipeline( name = "integration-tests", @@ -440,7 +248,7 @@ def verify_release_pipeline( bucket = from_secret(prerelease_bucket), gcp_key = from_secret(rgm_gcp_key_base64), version = "${DRONE_TAG}", - trigger = release_trigger, + trigger = {}, depends_on = [ "release-build-e2e-publish", "release-windows", @@ -454,7 +262,7 @@ def verify_release_pipeline( step = { "name": "gsutil-stat", "depends_on": ["clone"], - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "environment": { "BUCKET": bucket, "GCP_KEY": gcp_key, diff --git a/scripts/drone/pipelines/benchmarks.star b/scripts/drone/pipelines/benchmarks.star new file mode 100644 index 00000000000..ae5a9611675 --- /dev/null +++ b/scripts/drone/pipelines/benchmarks.star @@ -0,0 +1,80 @@ +""" +This module returns the pipeline used for integration benchmarks. +""" + +load( + "scripts/drone/services/services.star", + "integration_test_services", + "integration_test_services_volumes", +) +load( + "scripts/drone/steps/lib.star", + "compile_build_cmd", + "enterprise_setup_step", + "integration_benchmarks_step", + "verify_gen_cue_step", + "verify_gen_jsonnet_step", + "wire_install_step", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", +) + +def integration_benchmarks(prefix): + """Generate a pipeline for integration tests. + + Args: + prefix: used in the naming of the pipeline. + Returns: + Drone pipeline. + """ + environment = {"EDITION": "oss"} + + services = integration_test_services() + volumes = integration_test_services_volumes() + + # In pull requests, attempt to clone grafana enterprise. + init_steps = [enterprise_setup_step(isPromote = True)] + + verify_step = verify_gen_cue_step() + verify_jsonnet_step = verify_gen_jsonnet_step() + + # Ensure that verif_gen_cue happens after we clone enterprise + # At the time of writing this, very_gen_cue is depended on by the wire step which is what everything else depends on. + verify_step["depends_on"].append("clone-enterprise") + verify_jsonnet_step["depends_on"].append("clone-enterprise") + + init_steps += [ + compile_build_cmd(), + verify_step, + verify_jsonnet_step, + wire_install_step(), + ] + + benchmark_steps = integration_benchmarks_step("sqlite") + \ + integration_benchmarks_step("postgres", { + "PGPASSWORD": "grafanatest", + "GRAFANA_TEST_DB": "postgres", + "POSTGRES_HOST": "postgres", + }) + \ + integration_benchmarks_step("mysql-5.7", { + "GRAFANA_TEST_DB": "mysql", + "MYSQL_HOST": "mysql57", + }) + \ + integration_benchmarks_step("mysql-8.0", { + "GRAFANA_TEST_DB": "mysql", + "MYSQL_HOST": "mysql80", + }) + + return pipeline( + name = "{}-integration-benchmarks".format(prefix), + trigger = { + "event": ["promote"], + "target": ["gobenchmarks"], + }, + environment = environment, + services = services, + volumes = volumes, + steps = init_steps + benchmark_steps, + ) diff --git a/scripts/drone/pipelines/build.star b/scripts/drone/pipelines/build.star index 7085a7f9d42..88b5ed8e52a 100644 --- a/scripts/drone/pipelines/build.star +++ b/scripts/drone/pipelines/build.star @@ -2,15 +2,10 @@ load( "scripts/drone/steps/lib.star", - "build_backend_step", - "build_docker_images_step", "build_frontend_package_step", - "build_frontend_step", - "build_plugins_step", "build_storybook_step", "cloud_plugins_e2e_tests_step", "compile_build_cmd", - "copy_packages_for_docker_step", "download_grabpl_step", "e2e_tests_artifacts", "e2e_tests_step", @@ -18,12 +13,12 @@ load( "frontend_metrics_step", "grafana_server_step", "identify_runner_step", - "package_step", "publish_images_step", "release_canary_npm_packages_step", "store_storybook_step", "test_a11y_frontend_step", "trigger_oss", + "update_package_json_version", "upload_cdn_step", "upload_packages_step", "verify_gen_cue_step", @@ -31,6 +26,15 @@ load( "wire_install_step", "yarn_install_step", ) +load( + "scripts/drone/steps/rgm.star", + "rgm_build_docker_step", + "rgm_package_step", +) +load( + "scripts/drone/utils/images.star", + "images", +) load( "scripts/drone/utils/utils.star", "pipeline", @@ -42,7 +46,7 @@ def build_e2e(trigger, ver_mode): Args: trigger: controls which events can trigger the pipeline execution. - ver_mode: used in the naming of the pipeline. + ver_mode: used in the naming of the pipeline. Either 'pr' or 'main'. Returns: Drone pipeline. @@ -64,17 +68,19 @@ def build_e2e(trigger, ver_mode): if ver_mode == "pr": build_steps.extend( [ + build_frontend_package_step(), enterprise_downstream_step(ver_mode = ver_mode), ], ) + else: + build_steps.extend([ + update_package_json_version(), + build_frontend_package_step(depends_on = ["update-package-json-version"]), + ]) build_steps.extend( [ - build_backend_step(ver_mode = ver_mode), - build_frontend_step(ver_mode = ver_mode), - build_frontend_package_step(ver_mode = ver_mode), - build_plugins_step(ver_mode = ver_mode), - package_step(ver_mode = ver_mode), + rgm_package_step(distros = "linux/amd64,linux/arm64", file = "packages.txt"), grafana_server_step(), e2e_tests_step("dashboards-suite"), e2e_tests_step("smoke-tests-suite"), @@ -87,7 +93,6 @@ def build_e2e(trigger, ver_mode): ), e2e_tests_artifacts(), build_storybook_step(ver_mode = ver_mode), - copy_packages_for_docker_step(), test_a11y_frontend_step(ver_mode = ver_mode), ], ) @@ -97,12 +102,12 @@ def build_e2e(trigger, ver_mode): [ store_storybook_step(trigger = trigger_oss, ver_mode = ver_mode), frontend_metrics_step(trigger = trigger_oss), - build_docker_images_step( - publish = False, - ), - build_docker_images_step( - publish = False, - ubuntu = True, + rgm_build_docker_step( + "packages.txt", + images["ubuntu"], + images["alpine"], + tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}", + ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}", ), publish_images_step( docker_repo = "grafana", @@ -128,16 +133,12 @@ def build_e2e(trigger, ver_mode): elif ver_mode == "pr": build_steps.extend( [ - build_docker_images_step( - archs = [ - "amd64", - ], - ), - build_docker_images_step( - archs = [ - "amd64", - ], - ubuntu = True, + rgm_build_docker_step( + "packages.txt", + images["ubuntu"], + images["alpine"], + tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}", + ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}", ), publish_images_step( docker_repo = "grafana", diff --git a/scripts/drone/pipelines/ci_images.star b/scripts/drone/pipelines/ci_images.star index 65df476754b..e8180d6d0ef 100644 --- a/scripts/drone/pipelines/ci_images.star +++ b/scripts/drone/pipelines/ci_images.star @@ -3,21 +3,21 @@ This module contains steps and pipelines relating to creating CI Docker images. """ load( - "scripts/drone/utils/utils.star", - "pipeline", + "scripts/drone/utils/images.star", + "images", ) load( - "scripts/drone/vault.star", - "from_secret", - "gcp_download_build_container_assets_key", + "scripts/drone/utils/utils.star", + "pipeline", ) load( "scripts/drone/utils/windows_images.star", "windows_images", ) load( - "scripts/drone/utils/images.star", - "images", + "scripts/drone/vault.star", + "from_secret", + "gcp_download_build_container_assets_key", ) def publish_ci_windows_test_image_pipeline(): @@ -32,7 +32,7 @@ def publish_ci_windows_test_image_pipeline(): steps = [ { "name": "clone", - "image": windows_images["wix_image"], + "image": windows_images["wix"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, @@ -43,7 +43,7 @@ def publish_ci_windows_test_image_pipeline(): }, { "name": "build-and-publish", - "image": windows_images["windows_server_core_image"], + "image": windows_images["windows_server_core"], "environment": { "DOCKER_USERNAME": from_secret("docker_username"), "DOCKER_PASSWORD": from_secret("docker_password"), @@ -81,14 +81,14 @@ def publish_ci_build_container_image_pipeline(): steps = [ { "name": "validate-version", - "image": images["alpine_image"], + "image": images["alpine"], "commands": [ "if [ -z \"${BUILD_CONTAINER_VERSION}\" ]; then echo Missing BUILD_CONTAINER_VERSION; false; fi", ], }, { "name": "download-macos-sdk", - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "environment": { "GCP_KEY": from_secret(gcp_download_build_container_assets_key), }, @@ -100,7 +100,7 @@ def publish_ci_build_container_image_pipeline(): }, { "name": "build-and-publish", # Consider splitting the build and the upload task. - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], "environment": { "DOCKER_USERNAME": from_secret("docker_username"), diff --git a/scripts/drone/pipelines/docs.star b/scripts/drone/pipelines/docs.star index 5f2cbdb8ef3..536749b9e04 100644 --- a/scripts/drone/pipelines/docs.star +++ b/scripts/drone/pipelines/docs.star @@ -7,6 +7,7 @@ load( "build_docs_website_step", "codespell_step", "identify_runner_step", + "verify_gen_cue_step", "yarn_install_step", ) load( @@ -35,6 +36,7 @@ def docs_pipelines(ver_mode, trigger): codespell_step(), lint_docs(), build_docs_website_step(), + verify_gen_cue_step(), ] return pipeline( @@ -48,7 +50,7 @@ def docs_pipelines(ver_mode, trigger): def lint_docs(): return { "name": "lint-docs", - "image": images["build_image"], + "image": images["node"], "depends_on": [ "yarn-install", ], @@ -66,6 +68,9 @@ def trigger_docs_main(): "event": [ "push", ], + "repo": [ + "grafana/grafana", + ], "paths": docs_paths, } @@ -74,5 +79,8 @@ def trigger_docs_pr(): "event": [ "pull_request", ], + "repo": [ + "grafana/grafana", + ], "paths": docs_paths, } diff --git a/scripts/drone/pipelines/integration_tests.star b/scripts/drone/pipelines/integration_tests.star index a6553e768b1..e51b55bf48f 100644 --- a/scripts/drone/pipelines/integration_tests.star +++ b/scripts/drone/pipelines/integration_tests.star @@ -2,25 +2,26 @@ This module returns the pipeline used for integration tests. """ +load( + "scripts/drone/services/services.star", + "integration_test_services", + "integration_test_services_volumes", +) load( "scripts/drone/steps/lib.star", "compile_build_cmd", "download_grabpl_step", "enterprise_setup_step", "identify_runner_step", - "memcached_integration_tests_step", - "mysql_integration_tests_step", - "postgres_integration_tests_step", - "redis_integration_tests_step", + "memcached_integration_tests_steps", + "mysql_integration_tests_steps", + "postgres_integration_tests_steps", + "redis_integration_tests_steps", + "remote_alertmanager_integration_tests_steps", "verify_gen_cue_step", "verify_gen_jsonnet_step", "wire_install_step", ) -load( - "scripts/drone/services/services.star", - "integration_test_services", - "integration_test_services_volumes", -) load( "scripts/drone/utils/utils.star", "pipeline", @@ -60,12 +61,13 @@ def integration_tests(trigger, prefix, ver_mode = "pr"): wire_install_step(), ] - test_steps = [ - postgres_integration_tests_step(), - mysql_integration_tests_step(), - redis_integration_tests_step(), - memcached_integration_tests_step(), - ] + # test_steps = [a, b] + [c, d] + [e, f]... + test_steps = postgres_integration_tests_steps() + \ + mysql_integration_tests_steps("mysql57", "5.7") + \ + mysql_integration_tests_steps("mysql80", "8.0") + \ + redis_integration_tests_steps() + \ + memcached_integration_tests_steps() + \ + remote_alertmanager_integration_tests_steps() return pipeline( name = "{}-integration-tests".format(prefix), diff --git a/scripts/drone/pipelines/lint_backend.star b/scripts/drone/pipelines/lint_backend.star index f8253d0c544..e0c25495b05 100644 --- a/scripts/drone/pipelines/lint_backend.star +++ b/scripts/drone/pipelines/lint_backend.star @@ -44,6 +44,10 @@ def lint_backend_pipeline(trigger, ver_mode): test_steps = [ lint_backend_step(), + # modowners doesn't exist for versions below 10.1.x. + # validate_modfile_step(), + # OpenAPI spec validation is disabled for versions below 10.2.x. + # validate_openapi_spec_step(), ] if ver_mode == "main": diff --git a/scripts/drone/pipelines/lint_frontend.star b/scripts/drone/pipelines/lint_frontend.star index 2f3724b303a..c0e035babcf 100644 --- a/scripts/drone/pipelines/lint_frontend.star +++ b/scripts/drone/pipelines/lint_frontend.star @@ -28,6 +28,8 @@ def lint_frontend_pipeline(trigger, ver_mode): init_steps = [] lint_step = lint_frontend_step() + # i18n step is disabled for versions below 10.1.x. + # i18n_step = verify_i18n_step() if ver_mode == "pr": # In pull requests, attempt to clone grafana enterprise. @@ -39,6 +41,8 @@ def lint_frontend_pipeline(trigger, ver_mode): ] test_steps = [ lint_step, + # i18n step is disabled for versions below 10.1.x. + # i18n_step, ] return pipeline( diff --git a/scripts/drone/pipelines/shellcheck.star b/scripts/drone/pipelines/shellcheck.star index 5b8d481094c..7f56c553976 100644 --- a/scripts/drone/pipelines/shellcheck.star +++ b/scripts/drone/pipelines/shellcheck.star @@ -3,14 +3,14 @@ This module returns a Drone step and pipeline for linting with shellcheck. """ load("scripts/drone/steps/lib.star", "compile_build_cmd") -load( - "scripts/drone/utils/utils.star", - "pipeline", -) load( "scripts/drone/utils/images.star", "images", ) +load( + "scripts/drone/utils/utils.star", + "pipeline", +) trigger = { "event": [ @@ -29,12 +29,10 @@ trigger = { def shellcheck_step(): return { "name": "shellcheck", - "image": images["build_image"], - "depends_on": [ - "compile-build-cmd", - ], + "image": images["ubuntu"], "commands": [ - "./bin/build shellcheck", + "apt-get update -yq && apt-get install shellcheck", + "shellcheck -e SC1071 -e SC2162 scripts/**/*.sh", ], } diff --git a/scripts/drone/pipelines/test_backend.star b/scripts/drone/pipelines/test_backend.star index ad1394196f3..c5bb47da11e 100644 --- a/scripts/drone/pipelines/test_backend.star +++ b/scripts/drone/pipelines/test_backend.star @@ -4,7 +4,6 @@ This module returns the pipeline used for testing backend code. load( "scripts/drone/steps/lib.star", - "compile_build_cmd", "enterprise_setup_step", "identify_runner_step", "test_backend_integration_step", @@ -41,7 +40,6 @@ def test_backend(trigger, ver_mode): steps += [ identify_runner_step(), - compile_build_cmd(), verify_step, verify_jsonnet_step, wire_install_step(), diff --git a/scripts/drone/pipelines/test_frontend.star b/scripts/drone/pipelines/test_frontend.star index e9e353720bd..9a01f101c58 100644 --- a/scripts/drone/pipelines/test_frontend.star +++ b/scripts/drone/pipelines/test_frontend.star @@ -5,7 +5,6 @@ This module returns the pipeline used for testing backend code. load( "scripts/drone/steps/lib.star", "betterer_frontend_step", - "download_grabpl_step", "enterprise_setup_step", "identify_runner_step", "test_frontend_step", @@ -30,7 +29,6 @@ def test_frontend(trigger, ver_mode): steps = [ identify_runner_step(), - download_grabpl_step(), yarn_install_step(), betterer_frontend_step(), ] diff --git a/scripts/drone/pipelines/whats_new_checker.star b/scripts/drone/pipelines/whats_new_checker.star index e15a8d7a291..2c56be573ce 100644 --- a/scripts/drone/pipelines/whats_new_checker.star +++ b/scripts/drone/pipelines/whats_new_checker.star @@ -2,14 +2,14 @@ This module contains logic for checking if the package.json whats new url matches with the in-flight tag. """ -load( - "scripts/drone/utils/images.star", - "images", -) load( "scripts/drone/steps/lib.star", "compile_build_cmd", ) +load( + "scripts/drone/utils/images.star", + "images", +) load( "scripts/drone/utils/utils.star", "pipeline", @@ -18,7 +18,7 @@ load( def whats_new_checker_step(): return { "name": "whats-new-checker", - "image": images["go_image"], + "image": images["go"], "depends_on": [ "compile-build-cmd", ], diff --git a/scripts/drone/pipelines/windows.star b/scripts/drone/pipelines/windows.star index fd5b99c9bf0..59c12a19536 100644 --- a/scripts/drone/pipelines/windows.star +++ b/scripts/drone/pipelines/windows.star @@ -3,15 +3,15 @@ This module returns the pipeline used for building Grafana on Windows. """ load( - "scripts/drone/utils/utils.star", - "pipeline", + "scripts/drone/steps/lib_windows.star", + "clone_step_windows", + "get_windows_steps", + "test_backend_step_windows", + "wire_install_step_windows", ) load( - "scripts/drone/steps/lib.star", - "get_windows_steps", - "windows_clone_step", - "windows_test_backend_step", - "windows_wire_install_step", + "scripts/drone/utils/utils.star", + "pipeline", ) load( "scripts/drone/utils/windows_images.star", @@ -30,19 +30,19 @@ def windows_test_backend(trigger, edition, ver_mode): """ environment = {"EDITION": edition} steps = [ - windows_clone_step(), + clone_step_windows(), ] steps.extend([{ "name": "windows-init", - "image": windows_images["windows_go_image"], + "image": windows_images["go"], "depends_on": ["clone"], "commands": [], }]) steps.extend([ - windows_wire_install_step(edition), - windows_test_backend_step(), + wire_install_step_windows(edition), + test_backend_step_windows(), ]) pl = pipeline( name = "{}-test-backend-windows".format(ver_mode), diff --git a/scripts/drone/rgm.star b/scripts/drone/rgm.star index 0721f76b32b..cd4df9b9de8 100644 --- a/scripts/drone/rgm.star +++ b/scripts/drone/rgm.star @@ -4,50 +4,52 @@ rgm uses 'github.com/grafana/grafana-build' to build Grafana on the following ev * A tag that begins with a 'v' """ -load( - "scripts/drone/steps/lib.star", - "get_windows_steps", -) -load( - "scripts/drone/utils/utils.star", - "ignore_failure", - "pipeline", -) load( "scripts/drone/events/release.star", "verify_release_pipeline", ) -load( - "scripts/drone/pipelines/test_frontend.star", - "test_frontend", -) load( "scripts/drone/pipelines/test_backend.star", "test_backend", ) +load( + "scripts/drone/pipelines/test_frontend.star", + "test_frontend", +) load( "scripts/drone/pipelines/whats_new_checker.star", "whats_new_checker_pipeline", ) +load( + "scripts/drone/steps/lib_windows.star", + "get_windows_steps", +) +load( + "scripts/drone/utils/images.star", + "images", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", + "with_deps", +) +load( + "scripts/drone/variables.star", + "golang_version", +) load( "scripts/drone/vault.star", "from_secret", + "npm_token", + "rgm_cdn_destination", "rgm_dagger_token", "rgm_destination", + "rgm_downloads_destination", "rgm_gcp_key_base64", "rgm_github_token", + "rgm_storybook_destination", ) -rgm_env_secrets = { - "GCP_KEY_BASE64": from_secret(rgm_gcp_key_base64), - "DESTINATION": from_secret(rgm_destination), - "GITHUB_TOKEN": from_secret(rgm_github_token), - "_EXPERIMENTAL_DAGGER_CLOUD_TOKEN": from_secret(rgm_dagger_token), - "GPG_PRIVATE_KEY": from_secret("packages_gpg_private_key"), - "GPG_PUBLIC_KEY": from_secret("packages_gpg_public_key"), - "GPG_PASSPHRASE": from_secret("packages_gpg_passphrase"), -} - docs_paths = { "exclude": [ "*.md", @@ -73,75 +75,233 @@ tag_trigger = { }, } -def rgm_build(script = "drone_publish_main.sh", canFail = True): - rgm_build_step = { - "name": "rgm-build", +main_trigger = { + "event": [ + "push", + ], + "branch": "main", + "paths": docs_paths, + "repo": [ + "grafana/grafana", + ], +} + +nightly_trigger = { + "event": { + "include": [ + "cron", + ], + }, + "cron": { + "include": [ + "nightly-release", + ], + }, +} + +version_branch_trigger = {"ref": ["refs/heads/v[0-9]*"]} + +def rgm_env_secrets(env): + """Adds the rgm secret ENV variables to the given env arg + + Args: + env: A map of environment varables. This function will adds the necessary secrets to it (and potentially overwrite them). + Returns: + Drone step. + """ + env["DESTINATION"] = from_secret(rgm_destination) + env["STORYBOOK_DESTINATION"] = from_secret(rgm_storybook_destination) + env["CDN_DESTINATION"] = from_secret(rgm_cdn_destination) + env["DOWNLOADS_DESTINATION"] = from_secret(rgm_downloads_destination) + + env["GCP_KEY_BASE64"] = from_secret(rgm_gcp_key_base64) + env["GITHUB_TOKEN"] = from_secret(rgm_github_token) + env["_EXPERIMENTAL_DAGGER_CLOUD_TOKEN"] = from_secret(rgm_dagger_token) + env["GPG_PRIVATE_KEY"] = from_secret("packages_gpg_private_key") + env["GPG_PUBLIC_KEY"] = from_secret("packages_gpg_public_key") + env["GPG_PASSPHRASE"] = from_secret("packages_gpg_passphrase") + env["DOCKER_USERNAME"] = from_secret("docker_username") + env["DOCKER_PASSWORD"] = from_secret("docker_password") + env["NPM_TOKEN"] = from_secret(npm_token) + env["GCOM_API_KEY"] = from_secret("grafana_api_key") + return env + +def rgm_run(name, script): + """Returns a pipeline that does a full build & package of Grafana. + + Args: + name: The name of the pipeline step. + script: The script in the container to run. + Returns: + Drone step. + """ + env = { + "GO_VERSION": golang_version, + } + rgm_run_step = { + "name": name, "image": "grafana/grafana-build:main", + "pull": "always", "commands": [ "export GRAFANA_DIR=$$(pwd)", "cd /src && ./scripts/{}".format(script), ], - "environment": rgm_env_secrets, + "environment": rgm_env_secrets(env), # The docker socket is a requirement for running dagger programs # In the future we should find a way to use dagger without mounting the docker socket. "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], } - if canFail: - rgm_build_step["failure"] = "ignore" return [ - rgm_build_step, + rgm_run_step, ] -def rgm_main(): - trigger = { - "event": [ - "push", - ], - "branch": "main", - "paths": docs_paths, - "repo": [ - "grafana/grafana", - ], +def rgm_copy(src, dst): + """Copies file from/to GCS. + + Args: + src: source of the files. + dst: destination of the files. + + Returns: + Drone steps. + """ + commands = [ + "printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json", + "gcloud auth activate-service-account --key-file=/tmp/key.json", + "gcloud storage cp -r {} {}".format(src, dst), + ] + + if not dst.startswith("gs://"): + commands.insert(0, "mkdir -p {}".format(dst)) + + rgm_copy_step = { + "name": "rgm-copy", + "image": "google/cloud-sdk:alpine", + "commands": commands, + "environment": rgm_env_secrets({}), } + return [ + rgm_copy_step, + ] + +def rgm_publish_packages(bucket = "grafana-packages"): + """Publish deb and rpm packages. + + Args: + bucket: target bucket to publish the packages. + + Returns: + Drone steps. + """ + steps = [] + for package_manager in ["deb", "rpm"]: + steps.append({ + "name": "publish-{}".format(package_manager), + # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image + "image": images["package_publish"], + "privileged": True, + "settings": { + "access_key_id": from_secret("packages_access_key_id"), + "secret_access_key": from_secret("packages_secret_access_key"), + "service_account_json": from_secret("packages_service_account"), + "target_bucket": bucket, + "gpg_passphrase": from_secret("packages_gpg_passphrase"), + "gpg_public_key": from_secret("packages_gpg_public_key"), + "gpg_private_key": from_secret("packages_gpg_private_key"), + "package_path": "file:///drone/src/dist/*.{}".format(package_manager), + }, + }) + + return steps + +def rgm_main(): + # Runs a package / build process (with some distros) when commits are merged to main return pipeline( name = "rgm-main-prerelease", - trigger = trigger, - steps = rgm_build(canFail = True), + trigger = main_trigger, + steps = rgm_run("rgm-build", "drone_publish_main.sh"), depends_on = ["main-test-backend", "main-test-frontend"], ) def rgm_tag(): + # Runs a package / build process (with all distros) when a tag is made return pipeline( name = "rgm-tag-prerelease", trigger = tag_trigger, - steps = rgm_build(script = "drone_publish_tag_grafana.sh", canFail = False), + steps = rgm_run("rgm-build", "drone_publish_tag_grafana.sh"), depends_on = ["release-test-backend", "release-test-frontend"], ) -def rgm_windows(): +def rgm_tag_windows(): return pipeline( name = "rgm-tag-prerelease-windows", trigger = tag_trigger, - steps = ignore_failure( - get_windows_steps( - ver_mode = "release", - bucket = "grafana-prerelease", - ), + steps = get_windows_steps( + ver_mode = "release", + bucket = "grafana-prerelease", ), depends_on = ["rgm-tag-prerelease"], platform = "windows", ) -def rgm(): +def rgm_version_branch(): + # Runs a package / build proces (with all distros) when a commit lands on a version branch + return pipeline( + name = "rgm-version-branch-prerelease", + trigger = version_branch_trigger, + steps = rgm_run("rgm-build", "drone_publish_tag_grafana.sh"), + depends_on = ["release-test-backend", "release-test-frontend"], + ) + +def rgm_nightly_build(): + src = "$${DRONE_WORKSPACE}/dist/*" + dst = "$${DESTINATION}/$${DRONE_BUILD_EVENT}" + + copy_steps = with_deps(rgm_copy(src, dst), ["rgm-build"]) + + return pipeline( + name = "rgm-nightly-build", + trigger = nightly_trigger, + steps = rgm_run("rgm-build", "drone_build_nightly_grafana.sh") + copy_steps, + depends_on = ["nightly-test-backend", "nightly-test-frontend"], + ) + +def rgm_nightly_publish(): + """Nightly publish pipeline. + + Returns: + Drone pipeline. + """ + src = "$${DESTINATION}/$${DRONE_BUILD_EVENT}/*_$${DRONE_BUILD_NUMBER}_*" + dst = "$${DRONE_WORKSPACE}/dist" + + publish_steps = with_deps(rgm_run("rgm-publish", "drone_publish_nightly_grafana.sh"), ["rgm-copy"]) + package_steps = with_deps(rgm_publish_packages(), ["rgm-publish"]) + + return pipeline( + name = "rgm-nightly-publish", + trigger = nightly_trigger, + steps = rgm_copy(src, dst) + publish_steps + package_steps, + depends_on = ["rgm-nightly-build"], + ) + +def rgm_nightly_pipeline(): + return [ + test_frontend(nightly_trigger, "nightly"), + test_backend(nightly_trigger, "nightly"), + rgm_nightly_build(), + rgm_nightly_publish(), + ] + +def rgm_tag_pipeline(): return [ whats_new_checker_pipeline(tag_trigger), test_frontend(tag_trigger, "release"), test_backend(tag_trigger, "release"), - rgm_main(), rgm_tag(), - rgm_windows(), + rgm_tag_windows(), verify_release_pipeline( trigger = tag_trigger, name = "rgm-tag-verify-prerelease-assets", @@ -152,3 +312,29 @@ def rgm(): ], ), ] + +def rgm_version_branch_pipeline(): + return [ + rgm_version_branch(), + verify_release_pipeline( + trigger = version_branch_trigger, + name = "rgm-prerelease-verify-prerelease-assets", + bucket = "grafana-prerelease", + depends_on = [ + "rgm-version-branch-prerelease", + ], + ), + ] + +def rgm_main_pipeline(): + return [ + rgm_main(), + ] + +def rgm(): + return ( + rgm_main_pipeline() + + rgm_tag_pipeline() + + rgm_version_branch_pipeline() + + rgm_nightly_pipeline() + ) diff --git a/scripts/drone/services/services.star b/scripts/drone/services/services.star index ad35f4523aa..911aacfb000 100644 --- a/scripts/drone/services/services.star +++ b/scripts/drone/services/services.star @@ -10,14 +10,15 @@ load( def integration_test_services_volumes(): return [ {"name": "postgres", "temp": {"medium": "memory"}}, - {"name": "mysql", "temp": {"medium": "memory"}}, + {"name": "mysql57", "temp": {"medium": "memory"}}, + {"name": "mysql80", "temp": {"medium": "memory"}}, ] def integration_test_services(): services = [ { "name": "postgres", - "image": images["postgres_alpine_image"], + "image": images["postgres_alpine"], "environment": { "POSTGRES_USER": "grafanatest", "POSTGRES_PASSWORD": "grafanatest", @@ -29,24 +30,43 @@ def integration_test_services(): ], }, { - "name": "mysql", - "image": images["mysql5_image"], + "name": "mysql57", + "image": images["mysql5"], "environment": { "MYSQL_ROOT_PASSWORD": "rootpass", "MYSQL_DATABASE": "grafana_tests", "MYSQL_USER": "grafana", "MYSQL_PASSWORD": "password", }, - "volumes": [{"name": "mysql", "path": "/var/lib/mysql"}], + "volumes": [{"name": "mysql57", "path": "/var/lib/mysql"}], + "commands": ["docker-entrypoint.sh mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci"], + }, + { + "name": "mysql80", + "image": images["mysql8"], + "environment": { + "MYSQL_ROOT_PASSWORD": "rootpass", + "MYSQL_DATABASE": "grafana_tests", + "MYSQL_USER": "grafana", + "MYSQL_PASSWORD": "password", + }, + "volumes": [{"name": "mysql80", "path": "/var/lib/mysql"}], + "commands": ["docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password"], + }, + { + "name": "mimir_backend", + "image": images["mimir"], + "environment": {}, + "commands": ["/bin/mimir -target=backend"], }, { "name": "redis", - "image": images["redis_alpine_image"], + "image": images["redis_alpine"], "environment": {}, }, { "name": "memcached", - "image": images["memcached_alpine_image"], + "image": images["memcached_alpine"], "environment": {}, }, ] @@ -56,7 +76,7 @@ def integration_test_services(): def ldap_service(): return { "name": "ldap", - "image": images["openldap_image"], + "image": images["openldap"], "environment": { "LDAP_ADMIN_PASSWORD": "grafana", "LDAP_DOMAIN": "grafana.org", diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 9a5045c1798..c723d84dd61 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -2,6 +2,18 @@ This module is a library of Drone steps and other pipeline components. """ +load( + "scripts/drone/steps/rgm.star", + "rgm_build_backend_step", +) +load( + "scripts/drone/utils/images.star", + "images", +) +load( + "scripts/drone/variables.star", + "grabpl_version", +) load( "scripts/drone/vault.star", "from_secret", @@ -11,16 +23,6 @@ load( "npm_token", "prerelease_bucket", ) -load( - "scripts/drone/utils/images.star", - "images", -) -load( - "scripts/drone/utils/windows_images.star", - "windows_images", -) - -grabpl_version = "v3.0.41" trigger_oss = { "repo": [ @@ -28,21 +30,10 @@ trigger_oss = { ], } -def slack_step(channel, template, secret): - return { - "name": "slack", - "image": images["plugins_slack_image"], - "settings": { - "webhook": from_secret(secret), - "channel": channel, - "template": template, - }, - } - def yarn_install_step(): return { "name": "yarn-install", - "image": images["build_image"], + "image": images["node"], "commands": [ "yarn install --immutable", ], @@ -52,8 +43,9 @@ def yarn_install_step(): def wire_install_step(): return { "name": "wire-install", - "image": images["build_image"], + "image": images["go"], "commands": [ + "apk add --update make", "make gen-go", ], "depends_on": [ @@ -61,39 +53,26 @@ def wire_install_step(): ], } -def windows_wire_install_step(edition): +def identify_runner_step(): return { - "name": "wire-install", - "image": windows_images["windows_go_image"], + "name": "identify-runner", + "image": images["alpine"], "commands": [ - "go install github.com/google/wire/cmd/wire@v0.5.0", - "wire gen -tags {} ./pkg/server".format(edition), - ], - "depends_on": [ - "windows-init", + "echo $DRONE_RUNNER_NAME", ], } -def identify_runner_step(platform = "linux"): - if platform == "linux": - return { - "name": "identify-runner", - "image": images["alpine_image"], - "commands": [ - "echo $DRONE_RUNNER_NAME", - ], - } - else: - return { - "name": "identify-runner", - "image": windows_images["1809_image"], - "commands": [ - "echo $env:DRONE_RUNNER_NAME", - ], - } +def enterprise_setup_step(source = "${DRONE_SOURCE_BRANCH}", canFail = True, isPromote = False): + """Setup the enterprise source into the ./grafana-enterprise directory. -def enterprise_setup_step(source = "${DRONE_SOURCE_BRANCH}", canFail = True): - step = clone_enterprise_step_pr(source = source, target = "${DRONE_TARGET_BRANCH}", canFail = canFail, location = "../grafana-enterprise") + Args: + source: controls which revision of grafana-enterprise is checked out, if it exists. The name 'source' derives from the 'source branch' of a pull request. + canFail: controls whether the step can fail. This is useful for pull requests where the enterprise source may not exist. + isPromote: controls whether or not this step is being used in a promote pipeline. If it is, then the clone enterprise step will not check if the pull request is a fork. + Returns: + Drone step. + """ + step = clone_enterprise_step_pr(source = source, target = "${DRONE_TARGET_BRANCH}", canFail = canFail, location = "../grafana-enterprise", isPromote = isPromote) step["commands"] += [ "cd ../", "ln -s src grafana", @@ -103,7 +82,7 @@ def enterprise_setup_step(source = "${DRONE_SOURCE_BRANCH}", canFail = True): return step -def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFail = False, location = "grafana-enterprise"): +def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFail = False, location = "grafana-enterprise", isPromote = False): """Clone the enterprise source into the ./grafana-enterprise directory. Args: @@ -111,18 +90,28 @@ def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFai target: controls which revision of grafana-enterprise is checked out, if it 'source' does not exist. The name 'target' derives from the 'target branch' of a pull request. If this does not exist, then 'main' will be checked out. canFail: controls whether or not this step is allowed to fail. If it fails and this is true, then the pipeline will continue. canFail is used in pull request pipelines where enterprise may be cloned but may not clone in forks. location: the path where grafana-enterprise is cloned. + isPromote: controls whether or not this step is being used in a promote pipeline. If it is, then the step will not check if the pull request is a fork. Returns: Drone step. """ + + if isPromote: + check = [] + else: + check = [ + 'is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork)', + 'if [ "$is_fork" != false ]; then return 1; fi', # Only clone if we're confident that 'fork' is 'false'. Fail if it's also empty. + ] + step = { "name": "clone-enterprise", - "image": images["build_image"], + "image": images["git"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, "commands": [ - 'is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork)', - 'if [ "$is_fork" != false ]; then return 1; fi', # Only clone if we're confident that 'fork' is 'false'. Fail if it's also empty. + "apk add --update curl jq bash", + ] + check + [ 'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" ' + location, "cd {}".format(location), 'if git checkout {0}; then echo "checked out {0}"; elif git checkout {1}; then echo "git checkout {1}"; else git checkout main; fi'.format(source, target), @@ -134,22 +123,10 @@ def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFai return step -def download_grabpl_step(platform = "linux"): - if platform == "windows": - return { - "name": "grabpl", - "image": windows_images["wix_image"], - "commands": [ - '$$ProgressPreference = "SilentlyContinue"', - "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( - grabpl_version, - ), - ], - } - +def download_grabpl_step(): return { "name": "grabpl", - "image": images["curl_image"], + "image": images["curl"], "commands": [ "mkdir -p bin", "curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/grabpl".format( @@ -162,7 +139,7 @@ def download_grabpl_step(platform = "linux"): def lint_drone_step(): return { "name": "lint-drone", - "image": images["curl_image"], + "image": images["curl"], "commands": [ "./bin/build verify-drone", ], @@ -174,9 +151,10 @@ def lint_drone_step(): def lint_starlark_step(): return { "name": "lint-starlark", - "image": images["build_image"], + "image": images["go"], "commands": [ - "./bin/build verify-starlark .", + "go install github.com/bazelbuild/buildtools/buildifier@latest", + "buildifier --lint=warn -mode=check -r .", ], "depends_on": [ "compile-build-cmd", @@ -201,7 +179,7 @@ def enterprise_downstream_step(ver_mode): step = { "name": "trigger-enterprise-downstream", - "image": images["drone_downstream_image"], + "image": images["drone_downstream"], "settings": { "server": "https://drone.grafana.net", "token": from_secret("drone_token"), @@ -224,8 +202,7 @@ def enterprise_downstream_step(ver_mode): def lint_backend_step(): return { "name": "lint-backend", - # TODO: build_image or go_image? - "image": images["go_image"], + "image": images["go"], "environment": { # We need CGO because of go-sqlite3 "CGO_ENABLED": "1", @@ -234,32 +211,47 @@ def lint_backend_step(): "wire-install", ], "commands": [ - "apt-get update && apt-get install make", + "apk add --update make build-base", # Don't use Make since it will re-download the linters "make lint-go", ], } -def benchmark_ldap_step(): +def validate_modfile_step(): return { - "name": "benchmark-ldap", - "image": images["build_image"], - "environment": { - "LDAP_HOSTNAME": "ldap", - }, + "name": "validate-modfile", + "image": images["go"], "commands": [ - "dockerize -wait tcp://ldap:389 -timeout 120s", - 'go test -benchmem -run=^$ ./pkg/extensions/ldapsync -bench "^(Benchmark50Users)$"', + "go run scripts/modowners/modowners.go check go.mod", + ], + } + +def validate_openapi_spec_step(): + return { + "name": "validate-openapi-spec", + "image": images["go"], + "commands": [ + "apk add --update make", + "make swagger-validate", + ], + } + +def dockerize_step(name, hostname, port): + return { + "name": name, + "image": images["dockerize"], + "commands": [ + "dockerize -wait tcp://{}:{} -timeout 120s".format(hostname, port), ], } def build_storybook_step(ver_mode): return { "name": "build-storybook", - "image": images["build_image"], + "image": images["node"], "depends_on": [ # Best to ensure that this step doesn't mess with what's getting built and packaged - "build-frontend", + "rgm-package", "build-frontend-packages", ], "environment": { @@ -300,7 +292,7 @@ def store_storybook_step(ver_mode, trigger = None): step = { "name": "store-storybook", - "image": images["publish_image"], + "image": images["publish"], "depends_on": [ "build-storybook", ] + @@ -330,7 +322,7 @@ def store_storybook_step(ver_mode, trigger = None): def e2e_tests_artifacts(): return { "name": "e2e-tests-artifacts-upload", - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "depends_on": [ "end-to-end-tests-dashboards-suite", "end-to-end-tests-panels-suite", @@ -378,7 +370,7 @@ def upload_cdn_step(ver_mode, trigger = None): step = { "name": "upload-cdn-assets", - "image": images["publish_image"], + "image": images["publish"], "depends_on": [ "grafana-server", ], @@ -394,75 +386,27 @@ def upload_cdn_step(ver_mode, trigger = None): step = dict(step, when = trigger) return step -def build_backend_step(ver_mode, variants = None): +def build_backend_step(distros = "linux/amd64,linux/arm64"): """Build the backend code using the Grafana build tool. Args: - ver_mode: if ver_mode != 'release', pass the DRONE_BUILD_NUMBER environment - variable as the value for the --build-id option. - TODO: is this option actually used by the build-backend subcommand? - variants: a list of variants be passed to the build-backend subcommand - using the --variants option. - Defaults to None. + distros: a list of distributes to be built. For a full list, see `go tool dist list`. Returns: Drone step. """ - variants_str = "" - if variants: - variants_str = " --variants {}".format(",".join(variants)) - # TODO: Convert number of jobs to percentage - if ver_mode == "release": - cmds = [ - "./bin/build build-backend --jobs 8 --edition oss ${DRONE_TAG}", - ] - else: - build_no = "${DRONE_BUILD_NUMBER}" - cmds = [ - "./bin/build build-backend --jobs 8 --edition oss --build-id {}{}".format( - build_no, - variants_str, - ), - ] + return rgm_build_backend_step(distros) - return { - "name": "build-backend", - "image": images["build_image"], - "depends_on": [ - "wire-install", - "compile-build-cmd", - ], - "commands": cmds, - } - -def build_frontend_step(ver_mode): - """Build the frontend code using the Grafana build tool. - - Args: - ver_mode: if ver_mode != 'release', use the DRONE_BUILD_NUMBER environment - variable as a build identifier. +def build_frontend_step(): + """Build the frontend code to ensure it's compilable Returns: Drone step. """ - build_no = "${DRONE_BUILD_NUMBER}" - - # TODO: Use percentage for num jobs - if ver_mode == "release": - cmds = [ - "./bin/build build-frontend --jobs 8 " + - "--edition oss ${DRONE_TAG}", - ] - else: - cmds = [ - "./bin/build build-frontend --jobs 8 --edition oss" + - "--build-id {}".format(build_no), - ] - return { "name": "build-frontend", - "image": images["build_image"], + "image": images["node"], "environment": { "NODE_OPTIONS": "--max_old_space_size=8192", }, @@ -470,45 +414,60 @@ def build_frontend_step(ver_mode): "compile-build-cmd", "yarn-install", ], - "commands": cmds, + "commands": [ + "yarn build", + ], } -def build_frontend_package_step(ver_mode): +def update_package_json_version(): + """Updates the packages/ to use a version that has the build ID in it: 10.0.0pre -> 10.0.0-5432pre + + Returns: + Drone step that updates the 'version' key in package.json + """ + + return { + "name": "update-package-json-version", + "image": images["node"], + "depends_on": [ + "yarn-install", + ], + "commands": [ + "apk add --update jq", + "new_version=$(cat package.json | jq .version | sed s/pre/${DRONE_BUILD_NUMBER}/g)", + "echo \"New version: $new_version\"", + "yarn run lerna version $new_version --exact --no-git-tag-version --no-push --force-publish -y", + "yarn install --mode=update-lockfile", + ], + } + +def build_frontend_package_step(depends_on = []): """Build the frontend packages using the Grafana build tool. Args: - ver_mode: if ver_mode != 'release', use the DRONE_BUILD_NUMBER environment - variable as a build identifier. + depends_on: a list of step names (strings) that must complete before this step runs. Returns: Drone step. """ - build_no = "${DRONE_BUILD_NUMBER}" - # TODO: Use percentage for num jobs - if ver_mode == "release": - cmds = [ - "./bin/build build-frontend-packages --jobs 8 " + - "--edition oss ${DRONE_TAG}", - ] - else: - cmds = [ - "./bin/build build-frontend-packages --jobs 8 --edition oss" + - "--build-id {}".format(build_no), - "yarn packages:pack", - "./scripts/validate-npm-packages.sh", - ] + cmds = [ + "apk add --update jq bash", # bash is needed for the validate-npm-packages.sh script since it has a 'bash' + # shebang. + "yarn packages:build", + "yarn packages:pack", + "./scripts/validate-npm-packages.sh", + ] return { "name": "build-frontend-packages", - "image": images["build_image"], + "image": images["node"], "environment": { "NODE_OPTIONS": "--max_old_space_size=8192", }, "depends_on": [ - "compile-build-cmd", "yarn-install", - ], + ] + depends_on, "commands": cmds, } @@ -521,42 +480,41 @@ def build_plugins_step(ver_mode): env = None return { "name": "build-plugins", - "image": images["build_image"], + "image": images["node"], "environment": env, "depends_on": [ - "compile-build-cmd", "yarn-install", ], "commands": [ - # TODO: Use percentage for num jobs - "./bin/build build-plugins --jobs 8 --edition oss", + "apk add --update findutils", # Replaces the busybox 'find' with the GNU one. + "yarn plugins:build", ], } -def test_backend_step(image = images["build_image"]): +def test_backend_step(): return { "name": "test-backend", - "image": image, + "image": images["go"], "depends_on": [ "wire-install", ], "commands": [ + # shared-mime-info and shared-mime-info-lang is used for exactly 1 test for the + # mime.TypeByExtension function. + "apk add --update build-base shared-mime-info shared-mime-info-lang", "go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/...", ], } -def windows_test_backend_step(): - step = test_backend_step(image = windows_images["windows_go_image"]) - return step - def test_backend_integration_step(): return { "name": "test-backend-integration", - "image": images["build_image"], + "image": images["go"], "depends_on": [ "wire-install", ], "commands": [ + "apk add --update build-base", "go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\\(.*\\)/' | sort -u)", ], } @@ -570,11 +528,12 @@ def betterer_frontend_step(): return { "name": "betterer-frontend", - "image": images["build_image"], + "image": images["node"], "depends_on": [ "yarn-install", ], "commands": [ + "apk add --update git bash", "yarn betterer ci", ], } @@ -588,7 +547,7 @@ def test_frontend_step(): return { "name": "test-frontend", - "image": images["build_image"], + "image": images["node"], "environment": { "TEST_MAX_WORKERS": "50%", }, @@ -603,7 +562,7 @@ def test_frontend_step(): def lint_frontend_step(): return { "name": "lint-frontend", - "image": images["build_image"], + "image": images["node"], "environment": { "TEST_MAX_WORKERS": "50%", }, @@ -613,11 +572,35 @@ def lint_frontend_step(): "commands": [ "yarn run prettier:check", "yarn run lint", - "yarn run i18n:compile", # TODO: right place for this? "yarn run typecheck", ], } +def verify_i18n_step(): + extract_error_message = "\nExtraction failed. Make sure that you have no dynamic translation phrases, such as 't(\\`preferences.theme.\\$${themeID}\\`, themeName)' and that no translation key is used twice. Search the output for '[warning]' to find the offending file." + uncommited_error_message = "\nTranslation extraction has not been committed. Please run 'yarn i18n:extract', commit the changes and push again." + return { + "name": "verify-i18n", + "image": images["node"], + "depends_on": [ + "yarn-install", + ], + "commands": [ + "apk add --update git", + "yarn run i18n:extract || (echo \"{}\" && false)".format(extract_error_message), + # Verify that translation extraction has been committed + ''' + file_diff=$(git diff --dirstat public/locales) + if [ -n "$file_diff" ]; then + echo $file_diff + echo "{}" + exit 1 + fi + '''.format(uncommited_error_message), + "yarn run i18n:compile", + ], + } + def test_a11y_frontend_step(ver_mode, port = 3001): """Runs automated accessiblity tests against the frontend. @@ -651,7 +634,7 @@ def test_a11y_frontend_step(ver_mode, port = 3001): return { "name": "test-a11y-frontend", # TODO which image should be used? - "image": images["docker_puppeteer_image"], + "image": images["docker_puppeteer"], "depends_on": [ "grafana-server", ], @@ -676,7 +659,7 @@ def frontend_metrics_step(trigger = None): """ step = { "name": "publish-frontend-metrics", - "image": images["build_image"], + "image": images["node"], "depends_on": [ "test-a11y-frontend", ], @@ -685,7 +668,8 @@ def frontend_metrics_step(trigger = None): }, "failure": "ignore", "commands": [ - "./scripts/ci-frontend-metrics.sh | ./bin/build publish-metrics $${GRAFANA_MISC_STATS_API_KEY}", + "apk add --update bash grep git", + "./scripts/ci-frontend-metrics.sh ./grafana/public/build | ./bin/build publish-metrics $$GRAFANA_MISC_STATS_API_KEY", ], } if trigger: @@ -695,92 +679,38 @@ def frontend_metrics_step(trigger = None): def codespell_step(): return { "name": "codespell", - "image": images["build_image"], + "image": images["python"], "commands": [ + "pip3 install codespell", "codespell -I .codespellignore docs/", ], } -def package_step(ver_mode): - """Packages Grafana with the Grafana build tool. - - Args: - ver_mode: controls whether the packages are signed for a release. - If ver_mode != 'release', use the DRONE_BUILD_NUMBER environment - variable as a build identifier. - - Returns: - Drone step. - """ - deps = [ - "build-plugins", - "build-backend", - "build-frontend", - "build-frontend-packages", - ] - - if ver_mode in ("main", "release", "release-branch"): - sign_args = " --sign" - env = { - "GRAFANA_API_KEY": from_secret("grafana_api_key"), - "GPG_PRIV_KEY": from_secret("packages_gpg_private_key"), - "GPG_PUB_KEY": from_secret("packages_gpg_public_key"), - "GPG_KEY_PASSWORD": from_secret("packages_gpg_passphrase"), - } - test_args = "" - else: - sign_args = "" - env = None - - # TODO: env vars no longer needed by build if not signing - test_args = ". scripts/build/gpg-test-vars.sh && " - - # TODO: Use percentage for jobs - if ver_mode == "release": - cmds = [ - "{}./bin/build package --jobs 8 --edition oss ".format(test_args) + - "{} $${{DRONE_TAG}}".format(sign_args), - ] - else: - build_no = "${DRONE_BUILD_NUMBER}" - cmds = [ - "{}./bin/build package --jobs 8 --edition oss ".format(test_args) + - "--build-id {}{}".format(build_no, sign_args), - ] - - return { - "name": "package", - "image": images["build_image"], - "depends_on": deps, - "environment": env, - "commands": cmds, - } - -def grafana_server_step(port = 3001): +def grafana_server_step(): """Runs the grafana-server binary as a service. - Args: - port: port to listen on. - Defaults to 3001. - Returns: Drone step. """ - environment = {"PORT": port, "ARCH": "linux-amd64"} + environment = { + "GF_SERVER_HTTP_PORT": "3001", + "GF_SERVER_ROUTER_LOGGING": "1", + "GF_APP_MODE": "development", + } return { "name": "grafana-server", - "image": images["build_image"], + "image": images["alpine"], "detach": True, "depends_on": [ - "build-plugins", - "build-backend", - "build-frontend", - "build-frontend-packages", + "rgm-package", ], "environment": environment, "commands": [ - "./scripts/grafana-server/start-server", + "apk add --update tar bash", + "mkdir grafana", + "tar --strip-components=1 -xvf ./dist/*amd64.tar.gz -C grafana", + "cp -r devenv scripts tools grafana && cd grafana && ./scripts/grafana-server/start-server", ], } @@ -790,7 +720,7 @@ def e2e_tests_step(suite, port = 3001, tries = None): cmd += " --tries {}".format(tries) return { "name": "end-to-end-tests-{}".format(suite), - "image": images["cypress_image"], + "image": images["cypress"], "depends_on": [ "grafana-server", ], @@ -798,7 +728,6 @@ def e2e_tests_step(suite, port = 3001, tries = None): "HOST": "grafana-server", }, "commands": [ - "apt-get install -y netcat", cmd, ], } @@ -842,7 +771,7 @@ def cloud_plugins_e2e_tests_step(suite, cloud, trigger = None): branch = "${DRONE_SOURCE_BRANCH}".replace("/", "-") step = { "name": "end-to-end-tests-{}-{}".format(suite, cloud), - "image": images["cloud_datasources_e2e_image"], + "image": "us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:3.0.0", "depends_on": [ "grafana-server", ], @@ -856,73 +785,19 @@ def build_docs_website_step(): return { "name": "build-docs-website", # Use latest revision here, since we want to catch if it breaks - "image": images["docs_image"], + "image": images["docs"], "commands": [ "mkdir -p /hugo/content/docs/grafana/latest", + "echo -e '---\\nredirectURL: /docs/grafana/latest/\\ntype: redirect\\nversioned: true\\n---\\n' > /hugo/content/docs/grafana/_index.md", "cp -r docs/sources/* /hugo/content/docs/grafana/latest/", "cd /hugo && make prod", ], } -def copy_packages_for_docker_step(): - return { - "name": "copy-packages-for-docker", - "image": images["build_image"], - "depends_on": [ - "package", - ], - "commands": [ - "ls dist/*.tar.gz*", - "cp dist/*.tar.gz* packaging/docker/", - ], - } - -def build_docker_images_step(archs = None, ubuntu = False, publish = False): - """Build Docker images using the Grafana build tool. - - Args: - archs: a list of architectures to build the image for. - Defaults to None. - ubuntu: controls whether the final image is built from an Ubuntu base image. - Defaults to False. - publish: controls whether the built image is saved to a pre-release repository. - Defaults to False. - - Returns: - Drone step. - """ - cmd = "./bin/build build-docker --edition oss" - if publish: - cmd += " --shouldSave" - - ubuntu_sfx = "" - if ubuntu: - ubuntu_sfx = "-ubuntu" - cmd += " --ubuntu" - - if archs: - cmd += " -archs {}".format(",".join(archs)) - - environment = { - "GCP_KEY": from_secret(gcp_grafanauploads), - } - - return { - "name": "build-docker-images" + ubuntu_sfx, - "image": images["cloudsdk_image"], - "depends_on": [ - "copy-packages-for-docker", - "compile-build-cmd", - ], - "commands": [cmd], - "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], - "environment": environment, - } - def fetch_images_step(): return { "name": "fetch-images", - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "environment": { "GCP_KEY": from_secret(gcp_grafanauploads), "DOCKER_USER": from_secret("docker_username"), @@ -963,7 +838,7 @@ def publish_images_step(ver_mode, docker_repo, trigger = None): docker_repo, ) - deps = ["build-docker-images", "build-docker-images-ubuntu"] + deps = ["rgm-build-docker"] if ver_mode == "release": deps = ["fetch-images"] cmd += " --version-tag ${DRONE_TAG}" @@ -979,7 +854,7 @@ def publish_images_step(ver_mode, docker_repo, trigger = None): step = { "name": "publish-images-{}".format(name), - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "environment": environment, "commands": [cmd], "depends_on": deps, @@ -992,78 +867,126 @@ def publish_images_step(ver_mode, docker_repo, trigger = None): return step -def postgres_integration_tests_step(): +def integration_tests_steps(name, cmds, hostname = None, port = None, environment = None): + """Integration test steps + + Args: + name: the name of the step. + cmds: the commands to run to perform the integration tests. + hostname: the hostname where the remote server is available. + port: the port where the remote server is available. + environment: Any extra environment variables needed to run the integration tests. + + Returns: + A list of drone steps. If a hostname / port were provided, then a step to wait for the remove server to be + available is also returned. + """ + dockerize_name = "wait-for-{}".format(name) + + depends = [ + "wire-install", + ] + + step = { + "name": "{}-integration-tests".format(name), + "image": images["go"], + "depends_on": depends, + "commands": [ + "apk add --update build-base", + ] + cmds, + } + + if environment: + step["environment"] = environment + + if hostname == None: + return [step] + + depends = depends.append(dockerize_name) + + return [ + dockerize_step(dockerize_name, hostname, port), + step, + ] + +def integration_benchmarks_step(name, environment = None): cmds = [ - "apt-get update", - "apt-get install -yq postgresql-client", - "dockerize -wait tcp://postgres:5432 -timeout 120s", + "if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi", + "go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES}", + ] + + return integration_tests_steps("{}-benchmark".format(name), cmds, environment = environment) + +def postgres_integration_tests_steps(): + cmds = [ + "apk add --update postgresql-client", "psql -p 5432 -h postgres -U grafanatest -d grafanatest -f " + "devenv/docker/blocks/postgres_tests/setup.sql", "go clean -testcache", "go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\\(.*\\)/' | sort -u)", ] - return { - "name": "postgres-integration-tests", - "image": images["build_image"], - "depends_on": ["wire-install"], - "environment": { - "PGPASSWORD": "grafanatest", - "GRAFANA_TEST_DB": "postgres", - "POSTGRES_HOST": "postgres", - }, - "commands": cmds, + + environment = { + "PGPASSWORD": "grafanatest", + "GRAFANA_TEST_DB": "postgres", + "POSTGRES_HOST": "postgres", } -def mysql_integration_tests_step(): + return integration_tests_steps("postgres", cmds, "postgres", "5432", environment) + +def mysql_integration_tests_steps(hostname, version): cmds = [ - "apt-get update", - "apt-get install -yq default-mysql-client", - "dockerize -wait tcp://mysql:3306 -timeout 120s", - "cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root -prootpass", + "apk add --update mysql-client", + "cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h {} -P 3306 -u root -prootpass".format(hostname), "go clean -testcache", "go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\\(.*\\)/' | sort -u)", ] - return { - "name": "mysql-integration-tests", - "image": images["build_image"], - "depends_on": ["wire-install"], - "environment": { - "GRAFANA_TEST_DB": "mysql", - "MYSQL_HOST": "mysql", - }, - "commands": cmds, + + environment = { + "GRAFANA_TEST_DB": "mysql", + "MYSQL_HOST": hostname, } -def redis_integration_tests_step(): - return { - "name": "redis-integration-tests", - "image": images["build_image"], - "depends_on": ["wire-install"], - "environment": { - "REDIS_URL": "redis://redis:6379/0", - }, - "commands": [ - "dockerize -wait tcp://redis:6379/0 -timeout 120s", - "go clean -testcache", - "go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...", - ], + return integration_tests_steps("mysql-{}".format(version), cmds, hostname, "3306", environment) + +def redis_integration_tests_steps(): + cmds = [ + "go clean -testcache", + "go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...", + ] + + environment = { + "REDIS_URL": "redis://redis:6379/0", } -def memcached_integration_tests_step(): - return { - "name": "memcached-integration-tests", - "image": images["build_image"], - "depends_on": ["wire-install"], - "environment": { - "MEMCACHED_HOSTS": "memcached:11211", - }, - "commands": [ - "dockerize -wait tcp://memcached:11211 -timeout 120s", - "go clean -testcache", - "go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...", - ], + return integration_tests_steps("redis", cmds, "redis", "6379", environment = environment) + +def remote_alertmanager_integration_tests_steps(): + cmds = [ + "go clean -testcache", + "go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/...", + ] + + environment = { + "AM_TENANT_ID": "test", + "AM_PASSWORD": "test", + "AM_URL": "http://mimir_backend:8080", } + return integration_tests_steps("remote-alertmanager", cmds, "mimir_backend", "8080", environment = environment) + +def memcached_integration_tests_steps(): + cmds = [ + "go clean -testcache", + "go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...", + ] + + environment = { + "MEMCACHED_HOSTS": "memcached:11211", + } + + return integration_tests_steps("memcached", cmds, "memcached", "11211", environment) + def release_canary_npm_packages_step(trigger = None): """Releases canary NPM packages. @@ -1076,15 +999,17 @@ def release_canary_npm_packages_step(trigger = None): """ step = { "name": "release-canary-npm-packages", - "image": images["build_image"], - "depends_on": end_to_end_tests_deps(), + "image": images["node"], + "depends_on": end_to_end_tests_deps() + ["build-frontend-packages"], "environment": { "NPM_TOKEN": from_secret(npm_token), }, "commands": [ + "apk add --update bash", "./scripts/publish-npm-packages.sh --dist-tag 'canary' --registry 'https://registry.npmjs.org'", ], } + if trigger: step = dict( step, @@ -1097,6 +1022,7 @@ def release_canary_npm_packages_step(trigger = None): }, ), ) + return step def upload_packages_step(ver_mode, trigger = None): @@ -1113,7 +1039,7 @@ def upload_packages_step(ver_mode, trigger = None): """ step = { "name": "upload-packages", - "image": images["publish_image"], + "image": images["publish"], "depends_on": end_to_end_tests_deps(), "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), @@ -1134,7 +1060,7 @@ def publish_grafanacom_step(ver_mode): ver_mode: if ver_mode == 'main', pass the DRONE_BUILD_NUMBER environment variable as the value for the --build-id option. TODO: is this actually used by the grafanacom subcommand? I think it might - just use the environment varaiable directly. + just use the environment variable directly. Returns: Drone step. @@ -1151,7 +1077,7 @@ def publish_grafanacom_step(ver_mode): return { "name": "publish-grafanacom", - "image": images["publish_image"], + "image": images["publish"], "depends_on": [ "publish-linux-packages-deb", "publish-linux-packages-rpm", @@ -1169,7 +1095,7 @@ def publish_linux_packages_step(package_manager = "deb"): return { "name": "publish-linux-packages-{}".format(package_manager), # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image - "image": images["package_publish_image"], + "image": images["package_publish"], "depends_on": ["compile-build-cmd"], "privileged": True, "settings": { @@ -1187,122 +1113,15 @@ def publish_linux_packages_step(package_manager = "deb"): }, } -def windows_clone_step(): - return { - "name": "clone", - "image": windows_images["wix_image"], - "environment": { - "GITHUB_TOKEN": from_secret("github_token"), - }, - "commands": [ - 'git clone "https://$$env:GITHUB_TOKEN@github.com/$$env:DRONE_REPO.git" .', - "git checkout -f $$env:DRONE_COMMIT", - ], - } - -def get_windows_steps(ver_mode, bucket = "%PRERELEASE_BUCKET%"): - """Generate the list of Windows steps. - - Args: - ver_mode: used to differentiate steps for different version modes. - bucket: used to override prerelease bucket. - - Returns: - List of Drone steps. - """ - steps = [ - identify_runner_step("windows"), - ] - - init_cmds = [ - '$$ProgressPreference = "SilentlyContinue"', - "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( - grabpl_version, - ), - ] - - steps.extend( - [ - { - "name": "windows-init", - "image": windows_images["wix_image"], - "commands": init_cmds, - }, - ], - ) - - if ver_mode in ( - "release", - "release-branch", - "main", - ): - gcp_bucket = "{}/artifacts/downloads".format(bucket) - if ver_mode == "release": - ver_part = "${DRONE_TAG}" - dir = "release" - else: - dir = "main" - gcp_bucket = "grafana-downloads" - build_no = "DRONE_BUILD_NUMBER" - ver_part = "--build-id $$env:{}".format(build_no) - installer_commands = [ - "$$gcpKey = $$env:GCP_KEY", - "[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json", - # gcloud fails to read the file unless converted with dos2unix - "dos2unix gcpkey.json", - "gcloud auth activate-service-account --key-file=gcpkey.json", - "rm gcpkey.json", - "cp C:\\App\\nssm-2.24.zip .", - ] - - if ver_mode == "release": - version = "${DRONE_TAG:1}" - installer_commands.extend( - [ - ".\\grabpl.exe windows-installer --target {} --edition oss {}".format("gs://{}/{}/oss/{}/grafana-{}.windows-amd64.zip".format(gcp_bucket, ver_part, ver_mode, version), ver_part), - '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', - 'gsutil cp "$$fname" gs://{}/{}/oss/{}/'.format(gcp_bucket, ver_part, dir), - 'gsutil cp "$$fname.sha256" gs://{}/{}/oss/{}/'.format(gcp_bucket, ver_part, dir), - ], - ) - if ver_mode in ("main"): - installer_commands.extend( - [ - ".\\grabpl.exe windows-installer --edition oss --build-id $$env:DRONE_BUILD_NUMBER", - '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', - 'gsutil cp "$$fname" gs://{}/oss/{}/'.format(gcp_bucket, dir), - 'gsutil cp "$$fname.sha256" gs://{}/oss/{}/'.format( - gcp_bucket, - dir, - ), - ], - ) - steps.append( - { - "name": "build-windows-installer", - "image": windows_images["wix_image"], - "depends_on": [ - "windows-init", - ], - "environment": { - "GCP_KEY": from_secret(gcp_grafanauploads_base64), - "PRERELEASE_BUCKET": from_secret(prerelease_bucket), - "GITHUB_TOKEN": from_secret("github_token"), - }, - "commands": installer_commands, - }, - ) - - return steps - def verify_gen_cue_step(): return { "name": "verify-gen-cue", - "image": images["build_image"], + "image": images["go"], "depends_on": [], "commands": [ "# It is required that code generated from Thema/CUE be committed and in sync with its inputs.", "# The following command will fail if running code generators produces any diff in output.", + "apk add --update make", "CODEGEN_VERIFY=1 make gen-cue", ], } @@ -1310,11 +1129,12 @@ def verify_gen_cue_step(): def verify_gen_jsonnet_step(): return { "name": "verify-gen-jsonnet", - "image": images["build_image"], + "image": images["go"], "depends_on": [], "commands": [ "# It is required that generated jsonnet is committed and in sync with its inputs.", "# The following command will fail if running code generators produces any diff in output.", + "apk add --update make", "CODEGEN_VERIFY=1 make gen-jsonnet", ], } @@ -1332,7 +1152,7 @@ def compile_build_cmd(): return { "name": "compile-build-cmd", - "image": images["go_image"], + "image": images["go"], "commands": [ "go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd", ], @@ -1363,3 +1183,14 @@ def get_trigger_storybook(ver_mode): }, } return trigger_storybook + +def slack_step(channel, template, secret): + return { + "name": "slack", + "image": images["plugins_slack"], + "settings": { + "webhook": from_secret(secret), + "channel": channel, + "template": template, + }, + } diff --git a/scripts/drone/steps/lib_windows.star b/scripts/drone/steps/lib_windows.star new file mode 100644 index 00000000000..f9b11bcf85e --- /dev/null +++ b/scripts/drone/steps/lib_windows.star @@ -0,0 +1,187 @@ +""" +This module is a library of Drone steps that exclusively run on windows machines. +""" + +load( + "scripts/drone/utils/windows_images.star", + "windows_images", +) +load( + "scripts/drone/variables.star", + "grabpl_version", +) +load( + "scripts/drone/vault.star", + "from_secret", + "gcp_grafanauploads_base64", + "prerelease_bucket", +) + +def identify_runner_step_windows(): + return { + "name": "identify-runner", + "image": windows_images["1809"], + "commands": [ + "echo $env:DRONE_RUNNER_NAME", + ], + } + +def get_windows_steps(ver_mode, bucket = "%PRERELEASE_BUCKET%"): + """Generate the list of Windows steps. + + Args: + ver_mode: used to differentiate steps for different version modes. + bucket: used to override prerelease bucket. + + Returns: + List of Drone steps. + """ + steps = [ + identify_runner_step_windows(), + ] + + init_cmds = [ + '$$ProgressPreference = "SilentlyContinue"', + "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( + grabpl_version, + ), + ] + + steps.extend( + [ + { + "name": "windows-init", + "image": windows_images["wix"], + "commands": init_cmds, + }, + ], + ) + + if ver_mode in ( + "release", + "release-branch", + ): + gcp_bucket = "{}/artifacts/downloads".format(bucket) + if ver_mode == "release": + ver_part = "${DRONE_TAG}" + dir = "release" + else: + dir = "main" + gcp_bucket = "grafana-downloads" + build_no = "DRONE_BUILD_NUMBER" + ver_part = "--build-id $$env:{}".format(build_no) + installer_commands = [ + "$$gcpKey = $$env:GCP_KEY", + "[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json", + # gcloud fails to read the file unless converted with dos2unix + "dos2unix gcpkey.json", + "gcloud auth activate-service-account --key-file=gcpkey.json", + "rm gcpkey.json", + "cp C:\\App\\nssm-2.24.zip .", + ] + + if ver_mode in ("release",): + version = "${DRONE_TAG:1}" + installer_commands.extend( + [ + ".\\grabpl.exe windows-installer --target {} --edition oss {}".format( + "gs://{}/{}/oss/{}/grafana-{}.windows-amd64.zip".format(gcp_bucket, ver_part, ver_mode, version), + ver_part, + ), + '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', + ], + ) + if ver_mode == "main": + installer_commands.extend( + [ + "gsutil cp $$fname gs://{}/oss/{}/".format(gcp_bucket, dir), + 'gsutil cp "$$fname.sha256" gs://{}/oss/{}/'.format( + gcp_bucket, + dir, + ), + ], + ) + else: + installer_commands.extend( + [ + "gsutil cp $$fname gs://{}/{}/oss/{}/".format( + gcp_bucket, + ver_part, + dir, + ), + 'gsutil cp "$$fname.sha256" gs://{}/{}/oss/{}/'.format( + gcp_bucket, + ver_part, + dir, + ), + ], + ) + steps.append( + { + "name": "build-windows-installer", + "image": windows_images["wix"], + "depends_on": [ + "windows-init", + ], + "environment": { + "GCP_KEY": from_secret(gcp_grafanauploads_base64), + "PRERELEASE_BUCKET": from_secret(prerelease_bucket), + "GITHUB_TOKEN": from_secret("github_token"), + }, + "commands": installer_commands, + }, + ) + + return steps + +def download_grabpl_step_windows(): + return { + "name": "grabpl", + "image": windows_images["wix"], + "commands": [ + '$$ProgressPreference = "SilentlyContinue"', + "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( + grabpl_version, + ), + ], + } + +def test_backend_step_windows(): + # TODO: This is mostly a duplicate of "test_backend_step" in lib.star; but this file can't import that one, + # otherwise it creates an import cycle. + return { + "name": "test-backend", + "image": windows_images["go"], + "depends_on": [ + "wire-install", + ], + "commands": [ + "go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/...", + ], + } + +def clone_step_windows(): + return { + "name": "clone", + "image": windows_images["wix"], + "environment": { + "GITHUB_TOKEN": from_secret("github_token"), + }, + "commands": [ + 'git clone "https://$$env:GITHUB_TOKEN@github.com/$$env:DRONE_REPO.git" .', + "git checkout -f $$env:DRONE_COMMIT", + ], + } + +def wire_install_step_windows(edition): + return { + "name": "wire-install", + "image": windows_images["go"], + "commands": [ + "go install github.com/google/wire/cmd/wire@v0.5.0", + "wire gen -tags {} ./pkg/server".format(edition), + ], + "depends_on": [ + "windows-init", + ], + } diff --git a/scripts/drone/steps/rgm.star b/scripts/drone/steps/rgm.star new file mode 100644 index 00000000000..495b443e4a0 --- /dev/null +++ b/scripts/drone/steps/rgm.star @@ -0,0 +1,61 @@ +""" +Individual steps that use 'grafana-build' to replace existing individual steps. +These aren't used in releases. +""" + +load( + "scripts/drone/variables.star", + "golang_version", +) + +# rgm_package_step will create a tar.gz for use in e2e tests or other PR testing related activities.. +def rgm_package_step(distros = "linux/amd64,linux/arm64", file = "packages.txt"): + return { + "name": "rgm-package", + "image": "grafana/grafana-build:main", + "pull": "always", + "depends_on": ["yarn-install"], + "commands": [ + "/src/grafana-build package --distro={} ".format(distros) + + "--go-version={} ".format(golang_version) + + "--yarn-cache=$$YARN_CACHE_FOLDER " + + "--build-id=$$DRONE_BUILD_NUMBER " + + "--grafana-dir=$$PWD > {}".format(file), + ], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + } + +# rgm_build_backend will create compile the grafana backend for various platforms. It's preferred to use +# 'rgm_package_step' if you creating a "usable" artifact. This should really only be used to verify that the code is +# compilable. +def rgm_build_backend_step(distros = "linux/amd64,linux/arm64"): + return { + "name": "rgm-package", + "image": "grafana/grafana-build:main", + "pull": "always", + "commands": [ + "/src/grafana-build build " + + "--go-version={} ".format(golang_version) + + "--distro={} --grafana-dir=$$PWD".format(distros), + ], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + } + +def rgm_build_docker_step(packages, ubuntu, alpine, depends_on = ["rgm-package"], file = "docker.txt", tag_format = "{{ .version }}-{{ .arch }}", ubuntu_tag_format = "{{ .version }}-ubuntu-{{ .arch }}"): + return { + "name": "rgm-build-docker", + "image": "grafana/grafana-build:main", + "pull": "always", + "commands": [ + "docker run --privileged --rm tonistiigi/binfmt --install all", + "/src/grafana-build docker " + + "$(cat {} | grep tar.gz | grep -v docker | grep -v sha256 | awk '{{print \"--package=\" $0}}') ".format(packages) + + "--ubuntu-base={} ".format(ubuntu) + + "--alpine-base={} ".format(alpine) + + "--tag-format='{}' ".format(tag_format) + + "--ubuntu-tag-format='{}' > {}".format(ubuntu_tag_format, file), + "find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i", + ], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + "depends_on": depends_on, + } diff --git a/scripts/drone/utils/images.star b/scripts/drone/utils/images.star index b621583366a..7f5b40c0e1e 100644 --- a/scripts/drone/utils/images.star +++ b/scripts/drone/utils/images.star @@ -2,24 +2,35 @@ This module contains all the docker images that are used to build test and publish Grafana. """ +load( + "scripts/drone/variables.star", + "golang_version", + "nodejs_version", +) + images = { - "cloudsdk_image": "google/cloud-sdk:431.0.0", - "build_image": "grafana/build-container:1.7.5", - "publish_image": "grafana/grafana-ci-deploy:1.3.3", - "alpine_image": "alpine:3.18.3", - "curl_image": "byrnedo/alpine-curl:0.1.8", - "go_image": "golang:1.20.10", - "plugins_slack_image": "plugins/slack", - "postgres_alpine_image": "postgres:12.3-alpine", - "mysql5_image": "mysql:5.7.39", - "mysql8_image": "mysql:8.0.32", - "redis_alpine_image": "redis:6.2.11-alpine", - "memcached_alpine_image": "memcached:1.6.9-alpine", - "package_publish_image": "us.gcr.io/kubernetes-dev/package-publish:latest", - "openldap_image": "osixia/openldap:1.4.0", - "drone_downstream_image": "grafana/drone-downstream", - "docker_puppeteer_image": "grafana/docker-puppeteer:1.1.0", - "docs_image": "grafana/docs-base:dbd975af06", - "cypress_image": "cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97", - "cloud_datasources_e2e_image": "us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest", + "git": "alpine/git:2.40.1", + "go": "golang:{}-alpine".format(golang_version), + "node": "node:{}-alpine".format(nodejs_version), + "cloudsdk": "google/cloud-sdk:431.0.0", + "publish": "grafana/grafana-ci-deploy:1.3.3", + "alpine": "alpine:3.18.3", + "ubuntu": "ubuntu:22.04", + "curl": "byrnedo/alpine-curl:0.1.8", + "plugins_slack": "plugins/slack", + "python": "python:3.8", + "postgres_alpine": "postgres:12.3-alpine", + "mimir": "grafana/mimir:latest", + "mysql5": "mysql:5.7.39", + "mysql8": "mysql:8.0.32", + "redis_alpine": "redis:6.2.11-alpine", + "memcached_alpine": "memcached:1.6.9-alpine", + "package_publish": "us.gcr.io/kubernetes-dev/package-publish:latest", + "openldap": "osixia/openldap:1.4.0", + "drone_downstream": "grafana/drone-downstream", + "docker_puppeteer": "grafana/docker-puppeteer:1.1.0", + "docs": "grafana/docs-base:dbd975af06", + "cypress": "cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97", + "dockerize": "jwilder/dockerize:0.6.1", + "shellcheck": "koalaman/shellcheck:stable", } diff --git a/scripts/drone/utils/utils.star b/scripts/drone/utils/utils.star index 63961a1d3ca..aea832018cf 100644 --- a/scripts/drone/utils/utils.star +++ b/scripts/drone/utils/utils.star @@ -9,7 +9,6 @@ load( load("scripts/drone/vault.star", "pull_secret") failure_template = "Build {{build.number}} failed for commit: : {{build.link}}\nBranch: \nAuthor: {{build.author}}" -drone_change_template = "`.drone.yml` and `starlark` files have been changed on the OSS repo, by: {{build.author}}. \nBranch: \nCommit hash: " def pipeline( name, @@ -31,7 +30,7 @@ def pipeline( name: controls the pipeline name. trigger: a Drone trigger for the pipeline. steps: the Drone steps for the pipeline. - services: auxilliary services used during the pipeline. + services: auxiliary services used during the pipeline. Defaults to []. platform: abstracts platform specific configuration primarily for different Drone behavior on Windows. Defaults to 'linux'. diff --git a/scripts/drone/utils/windows_images.star b/scripts/drone/utils/windows_images.star index 2a605727ba8..a25c637a11e 100644 --- a/scripts/drone/utils/windows_images.star +++ b/scripts/drone/utils/windows_images.star @@ -4,9 +4,14 @@ All the windows images needed to be in a different file than the other images, s by trivy. Related issue: https://github.com/aquasecurity/trivy/issues/1392 """ +load( + "scripts/drone/variables.star", + "golang_version", +) + windows_images = { - "1809_image": "mcr.microsoft.com/windows:1809", - "wix_image": "grafana/ci-wix:0.1.1", - "windows_server_core_image": "docker:windowsservercore-1809", - "windows_go_image": "grafana/grafana-ci-windows-test:0.1.0", + "1809": "mcr.microsoft.com/windows:1809", + "wix": "grafana/ci-wix:0.1.1", + "windows_server_core": "docker:windowsservercore-1809", + "go": "golang:{}-windowsservercore-1809".format(golang_version), } diff --git a/scripts/drone/variables.star b/scripts/drone/variables.star new file mode 100644 index 00000000000..f4d1b97b322 --- /dev/null +++ b/scripts/drone/variables.star @@ -0,0 +1,9 @@ +""" +global variables +""" + +grabpl_version = "v3.0.42" +golang_version = "1.20.10" + +# nodejs_version should match what's in ".nvmrc", but without the v prefix. +nodejs_version = "18.12.0" diff --git a/scripts/drone/vault.star b/scripts/drone/vault.star index 236ed39e9b7..6a27cf51785 100644 --- a/scripts/drone/vault.star +++ b/scripts/drone/vault.star @@ -14,6 +14,9 @@ azure_tenant = "azure_tenant" rgm_gcp_key_base64 = "gcp_key_base64" rgm_destination = "destination" +rgm_storybook_destination = "rgm_storybook_destination" +rgm_cdn_destination = "rgm_cdn_destination" +rgm_downloads_destination = "rgm_downloads_destination" rgm_github_token = "github_token" rgm_dagger_token = "dagger_token" @@ -107,11 +110,6 @@ def secrets(): "infra/data/ci/packages-publish/bucket-credentials", "Secret", ), - vault_secret( - "security_dest_bucket", - "infra/data/ci/grafana-release-eng/security-bucket", - "bucket", - ), vault_secret( "static_asset_editions", "infra/data/ci/grafana-release-eng/artifact-publishing", @@ -127,6 +125,21 @@ def secrets(): "infra/data/ci/grafana-release-eng/rgm", "destination_prod", ), + vault_secret( + rgm_storybook_destination, + "infra/data/ci/grafana-release-eng/rgm", + "storybook_destination", + ), + vault_secret( + rgm_cdn_destination, + "infra/data/ci/grafana-release-eng/rgm", + "cdn_destination", + ), + vault_secret( + rgm_downloads_destination, + "infra/data/ci/grafana-release-eng/rgm", + "downloads_destination", + ), vault_secret( rgm_dagger_token, "infra/data/ci/grafana-release-eng/rgm", diff --git a/scripts/drone/version.star b/scripts/drone/version.star deleted file mode 100644 index 140cd4d35d8..00000000000 --- a/scripts/drone/version.star +++ /dev/null @@ -1,16 +0,0 @@ -""" -This module returns the pipeline used for version branches. -""" - -load( - "scripts/drone/events/release.star", - "oss_pipelines", -) - -ver_mode = "release-branch" -trigger = {"ref": ["refs/heads/v[0-9]*"]} - -def version_branch_pipelines(): - return ( - oss_pipelines(ver_mode = ver_mode, trigger = trigger) - ) diff --git a/scripts/validate-npm-packages.sh b/scripts/validate-npm-packages.sh index a07ebe5a662..4e2a27184fa 100755 --- a/scripts/validate-npm-packages.sh +++ b/scripts/validate-npm-packages.sh @@ -8,7 +8,7 @@ ARTIFACTS_DIR="./npm-artifacts" for file in "$ARTIFACTS_DIR"/*.tgz; do echo "🔍 Checking NPM package: $file" # get filename then strip everything after package name. - dir_name=$(basename "$file" .tgz | sed 's/^@\(.*\)-[0-9]*[.]*[0-9]*[.]*[0-9]*-\([0-9]*[a-zA-Z]*\)/\1/') + dir_name=$(basename "$file" .tgz | sed -E 's/@([a-zA-Z0-9-]+)-[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9-]+)?/\1/') mkdir -p "./npm-artifacts/$dir_name" tar -xzf "$file" -C "./npm-artifacts/$dir_name" --strip-components=1