diff --git a/.drone.star b/.drone.star index ce31860e844..a14146a7c14 100644 --- a/.drone.star +++ b/.drone.star @@ -7,8 +7,9 @@ This module returns a Drone configuration including pipelines and secrets. """ -load("scripts/drone/events/pr.star", "pr_pipelines") +load("scripts/drone/events/cron.star", "cronjobs") load("scripts/drone/events/main.star", "main_pipelines") +load("scripts/drone/events/pr.star", "pr_pipelines") load( "scripts/drone/events/release.star", "integration_test_pipelines", @@ -17,24 +18,22 @@ load( "publish_packages_pipeline", ) load( - "scripts/drone/rgm.star", - "rgm", + "scripts/drone/pipelines/ci_images.star", + "publish_ci_build_container_image_pipeline", + "publish_ci_windows_test_image_pipeline", ) load( "scripts/drone/pipelines/publish_images.star", "publish_image_pipelines_public", ) -load( - "scripts/drone/pipelines/ci_images.star", - "publish_ci_build_container_image_pipeline", - "publish_ci_windows_test_image_pipeline", -) load( "scripts/drone/pipelines/windows.star", "windows_test_backend", ) -load("scripts/drone/version.star", "version_branch_pipelines") -load("scripts/drone/events/cron.star", "cronjobs") +load( + "scripts/drone/rgm.star", + "rgm", +) load("scripts/drone/vault.star", "secrets") def main(_ctx): @@ -50,7 +49,6 @@ def main(_ctx): "event": ["promote"], "target": ["test-windows"], }, "oss", "testing")] + - version_branch_pipelines() + integration_test_pipelines() + publish_ci_windows_test_image_pipeline() + publish_ci_build_container_image_pipeline() + diff --git a/.drone.yml b/.drone.yml index f514da16ff1..0c823335397 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,7 +24,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - ./bin/build verify-drone @@ -74,13 +74,14 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - - ./bin/build verify-starlark . + - go install github.com/bazelbuild/buildtools/buildifier@latest + - buildifier --lint=warn -mode=check -r . depends_on: - compile-build-cmd - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: lint-starlark trigger: event: @@ -118,24 +119,20 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: + - apk add --update git bash - yarn betterer ci depends_on: - yarn-install - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: betterer-frontend - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -153,7 +150,7 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - yarn run ci:test-frontend @@ -161,7 +158,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: test-frontend trigger: event: @@ -198,6 +195,7 @@ platform: services: [] steps: - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -215,7 +213,7 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - echo $DRONE_RUNNER_NAME @@ -224,7 +222,7 @@ steps: - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: - yarn run prettier:check @@ -234,9 +232,10 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: lint-frontend - commands: + - apk add --update git - |- yarn run i18n:extract || (echo " Extraction failed. Make sure that you have no dynamic translation phrases, such as 't(\`preferences.theme.\$${themeID}\`, themeName)' and that no translation key is used twice. Search the output for '[warning]' to find the offending file." && false) @@ -248,7 +247,7 @@ steps: - yarn run i18n:compile depends_on: - yarn-install - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: verify-i18n trigger: event: @@ -285,6 +284,7 @@ platform: services: [] steps: - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -302,55 +302,53 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: + - apk add --update build-base shared-mime-info shared-mime-info-lang - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: test-backend - commands: + - apk add --update build-base - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: test-backend-integration trigger: event: @@ -399,9 +397,10 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -419,26 +418,26 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: + - apk add --update make - make gen-go depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: - - apt-get update && apt-get install make + - apk add --update make build-base - make lint-go depends_on: - wire-install environment: CGO_ENABLED: "1" - image: golang:1.20.10 + image: golang:1.20.10-alpine name: lint-backend - commands: - go run scripts/modowners/modowners.go check go.mod - failure: ignore - image: golang:1.20.10 + image: golang:1.20.10-alpine name: validate-modfile trigger: event: @@ -485,7 +484,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -494,36 +493,50 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install +- commands: + - apk add --update jq bash + - yarn packages:build + - yarn packages:pack + - ./scripts/validate-npm-packages.sh + depends_on: + - yarn-install + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: node:18.12.0-alpine + name: build-frontend-packages - failure: ignore image: grafana/drone-downstream name: trigger-enterprise-downstream @@ -538,66 +551,32 @@ steps: token: from_secret: drone_token - commands: - - ./bin/build build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - /src/grafana-build package --distro=linux/amd64,linux/arm64 --go-version=1.20.10 + --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD + > packages.txt depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.7.5 - name: build-backend -- commands: - - ./bin/build build-frontend --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 - name: build-frontend + image: grafana/grafana-build:main + name: rgm-package + pull: always + volumes: + - name: docker + path: /var/run/docker.sock - commands: - - ./bin/build build-frontend-packages --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - - yarn packages:pack - - ./scripts/validate-npm-packages.sh + - apk add --update tar bash + - mkdir grafana + - tar --strip-components=1 -xvf ./dist/*amd64.tar.gz -C grafana + - cp -r devenv scripts tools grafana && cd grafana && ./scripts/grafana-server/start-server depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 - name: build-frontend-packages -- commands: - - ./bin/build build-plugins --jobs 8 --edition oss - depends_on: - - compile-build-cmd - - yarn-install - environment: null - image: grafana/build-container:1.7.5 - name: build-plugins -- commands: - - . scripts/build/gpg-test-vars.sh && ./bin/build package --jobs 8 --edition oss - --build-id ${DRONE_BUILD_NUMBER} - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - environment: null - image: grafana/build-container:1.7.5 - name: package -- commands: - - ./scripts/grafana-server/start-server - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages + - rgm-package detach: true environment: - ARCH: linux-amd64 - PORT: 3001 - image: grafana/build-container:1.7.5 + GF_APP_MODE: development + GF_SERVER_HTTP_PORT: "3001" + GF_SERVER_ROUTER_LOGGING: "1" + image: alpine:3.18.3 name: grafana-server - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite dashboards-suite depends_on: - grafana-server @@ -606,7 +585,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-dashboards-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite depends_on: - grafana-server @@ -615,7 +593,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-smoke-tests-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite panels-suite depends_on: - grafana-server @@ -624,7 +601,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-panels-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite various-suite depends_on: - grafana-server @@ -648,7 +624,7 @@ steps: GITHUB_TOKEN: from_secret: github_token HOST: grafana-server - image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:3.0.0 name: end-to-end-tests-cloud-plugins-suite-azure when: paths: @@ -693,23 +669,16 @@ steps: - yarn storybook:build - ./bin/build verify-storybook depends_on: - - build-frontend + - rgm-package - build-frontend-packages environment: NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: build-storybook when: paths: include: - packages/grafana-ui/** -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.7.5 - name: copy-packages-for-docker - commands: - yarn wait-on http://$HOST:$PORT - pa11y-ci --config .pa11yci-pr.conf.js @@ -724,36 +693,24 @@ steps: image: grafana/docker-puppeteer:1.1.0 name: test-a11y-frontend - commands: - - ./bin/build build-docker --edition oss -archs amd64 + - docker run --privileged --rm tonistiigi/binfmt --install all + - /src/grafana-build docker $(cat packages.txt | grep tar.gz | grep -v docker | + grep -v sha256 | awk '{print "--package=" $0}') --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.18.3 + --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --ubuntu-tag-format='{{ + .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > docker.txt + - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build build-docker --edition oss --ubuntu -archs amd64 - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images-ubuntu + - rgm-package + image: grafana/grafana-build:main + name: rgm-build-docker + pull: always volumes: - name: docker path: /var/run/docker.sock - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana depends_on: - - build-docker-images - - build-docker-images-ubuntu + - rgm-build-docker environment: DOCKER_PASSWORD: from_secret: docker_password @@ -834,6 +791,11 @@ services: volumes: - name: mysql80 path: /var/lib/mysql +- commands: + - /bin/mimir -target=backend + environment: {} + image: grafana/mimir:latest + name: mimir_backend - environment: {} image: redis:6.2.11-alpine name: redis @@ -842,6 +804,7 @@ services: name: memcached steps: - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -859,11 +822,11 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -872,7 +835,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME @@ -883,28 +846,34 @@ steps: with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: - - apt-get update - - apt-get install -yq postgresql-client - dockerize -wait tcp://postgres:5432 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-postgres +- commands: + - apk add --update build-base + - apk add --update postgresql-client - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find @@ -912,16 +881,20 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-postgres environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: postgres-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - dockerize -wait tcp://mysql57:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-5.7 +- commands: + - apk add --update build-base + - apk add --update mysql-client - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql57 -P 3306 -u root -prootpass - go clean -testcache @@ -930,15 +903,19 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-5.7 environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql57 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: mysql-5.7-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - dockerize -wait tcp://mysql80:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-8.0 +- commands: + - apk add --update build-base + - apk add --update mysql-client - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql80 -P 3306 -u root -prootpass - go clean -testcache @@ -947,31 +924,59 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-8.0 environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql80 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: mysql-8.0-integration-tests - commands: - - dockerize -wait tcp://redis:6379/0 -timeout 120s + - dockerize -wait tcp://redis:6379 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-redis +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-redis environment: REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: redis-integration-tests - commands: - dockerize -wait tcp://memcached:11211 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-memcached +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-memcached environment: MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: memcached-integration-tests +- commands: + - dockerize -wait tcp://mimir_backend:8080 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-remote-alertmanager +- commands: + - apk add --update build-base + - go clean -testcache + - go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/... + depends_on: + - wire-install + - wait-for-remote-alertmanager + environment: + AM_PASSWORD: test + AM_TENANT_ID: test + AM_URL: http://mimir_backend:8080 + image: golang:1.20.10-alpine + name: remote-alertmanager-integration-tests trigger: event: - pull_request @@ -1025,11 +1030,12 @@ steps: - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: + - pip3 install codespell - codespell -I .codespellignore docs/ - image: grafana/build-container:1.7.5 + image: python:3.8 name: codespell - commands: - yarn run prettier:checkDocs @@ -1037,10 +1043,12 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: lint-docs - commands: - mkdir -p /hugo/content/docs/grafana/latest + - 'echo -e ''---\nredirectURL: /docs/grafana/latest/\ntype: redirect\nversioned: + true\n---\n'' > /hugo/content/docs/grafana/_index.md' - cp -r docs/sources/* /hugo/content/docs/grafana/latest/ - cd /hugo && make prod image: grafana/docs-base:dbd975af06 @@ -1050,9 +1058,10 @@ steps: with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue trigger: event: @@ -1092,13 +1101,12 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - - ./bin/build shellcheck - depends_on: - - compile-build-cmd - image: grafana/build-container:1.7.5 + - apt-get update -yq && apt-get install shellcheck + - shellcheck -e SC1071 -e SC2162 scripts/**/*.sh + image: ubuntu:22.04 name: shellcheck trigger: event: @@ -1165,6 +1173,11 @@ services: volumes: - name: mysql80 path: /var/lib/mysql +- commands: + - /bin/mimir -target=backend + environment: {} + image: grafana/mimir:latest + name: mimir_backend - environment: {} image: redis:6.2.11-alpine name: redis @@ -1173,6 +1186,7 @@ services: name: memcached steps: - commands: + - apk add --update curl jq bash - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" ../grafana-enterprise - cd ../grafana-enterprise @@ -1187,48 +1201,53 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.5 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: - clone-enterprise - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: - clone-enterprise - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: + - apk add --update build-base - if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: sqlite-benchmark-integration-tests - commands: + - apk add --update build-base - if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} depends_on: @@ -1237,9 +1256,10 @@ steps: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: postgres-benchmark-integration-tests - commands: + - apk add --update build-base - if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} depends_on: @@ -1247,9 +1267,10 @@ steps: environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql57 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: mysql-5.7-benchmark-integration-tests - commands: + - apk add --update build-base - if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} depends_on: @@ -1257,7 +1278,7 @@ steps: environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql80 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: mysql-8.0-benchmark-integration-tests trigger: event: @@ -1302,11 +1323,12 @@ steps: - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: + - pip3 install codespell - codespell -I .codespellignore docs/ - image: grafana/build-container:1.7.5 + image: python:3.8 name: codespell - commands: - yarn run prettier:checkDocs @@ -1314,10 +1336,12 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: lint-docs - commands: - mkdir -p /hugo/content/docs/grafana/latest + - 'echo -e ''---\nredirectURL: /docs/grafana/latest/\ntype: redirect\nversioned: + true\n---\n'' > /hugo/content/docs/grafana/_index.md' - cp -r docs/sources/* /hugo/content/docs/grafana/latest/ - cd /hugo && make prod image: grafana/docs-base:dbd975af06 @@ -1327,9 +1351,10 @@ steps: with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue trigger: branch: main @@ -1369,22 +1394,17 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: + - apk add --update git bash - yarn betterer ci depends_on: - yarn-install - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: betterer-frontend - commands: - yarn run ci:test-frontend @@ -1392,7 +1412,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: test-frontend trigger: branch: main @@ -1434,7 +1454,7 @@ steps: - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: - yarn run prettier:check @@ -1444,9 +1464,10 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: lint-frontend - commands: + - apk add --update git - |- yarn run i18n:extract || (echo " Extraction failed. Make sure that you have no dynamic translation phrases, such as 't(\`preferences.theme.\$${themeID}\`, themeName)' and that no translation key is used twice. Search the output for '[warning]' to find the offending file." && false) @@ -1458,7 +1479,7 @@ steps: - yarn run i18n:compile depends_on: - yarn-install - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: verify-i18n trigger: branch: main @@ -1497,49 +1518,47 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: + - apk add --update build-base shared-mime-info shared-mime-info-lang - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: test-backend - commands: + - apk add --update build-base - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: test-backend-integration trigger: branch: main @@ -1583,26 +1602,26 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: + - apk add --update make - make gen-go depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: - - apt-get update && apt-get install make + - apk add --update make build-base - make lint-go depends_on: - wire-install environment: CGO_ENABLED: "1" - image: golang:1.20.10 + image: golang:1.20.10-alpine name: lint-backend - commands: - go run scripts/modowners/modowners.go check go.mod - failure: ignore - image: golang:1.20.10 + image: golang:1.20.10-alpine name: validate-modfile - commands: - ./bin/build verify-drone @@ -1649,7 +1668,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1658,106 +1677,89 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: - - ./bin/build build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - apk add --update jq + - new_version=$(cat package.json | jq .version | sed s/pre/${DRONE_BUILD_NUMBER}/g) + - 'echo "New version: $new_version"' + - yarn run lerna version $new_version --exact --no-git-tag-version --no-push --force-publish + -y + - yarn install --mode=update-lockfile depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.7.5 - name: build-backend -- commands: - - ./bin/build build-frontend --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 - name: build-frontend + image: node:18.12.0-alpine + name: update-package-json-version - commands: - - ./bin/build build-frontend-packages --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} + - apk add --update jq bash + - yarn packages:build - yarn packages:pack - ./scripts/validate-npm-packages.sh depends_on: - - compile-build-cmd - yarn-install + - update-package-json-version environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: build-frontend-packages - commands: - - ./bin/build build-plugins --jobs 8 --edition oss + - /src/grafana-build package --distro=linux/amd64,linux/arm64 --go-version=1.20.10 + --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD + > packages.txt depends_on: - - compile-build-cmd - yarn-install - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.5 - name: build-plugins + image: grafana/grafana-build:main + name: rgm-package + pull: always + volumes: + - name: docker + path: /var/run/docker.sock - commands: - - ./bin/build package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --sign + - apk add --update tar bash + - mkdir grafana + - tar --strip-components=1 -xvf ./dist/*amd64.tar.gz -C grafana + - cp -r devenv scripts tools grafana && cd grafana && ./scripts/grafana-server/start-server depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - environment: - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.5 - name: package -- commands: - - ./scripts/grafana-server/start-server - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages + - rgm-package detach: true environment: - ARCH: linux-amd64 - PORT: 3001 - image: grafana/build-container:1.7.5 + GF_APP_MODE: development + GF_SERVER_HTTP_PORT: "3001" + GF_SERVER_ROUTER_LOGGING: "1" + image: alpine:3.18.3 name: grafana-server - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite dashboards-suite depends_on: - grafana-server @@ -1766,7 +1768,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-dashboards-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite depends_on: - grafana-server @@ -1775,7 +1776,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-smoke-tests-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite panels-suite depends_on: - grafana-server @@ -1784,7 +1784,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-panels-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite various-suite depends_on: - grafana-server @@ -1808,7 +1807,7 @@ steps: GITHUB_TOKEN: from_secret: github_token HOST: grafana-server - image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:3.0.0 name: end-to-end-tests-cloud-plugins-suite-azure when: paths: @@ -1853,23 +1852,16 @@ steps: - yarn storybook:build - ./bin/build verify-storybook depends_on: - - build-frontend + - rgm-package - build-frontend-packages environment: NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: build-storybook when: paths: include: - packages/grafana-ui/** -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.7.5 - name: copy-packages-for-docker - commands: - yarn wait-on http://$HOST:$PORT - pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json @@ -1905,49 +1897,39 @@ steps: repo: - grafana/grafana - commands: - - ./scripts/ci-frontend-metrics.sh | ./bin/build publish-metrics $${GRAFANA_MISC_STATS_API_KEY} + - apk add --update bash grep git + - ./scripts/ci-frontend-metrics.sh ./grafana/public/build | ./bin/build publish-metrics + $$GRAFANA_MISC_STATS_API_KEY depends_on: - test-a11y-frontend environment: GRAFANA_MISC_STATS_API_KEY: from_secret: grafana_misc_stats_api_key failure: ignore - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: publish-frontend-metrics when: repo: - grafana/grafana - commands: - - ./bin/build build-docker --edition oss + - docker run --privileged --rm tonistiigi/binfmt --install all + - /src/grafana-build docker $(cat packages.txt | grep tar.gz | grep -v docker | + grep -v sha256 | awk '{print "--package=" $0}') --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.18.3 + --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --ubuntu-tag-format='{{ + .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > docker.txt + - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build build-docker --edition oss --ubuntu - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images-ubuntu + - rgm-package + image: grafana/grafana-build:main + name: rgm-build-docker + pull: always volumes: - name: docker path: /var/run/docker.sock - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana depends_on: - - build-docker-images - - build-docker-images-ubuntu + - rgm-build-docker environment: DOCKER_PASSWORD: from_secret: docker_password @@ -1972,8 +1954,7 @@ steps: - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana-oss depends_on: - - build-docker-images - - build-docker-images-ubuntu + - rgm-build-docker environment: DOCKER_PASSWORD: from_secret: docker_password @@ -1996,16 +1977,18 @@ steps: repo: - grafana/grafana - commands: + - apk add --update bash - ./scripts/publish-npm-packages.sh --dist-tag 'canary' --registry 'https://registry.npmjs.org' depends_on: - end-to-end-tests-dashboards-suite - end-to-end-tests-panels-suite - end-to-end-tests-smoke-tests-suite - end-to-end-tests-various-suite + - build-frontend-packages environment: NPM_TOKEN: from_secret: npm_token - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: release-canary-npm-packages when: paths: @@ -2110,6 +2093,11 @@ services: volumes: - name: mysql80 path: /var/lib/mysql +- commands: + - /bin/mimir -target=backend + environment: {} + image: grafana/mimir:latest + name: mimir_backend - environment: {} image: redis:6.2.11-alpine name: redis @@ -2119,7 +2107,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2128,7 +2116,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME @@ -2139,28 +2127,34 @@ steps: with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: - - apt-get update - - apt-get install -yq postgresql-client - dockerize -wait tcp://postgres:5432 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-postgres +- commands: + - apk add --update build-base + - apk add --update postgresql-client - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find @@ -2168,16 +2162,20 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-postgres environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: postgres-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - dockerize -wait tcp://mysql57:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-5.7 +- commands: + - apk add --update build-base + - apk add --update mysql-client - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql57 -P 3306 -u root -prootpass - go clean -testcache @@ -2186,15 +2184,19 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-5.7 environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql57 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: mysql-5.7-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - dockerize -wait tcp://mysql80:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-8.0 +- commands: + - apk add --update build-base + - apk add --update mysql-client - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql80 -P 3306 -u root -prootpass - go clean -testcache @@ -2203,31 +2205,59 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-8.0 environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql80 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: mysql-8.0-integration-tests - commands: - - dockerize -wait tcp://redis:6379/0 -timeout 120s + - dockerize -wait tcp://redis:6379 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-redis +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-redis environment: REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: redis-integration-tests - commands: - dockerize -wait tcp://memcached:11211 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-memcached +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-memcached environment: MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: memcached-integration-tests +- commands: + - dockerize -wait tcp://mimir_backend:8080 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-remote-alertmanager +- commands: + - apk add --update build-base + - go clean -testcache + - go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/... + depends_on: + - wire-install + - wait-for-remote-alertmanager + environment: + AM_PASSWORD: test + AM_TENANT_ID: test + AM_URL: http://mimir_backend:8080 + image: golang:1.20.10-alpine + name: remote-alertmanager-integration-tests trigger: branch: main event: @@ -2279,33 +2309,10 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: windows-init -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - - .\grabpl.exe windows-installer --edition oss --build-id $$env:DRONE_BUILD_NUMBER - - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp "$$fname" gs://grafana-downloads/oss/main/ - - gsutil cp "$$fname.sha256" gs://grafana-downloads/oss/main/ - depends_on: - - windows-init - environment: - GCP_KEY: - from_secret: gcp_key - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer trigger: branch: main event: @@ -2323,38 +2330,6 @@ volumes: path: //./pipe/docker_engine/ name: docker --- -clone: - retries: 3 -depends_on: [] -kind: pipeline -name: notify-drone-changes -platform: - arch: amd64 - os: linux -steps: -- image: plugins/slack - name: slack - settings: - channel: slack-webhooks-test - template: "`.drone.yml` and `starlark` files have been changed on the OSS repo, - by: {{build.author}}. \nBranch: \nCommit hash: " - webhook: - from_secret: drone-changes-webhook -trigger: - branch: main - event: - - push - paths: - exclude: - - exclude - include: - - .drone.yml - repo: - - grafana/grafana -type: docker ---- clone: retries: 3 depends_on: @@ -2462,7 +2437,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2471,7 +2446,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - ./bin/build artifacts docker fetch --edition oss @@ -2567,7 +2542,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - ./bin/build artifacts packages --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET} @@ -2636,12 +2611,12 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: - ./bin/build artifacts npm retrieve --tag ${DRONE_TAG} @@ -2665,7 +2640,7 @@ steps: NPM_TOKEN: from_secret: npm_token failure: ignore - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: release-npm-packages trigger: event: @@ -2701,7 +2676,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - depends_on: - compile-build-cmd @@ -2768,6 +2743,79 @@ volumes: path: /var/run/docker.sock name: docker --- +clone: + retries: 3 +depends_on: +- main-test-backend +- main-test-frontend +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-main-prerelease +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_publish_main.sh + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.20.10 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: grafana/grafana-build:main + name: rgm-build + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + branch: main + event: + - push + paths: + exclude: + - '*.md' + - docs/** + - packages/**/*.md + - latest.json + repo: + - grafana/grafana +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- clone: retries: 3 depends_on: [] @@ -2789,13 +2837,13 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - ./bin/build whatsnew-checker depends_on: - compile-build-cmd - image: golang:1.20.10 + image: golang:1.20.10-alpine name: whats-new-checker trigger: event: @@ -2832,22 +2880,17 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: yarn-install - commands: + - apk add --update git bash - yarn betterer ci depends_on: - yarn-install - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: betterer-frontend - commands: - yarn run ci:test-frontend @@ -2855,7 +2898,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 + image: node:18.12.0-alpine name: test-frontend trigger: event: @@ -2892,49 +2935,47 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: + - apk add --update build-base shared-mime-info shared-mime-info-lang - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: test-backend - commands: + - apk add --update build-base - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: test-backend-integration trigger: event: @@ -2954,64 +2995,6 @@ volumes: clone: retries: 3 depends_on: -- main-test-backend -- main-test-frontend -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: rgm-main-prerelease -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - export GRAFANA_DIR=$$(pwd) - - cd /src && ./scripts/drone_publish_main.sh - environment: - _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: - from_secret: dagger_token - DESTINATION: - from_secret: destination - GCP_KEY_BASE64: - from_secret: gcp_key_base64 - GITHUB_TOKEN: - from_secret: github_token - GPG_PASSPHRASE: - from_secret: packages_gpg_passphrase - GPG_PRIVATE_KEY: - from_secret: packages_gpg_private_key - GPG_PUBLIC_KEY: - from_secret: packages_gpg_public_key - failure: ignore - image: grafana/grafana-build:main - name: rgm-build - volumes: - - name: docker - path: /var/run/docker.sock -trigger: - branch: main - event: - - push - paths: - exclude: - - '*.md' - - docs/** - - packages/**/*.md - - latest.json - repo: - - grafana/grafana -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: - release-test-backend - release-test-frontend image_pull_secrets: @@ -3031,20 +3014,36 @@ steps: environment: _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination DESTINATION: from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key GCP_KEY_BASE64: from_secret: gcp_key_base64 GITHUB_TOKEN: from_secret: github_token + GO_VERSION: 1.20.10 GPG_PASSPHRASE: from_secret: packages_gpg_passphrase GPG_PRIVATE_KEY: from_secret: packages_gpg_private_key GPG_PUBLIC_KEY: from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination image: grafana/grafana-build:main name: rgm-build + pull: always volumes: - name: docker path: /var/run/docker.sock @@ -3079,14 +3078,12 @@ services: [] steps: - commands: - echo $env:DRONE_RUNNER_NAME - failure: ignore image: mcr.microsoft.com/windows:1809 name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/windows/grabpl.exe -OutFile grabpl.exe - failure: ignore image: grafana/ci-wix:0.1.1 name: windows-init - commands: @@ -3100,18 +3097,17 @@ steps: - .\grabpl.exe windows-installer --target gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/grafana-${DRONE_TAG:1}.windows-amd64.zip --edition oss ${DRONE_TAG} - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp "$$fname" gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/ + - gsutil cp $$fname gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/ - gsutil cp "$$fname.sha256" gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/ depends_on: - windows-init environment: GCP_KEY: - from_secret: gcp_key + from_secret: gcp_grafanauploads_base64 GITHUB_TOKEN: from_secret: github_token PRERELEASE_BUCKET: from_secret: prerelease_bucket - failure: ignore image: grafana/ci-wix:0.1.1 name: build-windows-installer trigger: @@ -3174,6 +3170,488 @@ volumes: path: /var/run/docker.sock name: docker --- +clone: + retries: 3 +depends_on: +- release-test-backend +- release-test-frontend +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-version-branch-prerelease +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_publish_tag_grafana.sh + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.20.10 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: grafana/grafana-build:main + name: rgm-build + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + ref: + - refs/heads/v[0-9]* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: +- rgm-version-branch-prerelease +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-prerelease-verify-prerelease-assets +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - apt-get update && apt-get install -yq gettext + - printenv GCP_KEY | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - ./scripts/list-release-artifacts.sh ${DRONE_TAG} | xargs -n1 gsutil stat >> /tmp/stat.log + - '! cat /tmp/stat.log | grep "No URLs matched"' + depends_on: + - clone + environment: + BUCKET: grafana-prerelease + GCP_KEY: + from_secret: gcp_key_base64 + image: google/cloud-sdk:431.0.0 + name: gsutil-stat +trigger: + ref: + - refs/heads/v[0-9]* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: [] +environment: + EDITION: oss +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: nightly-test-frontend +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.18.3 + name: identify-runner +- commands: + - yarn install --immutable + depends_on: [] + image: node:18.12.0-alpine + name: yarn-install +- commands: + - apk add --update git bash + - yarn betterer ci + depends_on: + - yarn-install + image: node:18.12.0-alpine + name: betterer-frontend +- commands: + - yarn run ci:test-frontend + depends_on: + - yarn-install + environment: + TEST_MAX_WORKERS: 50% + image: node:18.12.0-alpine + name: test-frontend +trigger: + cron: + include: + - nightly-release + event: + include: + - cron +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: [] +environment: + EDITION: oss +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: nightly-test-backend +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.18.3 + name: identify-runner +- commands: + - '# It is required that code generated from Thema/CUE be committed and in sync + with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - apk add --update make + - CODEGEN_VERIFY=1 make gen-cue + depends_on: [] + image: golang:1.20.10-alpine + name: verify-gen-cue +- commands: + - '# It is required that generated jsonnet is committed and in sync with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - apk add --update make + - CODEGEN_VERIFY=1 make gen-jsonnet + depends_on: [] + image: golang:1.20.10-alpine + name: verify-gen-jsonnet +- commands: + - apk add --update make + - make gen-go + depends_on: + - verify-gen-cue + image: golang:1.20.10-alpine + name: wire-install +- commands: + - apk add --update build-base shared-mime-info shared-mime-info-lang + - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... + depends_on: + - wire-install + image: golang:1.20.10-alpine + name: test-backend +- commands: + - apk add --update build-base + - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) + depends_on: + - wire-install + image: golang:1.20.10-alpine + name: test-backend-integration +trigger: + cron: + include: + - nightly-release + event: + include: + - cron +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: +- nightly-test-backend +- nightly-test-frontend +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-nightly-build +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_build_nightly_grafana.sh + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.20.10 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: grafana/grafana-build:main + name: rgm-build + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - mkdir -p $${DESTINATION}/$${DRONE_BUILD_EVENT} + - printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - gcloud storage cp -r $${DRONE_WORKSPACE}/dist/* $${DESTINATION}/$${DRONE_BUILD_EVENT} + depends_on: + - rgm-build + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: google/cloud-sdk:alpine + name: rgm-copy +trigger: + cron: + include: + - nightly-release + event: + include: + - cron +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: +- rgm-nightly-build +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-nightly-publish +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p $${DRONE_WORKSPACE}/dist + - printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - gcloud storage cp -r $${DESTINATION}/$${DRONE_BUILD_EVENT}/*_$${DRONE_BUILD_NUMBER}_* + $${DRONE_WORKSPACE}/dist + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: google/cloud-sdk:alpine + name: rgm-copy +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_publish_nightly_grafana.sh + depends_on: + - rgm-copy + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.20.10 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: grafana/grafana-build:main + name: rgm-publish + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +- depends_on: + - rgm-publish + image: us.gcr.io/kubernetes-dev/package-publish:latest + name: publish-deb + privileged: true + settings: + access_key_id: + from_secret: packages_access_key_id + gpg_passphrase: + from_secret: packages_gpg_passphrase + gpg_private_key: + from_secret: packages_gpg_private_key + gpg_public_key: + from_secret: packages_gpg_public_key + package_path: file:///drone/src/dist/*.deb + secret_access_key: + from_secret: packages_secret_access_key + service_account_json: + from_secret: packages_service_account + target_bucket: grafana-packages +- depends_on: + - rgm-publish + image: us.gcr.io/kubernetes-dev/package-publish:latest + name: publish-rpm + privileged: true + settings: + access_key_id: + from_secret: packages_access_key_id + gpg_passphrase: + from_secret: packages_gpg_passphrase + gpg_private_key: + from_secret: packages_gpg_private_key + gpg_public_key: + from_secret: packages_gpg_public_key + package_path: file:///drone/src/dist/*.rpm + secret_access_key: + from_secret: packages_secret_access_key + service_account_json: + from_secret: packages_service_account + target_bucket: grafana-packages +trigger: + cron: + include: + - nightly-release + event: + include: + - cron +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- clone: disable: true depends_on: [] @@ -3200,20 +3678,20 @@ steps: - commands: [] depends_on: - clone - image: grafana/grafana-ci-windows-test:0.1.0 + image: golang:1.20.10-windowsservercore-1809 name: windows-init - commands: - go install github.com/google/wire/cmd/wire@v0.5.0 - wire gen -tags oss ./pkg/server depends_on: - windows-init - image: grafana/grafana-ci-windows-test:0.1.0 + image: golang:1.20.10-windowsservercore-1809 name: wire-install - commands: - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/grafana-ci-windows-test:0.1.0 + image: golang:1.20.10-windowsservercore-1809 name: test-backend trigger: event: @@ -3234,650 +3712,6 @@ environment: image_pull_secrets: - dockerconfigjson kind: pipeline -name: release-branch-build-e2e-publish -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 - name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: [] - image: grafana/build-container:1.7.5 - name: verify-gen-cue -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.7.5 - name: wire-install -- commands: - - yarn install --immutable - depends_on: [] - image: grafana/build-container:1.7.5 - name: yarn-install -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd -- commands: - - ./bin/build build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} - depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.7.5 - name: build-backend -- commands: - - ./bin/build build-frontend --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 - name: build-frontend -- commands: - - ./bin/build build-frontend-packages --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - - yarn packages:pack - - ./scripts/validate-npm-packages.sh - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.5 - name: build-frontend-packages -- commands: - - ./bin/build build-plugins --jobs 8 --edition oss - depends_on: - - compile-build-cmd - - yarn-install - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.5 - name: build-plugins -- commands: - - ./bin/build package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --sign - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - environment: - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.5 - name: package -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.7.5 - name: copy-packages-for-docker -- commands: - - ./bin/build build-docker --edition oss --shouldSave - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build build-docker --edition oss --shouldSave --ubuntu - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images-ubuntu - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./scripts/grafana-server/start-server - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - detach: true - environment: - ARCH: linux-amd64 - PORT: 3001 - image: grafana/build-container:1.7.5 - name: grafana-server -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite dashboards-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-dashboards-suite -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-smoke-tests-suite -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite panels-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-panels-suite -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite various-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-various-suite -- commands: - - apt-get update - - apt-get install -yq zip - - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ - - gsutil cp e2e/videos.zip gs://$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - export E2E_ARTIFACTS_VIDEO_ZIP=https://storage.googleapis.com/$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - 'echo "E2E Test artifacts uploaded to: $${E2E_ARTIFACTS_VIDEO_ZIP}"' - - 'curl -X POST https://api.github.com/repos/${DRONE_REPO}/statuses/${DRONE_COMMIT_SHA} - -H "Authorization: token $${GITHUB_TOKEN}" -d "{\"state\":\"success\",\"target_url\":\"$${E2E_ARTIFACTS_VIDEO_ZIP}\", - \"description\": \"Click on the details to download e2e recording videos\", \"context\": - \"e2e_artifacts\"}"' - depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - E2E_TEST_ARTIFACTS_BUCKET: releng-pipeline-artifacts-dev - GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY: - from_secret: gcp_upload_artifacts_key - GITHUB_TOKEN: - from_secret: github_token - failure: ignore - image: google/cloud-sdk:431.0.0 - name: e2e-tests-artifacts-upload - when: - status: - - success - - failure -- commands: - - yarn storybook:build - - ./bin/build verify-storybook - depends_on: - - build-frontend - - build-frontend-packages - environment: - NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.7.5 - name: build-storybook - when: - paths: - include: - - packages/grafana-ui/** -- commands: - - ./bin/build upload-cdn --edition oss - depends_on: - - grafana-server - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-cdn-assets - when: - repo: - - grafana/grafana -- commands: - - ./bin/build upload-packages --edition oss - depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - GCP_KEY: - from_secret: gcp_grafanauploads_base64 - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-packages - when: - repo: - - grafana/grafana -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql57 - temp: - medium: memory -- name: mysql80 - temp: - medium: memory ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-test-frontend -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 - name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - yarn install --immutable - depends_on: [] - image: grafana/build-container:1.7.5 - name: yarn-install -- commands: - - yarn betterer ci - depends_on: - - yarn-install - image: grafana/build-container:1.7.5 - name: betterer-frontend -- commands: - - yarn run ci:test-frontend - depends_on: - - yarn-install - environment: - TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.5 - name: test-frontend -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-test-backend -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 - name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: [] - image: grafana/build-container:1.7.5 - name: verify-gen-cue -- commands: - - '# It is required that generated jsonnet is committed and in sync with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-jsonnet - depends_on: [] - image: grafana/build-container:1.7.5 - name: verify-gen-jsonnet -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.7.5 - name: wire-install -- commands: - - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... - depends_on: - - wire-install - image: grafana/build-container:1.7.5 - name: test-backend -- commands: - - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find - ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' - | grep -o '\(.*\)/' | sort -u) - depends_on: - - wire-install - image: grafana/build-container:1.7.5 - name: test-backend-integration -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-integration-tests -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: -- environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_DB: grafanatest - POSTGRES_PASSWORD: grafanatest - POSTGRES_USER: grafanatest - image: postgres:12.3-alpine - name: postgres - volumes: - - name: postgres - path: /var/lib/postgresql/data/pgdata -- commands: - - docker-entrypoint.sh mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci - environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:5.7.39 - name: mysql57 - volumes: - - name: mysql57 - path: /var/lib/mysql -- commands: - - docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password - environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:8.0.32 - name: mysql80 - volumes: - - name: mysql80 - path: /var/lib/mysql -- environment: {} - image: redis:6.2.11-alpine - name: redis -- environment: {} - image: memcached:1.6.9-alpine - name: memcached -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 - name: identify-runner -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: [] - image: grafana/build-container:1.7.5 - name: verify-gen-cue -- commands: - - '# It is required that generated jsonnet is committed and in sync with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-jsonnet - depends_on: [] - image: grafana/build-container:1.7.5 - name: verify-gen-jsonnet -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.7.5 - name: wire-install -- commands: - - apt-get update - - apt-get install -yq postgresql-client - - dockerize -wait tcp://postgres:5432 -timeout 120s - - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - - go clean -testcache - - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find - ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' - | grep -o '\(.*\)/' | sort -u) - depends_on: - - wire-install - environment: - GRAFANA_TEST_DB: postgres - PGPASSWORD: grafanatest - POSTGRES_HOST: postgres - image: grafana/build-container:1.7.5 - name: postgres-integration-tests -- commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql57:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql57 -P 3306 -u root - -prootpass - - go clean -testcache - - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find - ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' - | grep -o '\(.*\)/' | sort -u) - depends_on: - - wire-install - environment: - GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql57 - image: grafana/build-container:1.7.5 - name: mysql-5.7-integration-tests -- commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql80:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql80 -P 3306 -u root - -prootpass - - go clean -testcache - - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find - ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' - | grep -o '\(.*\)/' | sort -u) - depends_on: - - wire-install - environment: - GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql80 - image: grafana/build-container:1.7.5 - name: mysql-8.0-integration-tests -- commands: - - dockerize -wait tcp://redis:6379/0 -timeout 120s - - go clean -testcache - - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... - depends_on: - - wire-install - environment: - REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.7.5 - name: redis-integration-tests -- commands: - - dockerize -wait tcp://memcached:11211 -timeout 120s - - go clean -testcache - - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... - depends_on: - - wire-install - environment: - MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.7.5 - name: memcached-integration-tests -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql57 - temp: - medium: memory -- name: mysql80 - temp: - medium: memory ---- -clone: - retries: 3 -depends_on: -- release-branch-build-e2e-publish -- release-branch-test-frontend -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-windows -platform: - arch: amd64 - os: windows - version: "1809" -services: [] -steps: -- commands: - - echo $env:DRONE_RUNNER_NAME - image: mcr.microsoft.com/windows:1809 - name: identify-runner -- commands: - - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/windows/grabpl.exe - -OutFile grabpl.exe - image: grafana/ci-wix:0.1.1 - name: windows-init -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - depends_on: - - windows-init - environment: - GCP_KEY: - from_secret: gcp_key - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: //./pipe/docker_engine/ - name: docker ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline name: integration-tests node: type: no-parallel @@ -3919,6 +3753,11 @@ services: volumes: - name: mysql80 path: /var/lib/mysql +- commands: + - /bin/mimir -target=backend + environment: {} + image: grafana/mimir:latest + name: mimir_backend - environment: {} image: redis:6.2.11-alpine name: redis @@ -3928,7 +3767,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -3941,28 +3780,34 @@ steps: with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: wire-install - commands: - - apt-get update - - apt-get install -yq postgresql-client - dockerize -wait tcp://postgres:5432 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-postgres +- commands: + - apk add --update build-base + - apk add --update postgresql-client - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find @@ -3970,16 +3815,20 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-postgres environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: postgres-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - dockerize -wait tcp://mysql57:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-5.7 +- commands: + - apk add --update build-base + - apk add --update mysql-client - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql57 -P 3306 -u root -prootpass - go clean -testcache @@ -3988,15 +3837,19 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-5.7 environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql57 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: mysql-5.7-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - dockerize -wait tcp://mysql80:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-8.0 +- commands: + - apk add --update build-base + - apk add --update mysql-client - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql80 -P 3306 -u root -prootpass - go clean -testcache @@ -4005,31 +3858,59 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-8.0 environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql80 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: mysql-8.0-integration-tests - commands: - - dockerize -wait tcp://redis:6379/0 -timeout 120s + - dockerize -wait tcp://redis:6379 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-redis +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-redis environment: REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: redis-integration-tests - commands: - dockerize -wait tcp://memcached:11211 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-memcached +- commands: + - apk add --update build-base - go clean -testcache - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install + - wait-for-memcached environment: MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.7.5 + image: golang:1.20.10-alpine name: memcached-integration-tests +- commands: + - dockerize -wait tcp://mimir_backend:8080 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-remote-alertmanager +- commands: + - apk add --update build-base + - go clean -testcache + - go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/... + depends_on: + - wire-install + - wait-for-remote-alertmanager + environment: + AM_PASSWORD: test + AM_TENANT_ID: test + AM_URL: http://mimir_backend:8080 + image: golang:1.20.10-alpine + name: remote-alertmanager-integration-tests trigger: event: - promote @@ -4171,7 +4052,7 @@ steps: - name: config path: /root/.docker/ - commands: - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest + - trivy image --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -4179,15 +4060,22 @@ steps: volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - commands: - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest + - trivy image --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest depends_on: - authenticate-gcr + environment: + GOOGLE_APPLICATION_CREDENTIALS: + from_secret: gcr_credentials_json image: aquasec/trivy:0.21.0 name: scan-high-critical-vulnerabilities volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - image: plugins/slack name: slack-notify-failure settings: @@ -4205,6 +4093,8 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: config + temp: {} --- clone: retries: 3 @@ -4227,7 +4117,7 @@ steps: - name: config path: /root/.docker/ - commands: - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main + - trivy image --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -4235,15 +4125,22 @@ steps: volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - commands: - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main + - trivy image --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main depends_on: - authenticate-gcr + environment: + GOOGLE_APPLICATION_CREDENTIALS: + from_secret: gcr_credentials_json image: aquasec/trivy:0.21.0 name: scan-high-critical-vulnerabilities volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - image: plugins/slack name: slack-notify-failure settings: @@ -4261,6 +4158,8 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: config + temp: {} --- clone: retries: 3 @@ -4283,7 +4182,7 @@ steps: - name: config path: /root/.docker/ - commands: - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest-ubuntu + - trivy image --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest-ubuntu depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -4291,15 +4190,22 @@ steps: volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - commands: - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest-ubuntu + - trivy image --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest-ubuntu depends_on: - authenticate-gcr + environment: + GOOGLE_APPLICATION_CREDENTIALS: + from_secret: gcr_credentials_json image: aquasec/trivy:0.21.0 name: scan-high-critical-vulnerabilities volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - image: plugins/slack name: slack-notify-failure settings: @@ -4318,6 +4224,8 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: config + temp: {} --- clone: retries: 3 @@ -4340,7 +4248,7 @@ steps: - name: config path: /root/.docker/ - commands: - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main-ubuntu + - trivy image --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main-ubuntu depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -4348,15 +4256,22 @@ steps: volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - commands: - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main-ubuntu + - trivy image --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main-ubuntu depends_on: - authenticate-gcr + environment: + GOOGLE_APPLICATION_CREDENTIALS: + from_secret: gcr_credentials_json image: aquasec/trivy:0.21.0 name: scan-high-critical-vulnerabilities volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - image: plugins/slack name: slack-notify-failure settings: @@ -4375,6 +4290,8 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: config + temp: {} --- clone: retries: 3 @@ -4397,14 +4314,18 @@ steps: - name: config path: /root/.docker/ - commands: + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine/git:2.40.1 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM golang:1.20.10-alpine + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM node:18.12.0-alpine - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM google/cloud-sdk:431.0.0 - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/build-container:1.7.5 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana-ci-deploy:1.3.3 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine:3.18.3 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM ubuntu:22.04 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM byrnedo/alpine-curl:0.1.8 - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM golang:1.20.10 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM plugins/slack + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM python:3.8 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM postgres:12.3-alpine + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/mimir:latest - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:5.7.39 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:8.0.32 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM redis:6.2.11-alpine @@ -4415,7 +4336,8 @@ steps: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docker-puppeteer:1.1.0 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docs-base:dbd975af06 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM jwilder/dockerize:0.6.1 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM koalaman/shellcheck:stable depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -4423,15 +4345,21 @@ steps: volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - commands: + - trivy --exit-code 1 --severity HIGH,CRITICAL alpine/git:2.40.1 + - trivy --exit-code 1 --severity HIGH,CRITICAL golang:1.20.10-alpine + - trivy --exit-code 1 --severity HIGH,CRITICAL node:18.12.0-alpine - trivy --exit-code 1 --severity HIGH,CRITICAL google/cloud-sdk:431.0.0 - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/build-container:1.7.5 - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana-ci-deploy:1.3.3 - trivy --exit-code 1 --severity HIGH,CRITICAL alpine:3.18.3 + - trivy --exit-code 1 --severity HIGH,CRITICAL ubuntu:22.04 - trivy --exit-code 1 --severity HIGH,CRITICAL byrnedo/alpine-curl:0.1.8 - - trivy --exit-code 1 --severity HIGH,CRITICAL golang:1.20.10 - trivy --exit-code 1 --severity HIGH,CRITICAL plugins/slack + - trivy --exit-code 1 --severity HIGH,CRITICAL python:3.8 - trivy --exit-code 1 --severity HIGH,CRITICAL postgres:12.3-alpine + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/mimir:latest - trivy --exit-code 1 --severity HIGH,CRITICAL mysql:5.7.39 - trivy --exit-code 1 --severity HIGH,CRITICAL mysql:8.0.32 - trivy --exit-code 1 --severity HIGH,CRITICAL redis:6.2.11-alpine @@ -4442,14 +4370,20 @@ steps: - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docker-puppeteer:1.1.0 - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docs-base:dbd975af06 - trivy --exit-code 1 --severity HIGH,CRITICAL cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - - trivy --exit-code 1 --severity HIGH,CRITICAL us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + - trivy --exit-code 1 --severity HIGH,CRITICAL jwilder/dockerize:0.6.1 + - trivy --exit-code 1 --severity HIGH,CRITICAL koalaman/shellcheck:stable depends_on: - authenticate-gcr + environment: + GOOGLE_APPLICATION_CREDENTIALS: + from_secret: gcr_credentials_json image: aquasec/trivy:0.21.0 name: scan-high-critical-vulnerabilities volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - image: plugins/slack name: slack-notify-failure settings: @@ -4467,41 +4401,8 @@ volumes: - host: path: /var/run/docker.sock name: docker ---- -clone: - retries: 3 -kind: pipeline -name: grafana-com-nightly -platform: - arch: amd64 - os: linux -steps: -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd -- commands: - - ./bin/build publish grafana-com --edition oss - depends_on: - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - GRAFANA_COM_API_KEY: - from_secret: grafana_api_key - image: grafana/grafana-ci-deploy:1.3.3 - name: post-to-grafana-com -trigger: - cron: grafana-com-nightly - event: cron -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker +- name: config + temp: {} --- get: name: credentials.json @@ -4647,6 +4548,24 @@ get: kind: secret name: destination --- +get: + name: storybook_destination + path: infra/data/ci/grafana-release-eng/rgm +kind: secret +name: rgm_storybook_destination +--- +get: + name: cdn_destination + path: infra/data/ci/grafana-release-eng/rgm +kind: secret +name: rgm_cdn_destination +--- +get: + name: downloads_destination + path: infra/data/ci/grafana-release-eng/rgm +kind: secret +name: rgm_downloads_destination +--- get: name: dagger_token path: infra/data/ci/grafana-release-eng/rgm @@ -4678,6 +4597,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: 982c0bb9d38f9013a02517a80c60acff809490cd2a45a7bc63e01bbca309e9ec +hmac: 560e3a9b25dd7cfec06be7db82cda143115dcd22e0058cac77693e0da0c4058a ... diff --git a/Makefile b/Makefile index ceeb2673bef..114f0e5c29c 100644 --- a/Makefile +++ b/Makefile @@ -36,36 +36,68 @@ node_modules: package.json yarn.lock ## Install node modules. ##@ Swagger SPEC_TARGET = public/api-spec.json -MERGED_SPEC_TARGET := public/api-merged.json +ENTERPRISE_SPEC_TARGET = public/api-enterprise-spec.json +MERGED_SPEC_TARGET = public/api-merged.json NGALERT_SPEC_TARGET = pkg/services/ngalert/api/tooling/api.json $(NGALERT_SPEC_TARGET): +$(MAKE) -C pkg/services/ngalert/api/tooling api.json -$(MERGED_SPEC_TARGET): $(SPEC_TARGET) $(NGALERT_SPEC_TARGET) $(SWAGGER) ## Merge generated and ngalert API specs +$(MERGED_SPEC_TARGET): swagger-oss-gen swagger-enterprise-gen $(NGALERT_SPEC_TARGET) $(SWAGGER) ## Merge generated and ngalert API specs # known conflicts DsPermissionType, AddApiKeyCommand, Json, Duration (identical models referenced by both specs) - $(SWAGGER) mixin $(SPEC_TARGET) $(NGALERT_SPEC_TARGET) --ignore-conflicts -o $(MERGED_SPEC_TARGET) + $(SWAGGER) mixin $(SPEC_TARGET) $(ENTERPRISE_SPEC_TARGET) $(NGALERT_SPEC_TARGET) --ignore-conflicts -o $(MERGED_SPEC_TARGET) -$(SPEC_TARGET): $(SWAGGER) ## Generate API Swagger specification +swagger-oss-gen: $(SWAGGER) ## Generate API Swagger specification + @echo "re-generating swagger for OSS" + rm -f $(SPEC_TARGET) SWAGGER_GENERATE_EXTENSION=false $(SWAGGER) generate spec -m -w pkg/server -o $(SPEC_TARGET) \ -x "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" \ -x "github.com/prometheus/alertmanager" \ -i pkg/api/swagger_tags.json \ - --exclude-tag=alpha - go run pkg/services/ngalert/api/tooling/cmd/clean-swagger/main.go -if $@ -of $@ + --exclude-tag=alpha \ + --exclude-tag=enterprise -swagger-api-spec: gen-go $(SPEC_TARGET) $(MERGED_SPEC_TARGET) validate-api-spec +# this file only exists if enterprise is enabled +ENTERPRISE_EXT_FILE = pkg/extensions/ext.go +ifeq ("$(wildcard $(ENTERPRISE_EXT_FILE))","") ## if enterprise is not enabled +swagger-enterprise-gen: + @echo "skipping re-generating swagger for enterprise: not enabled" +else +swagger-enterprise-gen: $(SWAGGER) ## Generate API Swagger specification + @echo "re-generating swagger for enterprise" + rm -f $(ENTERPRISE_SPEC_TARGET) + SWAGGER_GENERATE_EXTENSION=false $(SWAGGER) generate spec -m -w pkg/server -o $(ENTERPRISE_SPEC_TARGET) \ + -x "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" \ + -x "github.com/prometheus/alertmanager" \ + -i pkg/api/swagger_tags.json \ + --exclude-tag=alpha \ + --include-tag=enterprise +endif -validate-api-spec: $(MERGED_SPEC_TARGET) $(SWAGGER) ## Validate API spec +swagger-gen: gen-go $(MERGED_SPEC_TARGET) swagger-validate + +swagger-validate: $(MERGED_SPEC_TARGET) $(SWAGGER) ## Validate API spec $(SWAGGER) validate $(<) -clean-api-spec: +swagger-clean: rm -f $(SPEC_TARGET) $(MERGED_SPEC_TARGET) $(OAPI_SPEC_TARGET) +.PHONY: cleanup-old-git-hooks +cleanup-old-git-hooks: + ./scripts/cleanup-husky.sh + +.PHONY: lefthook-install +lefthook-install: cleanup-old-git-hooks $(LEFTHOOK) # install lefthook for pre-commit hooks + $(LEFTHOOK) install -f + +.PHONY: lefthook-uninstall +lefthook-uninstall: $(LEFTHOOK) + $(LEFTHOOK) uninstall + ##@ OpenAPI 3 OAPI_SPEC_TARGET = public/openapi3.json -openapi3-gen: swagger-api-spec ## Generates OpenApi 3 specs from the Swagger 2 already generated +openapi3-gen: swagger-gen ## Generates OpenApi 3 specs from the Swagger 2 already generated $(GO) run scripts/openapi3/openapi3conv.go $(MERGED_SPEC_TARGET) $(OAPI_SPEC_TARGET) ##@ Building @@ -132,6 +164,13 @@ test-go-integration: ## Run integration tests for backend with flags. @echo "test backend integration tests" $(GO) test -count=1 -run "^TestIntegration" -covermode=atomic -timeout=5m $(GO_INTEGRATION_TESTS) +.PHONY: test-go-integration-alertmanager +test-go-integration-alertmanager: ## Run integration tests for the remote alertmanager (config taken from the mimir_backend block). + @echo "test remote alertmanager integration tests" + $(GO) clean -testcache + AM_URL=http://localhost:8080 AM_TENANT_ID=test AM_PASSWORD=test \ + $(GO) test -count=1 -run "^TestIntegrationRemoteAlertmanager" -covermode=atomic -timeout=5m ./pkg/services/ngalert/notifier/... + .PHONY: test-go-integration-postgres test-go-integration-postgres: devenv-postgres ## Run integration tests for postgres backend with flags. @echo "test backend integration postgres tests" @@ -190,7 +229,7 @@ build-docker-full: ## Build Docker image for development. --build-arg BINGO=false \ --build-arg GO_BUILD_TAGS=$(GO_BUILD_TAGS) \ --build-arg WIRE_TAGS=$(WIRE_TAGS) \ - --build-arg COMMIT_SHA=$$(git rev-parse --short HEAD) \ + --build-arg COMMIT_SHA=$$(git rev-parse HEAD) \ --build-arg BUILD_BRANCH=$$(git rev-parse --abbrev-ref HEAD) \ --tag grafana/grafana$(TAG_SUFFIX):dev \ $(DOCKER_BUILD_ARGS) @@ -203,9 +242,9 @@ build-docker-full-ubuntu: ## Build Docker image based on Ubuntu for development. --build-arg BINGO=false \ --build-arg GO_BUILD_TAGS=$(GO_BUILD_TAGS) \ --build-arg WIRE_TAGS=$(WIRE_TAGS) \ - --build-arg COMMIT_SHA=$$(git rev-parse --short HEAD) \ + --build-arg COMMIT_SHA=$$(git rev-parse HEAD) \ --build-arg BUILD_BRANCH=$$(git rev-parse --abbrev-ref HEAD) \ - --build-arg BASE_IMAGE=ubuntu:20.04 \ + --build-arg BASE_IMAGE=ubuntu:22.04 \ --build-arg GO_IMAGE=golang:1.20.10 \ --tag grafana/grafana$(TAG_SUFFIX):dev-ubuntu \ $(DOCKER_BUILD_ARGS) @@ -251,6 +290,9 @@ devenv-mysql: protobuf: ## Compile protobuf definitions bash scripts/protobuf-check.sh bash pkg/plugins/backendplugin/pluginextensionv2/generate.sh + bash pkg/plugins/backendplugin/secretsmanagerplugin/generate.sh + bash pkg/services/store/entity/generate.sh + bash pkg/infra/grn/generate.sh clean: ## Clean up intermediate build artifacts. @echo "cleaning" @@ -276,7 +318,7 @@ scripts/drone/TAGS: $(shell find scripts/drone -name '*.star') etags --lang none --regex="/def \(\w+\)[^:]+:/\1/" --regex="/\s*\(\w+\) =/\1/" $^ -o $@ format-drone: - buildifier -r scripts/drone + buildifier --lint=fix -r scripts/drone help: ## Display this help. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/packages/grafana-data/src/datetime/formatter.test.ts b/packages/grafana-data/src/datetime/formatter.test.ts index 888f3d07c8e..18bea1cde78 100644 --- a/packages/grafana-data/src/datetime/formatter.test.ts +++ b/packages/grafana-data/src/datetime/formatter.test.ts @@ -97,17 +97,9 @@ describe('dateTimeFormat', () => { }); describe('dateTimeFormatTimeAgo', () => { - it('should return the correct format for 3 years ago', () => { + it('should return the correct format for years ago', () => { const options = { timeZone: 'Europe/Stockholm' }; - expect(dateTimeFormatTimeAgo(1587126975779, options)).toBe('3 years ago'); - }); - it('should return the correct format for 2 year ago', () => { - const options = { timeZone: 'Europe/Stockholm' }; - expect(dateTimeFormatTimeAgo(1626154993000, options)).toBe('2 years ago'); - }); - it('should return the correct format for 1 year ago', () => { - const options = { timeZone: 'Europe/Stockholm' }; - expect(dateTimeFormatTimeAgo(1657731795000, options)).toBe('a year ago'); + expect(dateTimeFormatTimeAgo(1587126975779, options)).toContain('years ago'); }); }); describe('dateTimeFormatWithAbbreviation', () => { diff --git a/pkg/build/cmd.go b/pkg/build/cmd.go index 7248c91080d..fa98316e9c2 100644 --- a/pkg/build/cmd.go +++ b/pkg/build/cmd.go @@ -222,15 +222,25 @@ func ldflags(opts BuildOpts) (string, error) { commitSha = v } + var enterpriseCommitSha string + if opts.enterprise { + enterpriseCommitSha = getGitEnterpriseSha() + if v := os.Getenv("ENTERPRISE_COMMIT_SHA"); v != "" { + enterpriseCommitSha = v + } + } + buildBranch := getGitBranch() if v := os.Getenv("BUILD_BRANCH"); v != "" { buildBranch = v } - var b bytes.Buffer b.WriteString("-w") b.WriteString(fmt.Sprintf(" -X main.version=%s", opts.version)) b.WriteString(fmt.Sprintf(" -X main.commit=%s", commitSha)) + if enterpriseCommitSha != "" { + b.WriteString(fmt.Sprintf(" -X main.enterpriseCommit=%s", enterpriseCommitSha)) + } b.WriteString(fmt.Sprintf(" -X main.buildstamp=%d", buildStamp)) b.WriteString(fmt.Sprintf(" -X main.buildBranch=%s", buildBranch)) if v := os.Getenv("LDFLAGS"); v != "" { diff --git a/pkg/build/cmd/grafanacom.go b/pkg/build/cmd/grafanacom.go index e56b3367200..9d44b9298c9 100644 --- a/pkg/build/cmd/grafanacom.go +++ b/pkg/build/cmd/grafanacom.go @@ -232,7 +232,7 @@ func getSHA256(u string) ([]byte, error) { return sha256, nil } -func postRequest(cfg packaging.PublishConfig, pth string, obj interface{}, descr string) error { +func postRequest(cfg packaging.PublishConfig, pth string, obj any, descr string) error { var sfx string switch cfg.Edition { case config.EditionOSS: diff --git a/pkg/build/cmd/verifydrone.go b/pkg/build/cmd/verifydrone.go index 4794efa4500..14596377110 100644 --- a/pkg/build/cmd/verifydrone.go +++ b/pkg/build/cmd/verifydrone.go @@ -68,7 +68,7 @@ func VerifyDrone(c *cli.Context) error { return nil } -func readConfig(fpath string) ([]map[string]interface{}, error) { +func readConfig(fpath string) ([]map[string]any, error) { //nolint:gosec f, err := os.Open(fpath) if err != nil { @@ -82,9 +82,9 @@ func readConfig(fpath string) ([]map[string]interface{}, error) { // The YAML stream may contain multiple pipeline configurations, read them all dec := yaml.NewDecoder(f) - var c []map[string]interface{} + var c []map[string]any for { - var m map[string]interface{} + var m map[string]any if err := dec.Decode(&m); err != nil { if errors.Is(err, io.EOF) { break diff --git a/pkg/build/config/revision.go b/pkg/build/config/revision.go index aa104916652..70be21cdf18 100644 --- a/pkg/build/config/revision.go +++ b/pkg/build/config/revision.go @@ -3,6 +3,7 @@ package config import ( "context" "fmt" + "log" "strconv" "time" @@ -10,9 +11,10 @@ import ( ) type Revision struct { - Timestamp int64 - SHA256 string - Branch string + Timestamp int64 + SHA256 string + EnterpriseCommit string + Branch string } func GrafanaTimestamp(ctx context.Context, dir string) (int64, error) { @@ -42,14 +44,26 @@ func GrafanaRevision(ctx context.Context, grafanaDir string) (Revision, error) { return Revision{}, err } + enterpriseCommit, err := executil.OutputAt(ctx, grafanaDir, "git", "-C", "../grafana-enterprise", "rev-parse", "--short", "HEAD") + if err != nil { + enterpriseCommit, err = executil.OutputAt(ctx, grafanaDir, "git", "-C", "..", "rev-parse", "--short", "HEAD") + if err != nil { + enterpriseCommit, err = executil.OutputAt(ctx, grafanaDir, "git", "-C", "/tmp/grafana-enterprise", "rev-parse", "--short", "HEAD") + if err != nil { + log.Println("Could not get enterprise commit. Error:", err) + } + } + } + branch, err := executil.OutputAt(ctx, grafanaDir, "git", "rev-parse", "--abbrev-ref", "HEAD") if err != nil { return Revision{}, err } return Revision{ - SHA256: sha, - Branch: branch, - Timestamp: stamp, + SHA256: sha, + EnterpriseCommit: enterpriseCommit, + Branch: branch, + Timestamp: stamp, }, nil } diff --git a/pkg/build/config/version.go b/pkg/build/config/version.go index 5eb96ab63f3..05b99cf4678 100644 --- a/pkg/build/config/version.go +++ b/pkg/build/config/version.go @@ -94,7 +94,7 @@ func GetPackageJSONVersion(grafanaDir string) (string, error) { if err != nil { return "", fmt.Errorf("failed to read %q: %w", pkgJSONPath, err) } - pkgObj := map[string]interface{}{} + pkgObj := map[string]any{} if err := json.Unmarshal(pkgJSONB, &pkgObj); err != nil { return "", fmt.Errorf("failed decoding %q: %w", pkgJSONPath, err) } diff --git a/pkg/build/config/versions.go b/pkg/build/config/versions.go index aa618e90358..8decc72593c 100644 --- a/pkg/build/config/versions.go +++ b/pkg/build/config/versions.go @@ -1,7 +1,5 @@ package config -const PublicBucket = "grafana-downloads" - var Versions = VersionMap{ PullRequestMode: { Variants: []Variant{ @@ -9,8 +7,9 @@ var Versions = VersionMap{ VariantLinuxAmd64Musl, VariantDarwinAmd64, VariantWindowsAmd64, - VariantArm64, - VariantArm64Musl, + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // VariantArm64, + // VariantArm64Musl, }, PluginSignature: PluginSignature{ Sign: false, @@ -29,9 +28,10 @@ var Versions = VersionMap{ }, MainMode: { Variants: []Variant{ - VariantArmV6, - VariantArmV7, - VariantArmV7Musl, + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // VariantArmV6, + // VariantArmV7, + // VariantArmV7Musl, VariantArm64, VariantArm64Musl, VariantDarwinAmd64, @@ -48,7 +48,8 @@ var Versions = VersionMap{ Architectures: []Architecture{ ArchAMD64, ArchARM64, - ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable. + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable. }, Distribution: []Distribution{ Alpine, @@ -64,9 +65,10 @@ var Versions = VersionMap{ }, DownstreamMode: { Variants: []Variant{ - VariantArmV6, - VariantArmV7, - VariantArmV7Musl, + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // VariantArmV6, + //VariantArmV7, + // VariantArmV7Musl, VariantArm64, VariantArm64Musl, VariantDarwinAmd64, @@ -83,7 +85,8 @@ var Versions = VersionMap{ Architectures: []Architecture{ ArchAMD64, ArchARM64, - ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable. + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable. }, Distribution: []Distribution{ Alpine, @@ -171,9 +174,10 @@ var Versions = VersionMap{ }, Enterprise2Mode: { Variants: []Variant{ - VariantArmV6, - VariantArmV7, - VariantArmV7Musl, + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // VariantArmV6, + // VariantArmV7, + // VariantArmV7Musl, VariantArm64, VariantArm64Musl, VariantDarwinAmd64, @@ -190,7 +194,8 @@ var Versions = VersionMap{ Architectures: []Architecture{ ArchAMD64, ArchARM64, - ArchARMv7, + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // ArchARMv7, }, Distribution: []Distribution{ Alpine, diff --git a/pkg/build/docker/build.go b/pkg/build/docker/build.go index 64016257b2a..29b2263e62b 100644 --- a/pkg/build/docker/build.go +++ b/pkg/build/docker/build.go @@ -74,7 +74,7 @@ func BuildImage(version string, arch config.Architecture, grafanaDir string, use tagSuffix := "" if useUbuntu { libc = "" - baseImage = fmt.Sprintf("%subuntu:20.04", baseArch) + baseImage = fmt.Sprintf("%subuntu:22.04", baseArch) tagSuffix = "-ubuntu" } diff --git a/pkg/build/gcloud/storage/gsutil.go b/pkg/build/gcloud/storage/gsutil.go index 197a4a06a33..711728f4fb0 100644 --- a/pkg/build/gcloud/storage/gsutil.go +++ b/pkg/build/gcloud/storage/gsutil.go @@ -418,7 +418,7 @@ func GetLatestMainBuild(ctx context.Context, bucket *storage.BucketHandle, path var latestVersion string for i := 0; i < len(files); i++ { - captureVersion := regexp.MustCompile(`(\d+\.\d+\.\d+-\d+pre)`) + captureVersion := regexp.MustCompile(`(\d+\.\d+\.\d+-\d+)`) if captureVersion.MatchString(files[i]) { latestVersion = captureVersion.FindString(files[i]) break diff --git a/pkg/build/git.go b/pkg/build/git.go index 6c86d74e004..f955d5b07c1 100644 --- a/pkg/build/git.go +++ b/pkg/build/git.go @@ -15,3 +15,16 @@ func getGitSha() string { } return string(v) } + +func getGitEnterpriseSha() string { + // supporting the old way of dev setup + v, err := runError("git", "-C", "../grafana-enterprise", "rev-parse", "--short", "HEAD") + if err != nil { + // supporting the new way of dev setup + v, err = runError("git", "-C", "..", "rev-parse", "--short", "HEAD") + if err != nil { + return "" + } + } + return string(v) +} diff --git a/pkg/build/grafana/build.go b/pkg/build/grafana/build.go index 0596d094875..586b2d22839 100644 --- a/pkg/build/grafana/build.go +++ b/pkg/build/grafana/build.go @@ -23,13 +23,19 @@ const ( ) func GrafanaLDFlags(version string, r config.Revision) []string { - return []string{ + cmd := []string{ "-w", fmt.Sprintf("-X main.version=%s", version), fmt.Sprintf("-X main.commit=%s", r.SHA256), fmt.Sprintf("-X main.buildstamp=%d", r.Timestamp), fmt.Sprintf("-X main.buildBranch=%s", r.Branch), } + + if r.EnterpriseCommit != "" { + cmd = append(cmd, fmt.Sprintf("-X main.enterpriseCommit=%s", r.EnterpriseCommit)) + } + + return cmd } // BinaryFolder returns the path to where the Grafana binary is build given the provided arguments. diff --git a/pkg/build/lerna/lerna.go b/pkg/build/lerna/lerna.go index de04f5f0e88..3bd67a87b44 100644 --- a/pkg/build/lerna/lerna.go +++ b/pkg/build/lerna/lerna.go @@ -47,7 +47,7 @@ func GetLernaVersion(grafanaDir string) (string, error) { if err != nil { return "", fmt.Errorf("failed to read %q: %w", lernaJSONPath, err) } - pkgObj := map[string]interface{}{} + pkgObj := map[string]any{} if err := json.Unmarshal(lernaJSONB, &pkgObj); err != nil { return "", fmt.Errorf("failed decoding %q: %w", lernaJSONPath, err) } diff --git a/pkg/build/npm/npm.go b/pkg/build/npm/npm.go index 9799451c0fa..5b1a895ec49 100644 --- a/pkg/build/npm/npm.go +++ b/pkg/build/npm/npm.go @@ -21,11 +21,11 @@ const NpmArtifactDir = "./npm-artifacts" var packages = []string{ "@grafana/ui", "@grafana/data", - "@grafana/toolkit", "@grafana/runtime", "@grafana/e2e", "@grafana/e2e-selectors", "@grafana/schema", + "@grafana/flamegraph", } // PublishNpmPackages will publish local NPM packages to NPM registry. diff --git a/pkg/build/packaging/grafana.go b/pkg/build/packaging/grafana.go index 88fef5c6793..58f4dbf7209 100644 --- a/pkg/build/packaging/grafana.go +++ b/pkg/build/packaging/grafana.go @@ -533,7 +533,7 @@ func copyPlugins(ctx context.Context, v config.Variant, grafanaDir, tmpDir strin if err != nil { return fmt.Errorf("failed to read %q: %w", filepath.Join(srcDir, "plugin.json"), err) } - var plugJSON map[string]interface{} + var plugJSON map[string]any if err := json.Unmarshal(jsonB, &plugJSON); err != nil { return err } diff --git a/public/app/features/alerting/unified/components/rule-editor/notificaton-preview/NotificationPreview.test.tsx b/public/app/features/alerting/unified/components/rule-editor/notificaton-preview/NotificationPreview.test.tsx index e9f00220b51..23169a7a4b9 100644 --- a/public/app/features/alerting/unified/components/rule-editor/notificaton-preview/NotificationPreview.test.tsx +++ b/public/app/features/alerting/unified/components/rule-editor/notificaton-preview/NotificationPreview.test.tsx @@ -139,7 +139,7 @@ describe('NotificationPreview', () => { mockOneAlertManager(); mockPreviewApiResponse(server, [{ labels: [{ tomato: 'red', avocate: 'green' }] }]); - render(, { + render(, { wrapper: TestProvider, }); @@ -147,13 +147,18 @@ describe('NotificationPreview', () => { await waitFor(() => { expect(ui.loadingIndicator.query()).not.toBeInTheDocument(); }); - // we expect the alert manager label to be missing as there is only one alert manager configured to receive alerts - expect(ui.grafanaAlertManagerLabel.query()).not.toBeInTheDocument(); - expect(ui.otherAlertManagerLabel.query()).not.toBeInTheDocument(); - const matchingPoliciesElements = ui.route.queryAll(); - expect(matchingPoliciesElements).toHaveLength(1); - expect(matchingPoliciesElements[0]).toHaveTextContent(/tomato = red/); + // we expect the alert manager label to be missing as there is only one alert manager configured to receive alerts + await waitFor(() => { + expect(ui.grafanaAlertManagerLabel.query()).not.toBeInTheDocument(); + expect(ui.otherAlertManagerLabel.query()).not.toBeInTheDocument(); + }); + + await waitFor(() => { + const matchingPoliciesElements = ui.route.queryAll(); + expect(matchingPoliciesElements).toHaveLength(1); + expect(matchingPoliciesElements[0]).toHaveTextContent(/tomato = red/); + }); }); it('should render notification preview with alert manager sections, when having more than one alert manager configured to receive alerts', async () => { // two alert managers configured to receive alerts diff --git a/scripts/drone/TAGS b/scripts/drone/TAGS deleted file mode 100644 index 1c9f9da89a5..00000000000 --- a/scripts/drone/TAGS +++ /dev/null @@ -1,628 +0,0 @@ - -events/release.star,6652 -ver_mode =ver_mode64,1602 -release_trigger =release_trigger65,1623 -def store_npm_packages_step():store_npm_packages_step74,1752 -def retrieve_npm_packages_step():retrieve_npm_packages_step90,2193 -def release_npm_packages_step():release_npm_packages_step107,2663 -def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):oss_pipelines123,3076 - environment =environment135,3492 - edition =edition136,3529 - services =services137,3549 - volumes =volumes138,3609 - package_steps =package_steps139,3659 - publish_steps =publish_steps140,3682 - should_publish =should_publish141,3705 - should_upload =should_upload142,3748 - init_steps =init_steps143,3818 - build_steps =build_steps152,4033 - integration_test_steps =integration_test_steps159,4342 - build_storybook =build_storybook182,5254 - publish_step =publish_step190,5674 - store_npm_step =store_npm_step191,5758 - windows_package_steps =windows_package_steps196,5957 - windows_pipeline =windows_pipeline198,6044 - name =name199,6077 - edition =edition200,6127 - trigger =trigger201,6154 - steps =steps202,6181 - platform =platform203,6256 - depends_on =depends_on204,6286 - environment =environment207,6393 - pipelines =pipelines209,6434 - name =name211,6470 - edition =edition212,6550 - trigger =trigger213,6581 - services =services214,6612 - steps =steps215,6639 - environment =environment216,6717 - volumes =volumes217,6756 - name =name225,6970 - edition =edition226,7038 - trigger =trigger227,7073 - services =services228,7108 - steps =steps229,7145 - environment =environment230,7329 - volumes =volumes231,7372 - deps =deps234,7433 -def enterprise_pipelines(ver_mode = ver_mode, trigger = release_trigger):enterprise_pipelines247,7856 - environment =environment259,8284 - edition =edition260,8328 - services =services261,8355 - volumes =volumes262,8415 - package_steps =package_steps263,8465 - publish_steps =publish_steps264,8488 - should_publish =should_publish265,8511 - should_upload =should_upload266,8554 - include_enterprise =include_enterprise267,8624 - edition2 =edition2268,8673 - init_steps =init_steps269,8702 - build_steps =build_steps277,8909 - integration_test_steps =integration_test_steps284,9218 - build_storybook =build_storybook312,10299 - publish_step =publish_step324,10892 - store_npm_step =store_npm_step325,10976 - windows_package_steps =windows_package_steps330,11175 - step =step333,11284 - deps_on_clone_enterprise_step =deps_on_clone_enterprise_step337,11418 - windows_pipeline =windows_pipeline347,11746 - name =name348,11779 - edition =edition349,11836 - trigger =trigger350,11863 - steps =steps351,11890 - platform =platform352,11965 - depends_on =depends_on353,11995 - environment =environment356,12109 - pipelines =pipelines358,12150 - name =name360,12186 - edition =edition361,12273 - trigger =trigger362,12304 - services =services363,12335 - steps =steps364,12362 - environment =environment365,12440 - volumes =volumes366,12479 - name =name374,12711 - edition =edition375,12786 - trigger =trigger376,12821 - services =services377,12856 - steps =steps378,12893 - environment =environment379,13213 - volumes =volumes380,13256 - deps =deps383,13317 -def enterprise2_pipelines(prefix = "", ver_mode = ver_mode, trigger = release_trigger):enterprise2_pipelines397,13769 - environment =environment412,14364 - edition =edition415,14424 - volumes =volumes416,14451 - package_steps =package_steps417,14501 - publish_steps =publish_steps418,14524 - should_publish =should_publish419,14547 - should_upload =should_upload420,14590 - include_enterprise =include_enterprise421,14660 - edition2 =edition2422,14709 - init_steps =init_steps423,14738 - build_steps =build_steps431,14945 - fetch_images =fetch_images442,15355 - upload_cdn =upload_cdn444,15497 - step =step458,16187 - deps_on_clone_enterprise_step =deps_on_clone_enterprise_step462,16321 - pipelines =pipelines472,16608 - name =name474,16644 - edition =edition475,16742 - trigger =trigger476,16773 - services =services477,16804 - steps =steps478,16831 - volumes =volumes479,16909 - environment =environment480,16940 -def publish_artifacts_step(mode):publish_artifacts_step486,17019 - security =security487,17053 - security =security489,17098 -def publish_artifacts_pipelines(mode):publish_artifacts_pipelines501,17538 - trigger =trigger502,17577 - steps =steps506,17655 - name =name512,17768 - trigger =trigger513,17820 - steps =steps514,17847 - edition =edition515,17870 - environment =environment516,17895 -def publish_packages_pipeline():publish_packages_pipeline519,17945 - trigger =trigger526,18162 - oss_steps =oss_steps530,18244 - enterprise_steps =enterprise_steps538,18560 - deps =deps545,18903 - name =name552,19062 - trigger =trigger553,19101 - steps =steps554,19128 - edition =edition555,19155 - depends_on =depends_on556,19180 - environment =environment557,19207 - name =name559,19266 - trigger =trigger560,19312 - steps =steps561,19339 - edition =edition562,19373 - depends_on =depends_on563,19398 - environment =environment564,19425 -def publish_npm_pipelines(mode):publish_npm_pipelines567,19482 - trigger =trigger568,19515 - steps =steps572,19593 - name =name580,19772 - trigger =trigger581,19827 - steps =steps582,19854 - edition =edition583,19877 - environment =environment584,19902 -def artifacts_page_pipeline():artifacts_page_pipeline587,19952 - trigger =trigger588,19983 - name =name593,20087 - trigger =trigger594,20128 - steps =steps595,20155 - edition =edition596,20220 - environment =environment597,20245 -def get_e2e_suffix():get_e2e_suffix600,20295 - -events/cron.star,1016 -aquasec_trivy_image =aquasec_trivy_image8,209 -def cronjobs(edition):cronjobs10,255 - grafana_com_nightly_pipeline =grafana_com_nightly_pipeline11,278 - cronName =cronName12,332 - name =name13,374 - steps =steps14,412 -def cron_job_pipeline(cronName, name, steps):cron_job_pipeline24,773 -def scan_docker_image_pipeline(edition, tag):scan_docker_image_pipeline43,1175 - edition =edition55,1530 - edition =edition57,1579 - docker_image =docker_image59,1608 - cronName =cronName62,1695 - name =name63,1725 - steps =steps64,1775 -def scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image):scan_docker_image_unkown_low_medium_vulnerabilities_step71,2047 -def scan_docker_image_high_critical_vulnerabilities_step(docker_image):scan_docker_image_high_critical_vulnerabilities_step80,2353 -def slack_job_failed_step(channel, image):slack_job_failed_step89,2646 -def post_to_grafana_com_step():post_to_grafana_com_step103,3069 - -events/main.star,633 -ver_mode =ver_mode49,966 -trigger =trigger50,984 -def main_pipelines(edition):main_pipelines62,1168 - drone_change_trigger =drone_change_trigger63,1197 - pipelines =pipelines79,1513 - name =name89,1951 - slack_channel =slack_channel90,1994 - trigger =trigger91,2045 - template =template92,2089 - secret =secret93,2135 - name =name97,2276 - slack_channel =slack_channel98,2310 - trigger =trigger99,2366 - depends_on =depends_on100,2425 - template =template101,2563 - secret =secret102,2604 - -events/pr.star,252 -ver_mode =ver_mode48,997 -trigger =trigger49,1013 -def pr_pipelines(edition):pr_pipelines62,1198 -def get_pr_trigger(include_paths = None, exclude_paths = None):get_pr_trigger76,2396 - paths_ex =paths_ex91,3080 - paths_in =paths_in92,3115 - -services/services.star,225 -def integration_test_services_volumes():integration_test_services_volumes5,79 -def integration_test_services(edition):integration_test_services14,292 - services =services15,332 -def ldap_service():ldap_service59,1616 - -utils/utils.star,561 -failure_template =failure_template11,191 -drone_change_template =drone_change_template12,509 - services =services19,932 - platform =platform20,955 - depends_on =depends_on21,983 - environment =environment22,1008 - volumes =volumes23,1036 - platform_conf =platform_conf50,2166 - platform_conf =platform_conf62,2534 - pipeline =pipeline70,2713 -def notify_pipeline(name, slack_channel, trigger, depends_on = [], template = None, secret = None):notify_pipeline105,3545 - trigger =trigger106,3645 - -pipelines/trigger_downstream.star,440 -trigger =trigger14,249 -def enterprise_downstream_pipeline(edition, ver_mode):enterprise_downstream_pipeline26,433 - environment =environment27,488 - steps =steps28,527 - deps =deps29,587 - name =name31,672 - edition =edition32,714 - trigger =trigger33,741 - services =services34,768 - steps =steps35,791 - depends_on =depends_on36,814 - environment =environment37,841 - -pipelines/verify_starlark.star,323 -def verify_starlark(trigger, ver_mode):verify_starlark17,305 - environment =environment18,345 - steps =steps19,382 - name =name26,546 - edition =edition27,600 - trigger =trigger28,625 - services =services29,652 - steps =steps30,675 - environment =environment31,698 - -pipelines/build.star,508 -def build_e2e(trigger, ver_mode, edition):build_e2e39,936 - environment =environment50,1096 - variants =variants51,1135 - init_steps =init_steps52,1219 - build_steps =build_steps61,1491 - publish_suffix =publish_suffix107,4049 - publish_suffix =publish_suffix109,4100 - name =name112,4158 - edition =edition113,4224 - environment =environment114,4249 - services =services115,4284 - steps =steps116,4307 - trigger =trigger117,4349 - -pipelines/shellcheck.star,386 -trigger =trigger15,235 -def shellcheck_step():shellcheck_step31,483 -def shellcheck_pipeline():shellcheck_pipeline43,725 - environment =environment44,752 - steps =steps45,789 - name =name50,886 - edition =edition51,918 - trigger =trigger52,943 - services =services53,970 - steps =steps54,993 - environment =environment55,1016 - -pipelines/verify_drone.star,317 -def verify_drone(trigger, ver_mode):verify_drone17,293 - environment =environment18,330 - steps =steps19,367 - name =name26,528 - edition =edition27,579 - trigger =trigger28,604 - services =services29,631 - steps =steps30,654 - environment =environment31,677 - -pipelines/test_backend.star,474 -def test_backend(trigger, ver_mode, edition = "oss"):test_backend23,463 - environment =environment35,882 - init_steps =init_steps36,921 - test_steps =test_steps46,1291 - pipeline_name =pipeline_name51,1387 - pipeline_name =pipeline_name53,1492 - name =name55,1584 - edition =edition56,1614 - trigger =trigger57,1641 - services =services58,1668 - steps =steps59,1691 - environment =environment60,1732 - -pipelines/lint_frontend.star,415 -def lint_frontend_pipeline(trigger, ver_mode):lint_frontend_pipeline16,260 - environment =environment26,546 - yarn_step =yarn_step27,583 - init_steps =init_steps29,660 - test_steps =test_steps33,736 - name =name37,812 - edition =edition38,864 - trigger =trigger39,889 - services =services40,916 - steps =steps41,939 - environment =environment42,980 - -pipelines/docs.star,494 -docs_paths =docs_paths19,383 -def docs_pipelines(edition, ver_mode, trigger):docs_pipelines28,511 - environment =environment29,559 - steps =steps30,598 - name =name40,815 - edition =edition41,858 - trigger =trigger42,885 - services =services43,912 - steps =steps44,935 - environment =environment45,958 -def lint_docs():lint_docs48,1000 -def trigger_docs_main():trigger_docs_main63,1328 -def trigger_docs_pr():trigger_docs_pr72,1478 - -pipelines/test_frontend.star,476 -def test_frontend(trigger, ver_mode, edition = "oss"):test_frontend20,374 - environment =environment32,794 - init_steps =init_steps33,833 - test_steps =test_steps41,1102 - pipeline_name =pipeline_name45,1205 - pipeline_name =pipeline_name47,1311 - name =name49,1404 - edition =edition50,1434 - trigger =trigger51,1461 - services =services52,1488 - steps =steps53,1511 - environment =environment54,1552 - -pipelines/integration_tests.star,483 -def integration_tests(trigger, ver_mode, edition):integration_tests26,542 - environment =environment37,900 - services =services38,939 - volumes =volumes39,989 - init_steps =init_steps40,1039 - test_steps =test_steps48,1282 - name =name54,1412 - edition =edition55,1468 - trigger =trigger56,1493 - services =services57,1520 - steps =steps58,1549 - environment =environment59,1590 - volumes =volumes60,1625 - -pipelines/windows.star,954 -def windows(trigger, edition, ver_mode):windows17,339 - environment =environment29,798 - init_cmds =init_cmds30,837 - steps =steps38,1205 - bucket =bucket49,1497 - ver_part =ver_part51,1590 - dir =dir52,1628 - dir =dir54,1670 - bucket =bucket55,1695 - build_no =build_no56,1736 - ver_part =ver_part57,1780 - installer_commands =installer_commands58,1842 - committish =committish100,3763 - committish =committish102,3846 - committish =committish104,3906 - download_grabpl_step_cmds =download_grabpl_step_cmds107,4057 - clone_cmds =clone_cmds113,4363 - name =name146,5711 - edition =edition147,5742 - trigger =trigger148,5769 - steps =steps149,5830 - depends_on =depends_on150,5889 - platform =platform151,6007 - environment =environment152,6037 - -pipelines/lint_backend.star,418 -def lint_backend_pipeline(trigger, ver_mode):lint_backend_pipeline18,306 - environment =environment28,590 - wire_step =wire_step29,627 - init_steps =init_steps31,704 - test_steps =test_steps36,809 - name =name43,959 - edition =edition44,1010 - trigger =trigger45,1035 - services =services46,1062 - steps =steps47,1085 - environment =environment48,1126 - -pipelines/publish_images.star,998 -def publish_image_steps(edition, mode, docker_repo):publish_image_steps17,303 - additional_docker_repo =additional_docker_repo31,922 - additional_docker_repo =additional_docker_repo33,979 - steps =steps34,1034 -def publish_image_pipelines_public():publish_image_pipelines_public45,1369 - mode =mode51,1521 - trigger =trigger52,1541 - name =name57,1641 - trigger =trigger58,1694 - steps =steps59,1721 - edition =edition60,1813 - environment =environment61,1835 - name =name63,1894 - trigger =trigger64,1954 - steps =steps65,1981 - edition =edition66,2091 - environment =environment67,2113 -def publish_image_pipelines_security():publish_image_pipelines_security70,2170 - mode =mode71,2210 - trigger =trigger72,2232 - name =name77,2332 - trigger =trigger78,2392 - steps =steps79,2419 - edition =edition80,2529 - environment =environment81,2551 - -steps/lib.star,8579 -grabpl_version =grabpl_version7,181 -build_image =build_image8,208 -publish_image =publish_image9,254 -deploy_docker_image =deploy_docker_image10,304 -alpine_image =alpine_image11,380 -curl_image =curl_image12,411 -windows_image =windows_image13,452 -wix_image =wix_image14,501 -go_image =go_image15,536 -disable_tests =disable_tests17,564 -trigger_oss =trigger_oss18,586 -def slack_step(channel, template, secret):slack_step24,653 -def yarn_install_step(edition = "oss"):yarn_install_step35,918 - deps =deps36,958 - deps =deps38,1004 -def wire_install_step():wire_install_step48,1222 -def identify_runner_step(platform = "linux"):identify_runner_step60,1454 -def clone_enterprise_step(ver_mode):clone_enterprise_step78,1916 - committish =committish87,2193 - committish =committish89,2268 - committish =committish91,2317 -def init_enterprise_step(ver_mode):init_enterprise_step105,2747 - source_commit =source_commit115,3098 - source_commit =source_commit117,3151 - environment =environment118,3191 - token =token121,3280 - environment =environment123,3369 - token =token126,3458 - environment =environment128,3518 - token =token129,3543 -def download_grabpl_step(platform = "linux"):download_grabpl_step148,4147 -def lint_drone_step():lint_drone_step173,4973 -def lint_starlark_step():lint_starlark_step185,5216 -def enterprise_downstream_step(edition, ver_mode):enterprise_downstream_step206,6000 - repo =repo219,6482 - step =step225,6623 -def lint_backend_step():lint_backend_step247,7248 -def benchmark_ldap_step():benchmark_ldap_step265,7713 -def build_storybook_step(edition, ver_mode):build_storybook_step278,8087 -def store_storybook_step(edition, ver_mode, trigger = None):store_storybook_step300,8743 - commands =commands314,9202 - commands =commands323,9521 - step =step325,9593 - when_cond =when_cond338,10125 - step =step346,10330 -def e2e_tests_artifacts(edition):e2e_tests_artifacts349,10391 -def upload_cdn_step(edition, ver_mode, trigger = None):upload_cdn_step386,12378 - deps =deps397,12763 - step =step407,12970 - step =step420,13423 -def build_backend_step(edition, ver_mode, variants = None):build_backend_step423,13482 - variants_str =variants_str437,14070 - variants_str =variants_str439,14109 - cmds =cmds443,14256 - build_no =build_no449,14418 - cmds =cmds450,14461 -def build_frontend_step(edition, ver_mode):build_frontend_step468,14906 - build_no =build_no478,15246 - cmds =cmds482,15356 - cmds =cmds487,15505 -def build_frontend_package_step(edition, ver_mode):build_frontend_package_step505,15960 - build_no =build_no515,16312 - cmds =cmds519,16422 - cmds =cmds524,16580 -def build_plugins_step(edition, ver_mode):build_plugins_step542,17053 - env =env544,17121 - env =env548,17220 -def test_backend_step():test_backend_step563,17607 -def test_backend_integration_step():test_backend_integration_step575,17880 -def betterer_frontend_step(edition = "oss"):betterer_frontend_step587,18187 - deps =deps596,18427 -def test_frontend_step(edition = "oss"):test_frontend_step609,18728 - deps =deps618,18962 -def lint_frontend_step():lint_frontend_step634,19343 -def test_a11y_frontend_step(ver_mode, edition, port = 3001):test_a11y_frontend_step652,19793 - commands =commands664,20279 - failure =failure667,20345 - failure =failure672,20483 -def frontend_metrics_step(edition, trigger = None):frontend_metrics_step693,21146 - step =step706,21507 - step =step721,22007 -def codespell_step():codespell_step724,22066 -def package_step(edition, ver_mode, variants = None):package_step736,22468 - deps =deps750,23006 - variants_str =variants_str757,23167 - variants_str =variants_str759,23206 - sign_args =sign_args762,23332 - env =env763,23362 - test_args =test_args769,23628 - sign_args =sign_args771,23661 - env =env772,23684 - test_args =test_args773,23703 - cmds =cmds777,23829 - build_no =build_no784,24036 - cmds =cmds785,24079 -def grafana_server_step(edition, port = 3001):grafana_server_step798,24459 - package_file_pfx =package_file_pfx808,24729 - package_file_pfx =package_file_pfx810,24788 - package_file_pfx =package_file_pfx812,24889 - environment =environment814,24938 -def e2e_tests_step(suite, edition, port = 3001, tries = None):e2e_tests_step837,25554 - cmd =cmd838,25617 -def cloud_plugins_e2e_tests_step(suite, edition, cloud, trigger = None):cloud_plugins_e2e_tests_step856,26186 - environment =environment869,26649 - when =when870,26670 - when =when872,26700 - environment =environment874,26748 - when =when882,27129 - branch =branch888,27345 - step =step889,27401 - step =step901,27822 -def build_docs_website_step():build_docs_website_step904,27874 -def copy_packages_for_docker_step(edition = None):copy_packages_for_docker_step916,28272 -def build_docker_images_step(edition, archs = None, ubuntu = False, publish = False):build_docker_images_step929,28622 - cmd =cmd943,29193 - ubuntu_sfx =ubuntu_sfx947,29307 - ubuntu_sfx =ubuntu_sfx949,29342 - environment =environment955,29468 -def fetch_images_step(edition):fetch_images_step979,30079 -def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None):publish_images_step997,30745 - name =name1013,31562 - docker_repo =docker_repo1014,31585 - mode =mode1016,31663 - mode =mode1018,31709 - environment =environment1020,31728 - cmd =cmd1026,31912 - deps =deps1029,32041 - deps =deps1032,32147 - name =name1035,32250 - docker_repo =docker_repo1036,32273 - cmd =cmd1038,32459 - step =step1040,32565 - step =step1052,32929 -def postgres_integration_tests_step():postgres_integration_tests_step1056,32989 - cmds =cmds1057,33028 -def mysql_integration_tests_step():mysql_integration_tests_step1079,33850 - cmds =cmds1080,33886 -def redis_integration_tests_step():redis_integration_tests_step1100,34629 -def memcached_integration_tests_step():memcached_integration_tests_step1114,35026 -def release_canary_npm_packages_step(edition, trigger = None):release_canary_npm_packages_step1128,35435 - step =step1141,35805 - step =step1153,36143 -def enterprise2_suffix(edition):enterprise2_suffix1156,36202 -def upload_packages_step(edition, ver_mode, trigger = None):upload_packages_step1161,36320 - deps =deps1176,36816 - step =step1184,37036 - step =step1195,37471 -def publish_grafanacom_step(edition, ver_mode):publish_grafanacom_step1198,37530 - cmd =cmd1211,38044 - build_no =build_no1215,38188 - cmd =cmd1216,38231 -def publish_linux_packages_step(edition, package_manager = "deb"):publish_linux_packages_step1239,38866 -def get_windows_steps(edition, ver_mode):get_windows_steps1261,39989 - init_cmds =init_cmds1270,40281 - steps =steps1278,40649 - bucket =bucket1289,40941 - ver_part =ver_part1291,41034 - dir =dir1292,41072 - dir =dir1294,41114 - bucket =bucket1295,41139 - build_no =build_no1296,41180 - ver_part =ver_part1297,41224 - installer_commands =installer_commands1298,41286 - committish =committish1340,43207 - committish =committish1342,43290 - committish =committish1344,43350 - download_grabpl_step_cmds =download_grabpl_step_cmds1347,43501 - clone_cmds =clone_cmds1353,43807 -def verify_gen_cue_step(edition):verify_gen_cue_step1387,45152 - deps =deps1388,45186 -def verify_gen_jsonnet_step(edition):verify_gen_jsonnet_step1402,45694 - deps =deps1403,45732 -def trigger_test_release():trigger_test_release1417,46236 -def artifacts_page_step():artifacts_page_step1451,47731 -def end_to_end_tests_deps():end_to_end_tests_deps1466,48058 -def compile_build_cmd(edition = "oss"):compile_build_cmd1476,48321 - dependencies =dependencies1477,48361 - dependencies =dependencies1479,48432 -def get_trigger_storybook(ver_mode):get_trigger_storybook1492,48780 - trigger_storybook =trigger_storybook1500,49031 - trigger_storybook =trigger_storybook1502,49088 - trigger_storybook =trigger_storybook1506,49168 - -vault.star,444 -pull_secret =pull_secret4,87 -github_token =github_token5,120 -drone_token =drone_token6,150 -prerelease_bucket =prerelease_bucket7,178 -gcp_upload_artifacts_key =gcp_upload_artifacts_key8,218 -azure_sp_app_id =azure_sp_app_id9,272 -azure_sp_app_pw =azure_sp_app_pw10,308 -azure_tenant =azure_tenant11,344 -def from_secret(secret):from_secret13,375 -def vault_secret(name, path, key):vault_secret18,451 -def secrets():secrets28,633 - -version.star,116 -ver_mode =ver_mode12,197 -trigger =trigger13,225 -def version_branch_pipelines():version_branch_pipelines15,268 diff --git a/scripts/drone/events/cron.star b/scripts/drone/events/cron.star index 6715c175b17..6f9e524139b 100644 --- a/scripts/drone/events/cron.star +++ b/scripts/drone/events/cron.star @@ -2,15 +2,11 @@ This module provides functions for cronjob pipelines and steps used within. """ -load("scripts/drone/vault.star", "from_secret") -load( - "scripts/drone/steps/lib.star", - "compile_build_cmd", -) load( "scripts/drone/utils/images.star", "images", ) +load("scripts/drone/vault.star", "from_secret") aquasec_trivy_image = "aquasec/trivy:0.21.0" @@ -21,7 +17,6 @@ def cronjobs(): scan_docker_image_pipeline("latest-ubuntu"), scan_docker_image_pipeline("main-ubuntu"), scan_build_test_publish_docker_image_pipeline(), - grafana_com_nightly_pipeline(), ] def authenticate_gcr_step(): @@ -59,6 +54,10 @@ def cron_job_pipeline(cronName, name, steps): "path": "/var/run/docker.sock", }, }, + { + "name": "config", + "temp": {}, + }, ], } @@ -117,13 +116,13 @@ def scan_docker_image_unknown_low_medium_vulnerabilities_step(docker_image): for key in images: cmds = cmds + ["trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + images[key]] else: - cmds = ["trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image] + cmds = ["trivy image --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image] return { "name": "scan-unknown-low-medium-vulnerabilities", "image": aquasec_trivy_image, "commands": cmds, "depends_on": ["authenticate-gcr"], - "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}, {"name": "config", "path": "/root/.docker/"}], } def scan_docker_image_high_critical_vulnerabilities_step(docker_image): @@ -141,19 +140,22 @@ def scan_docker_image_high_critical_vulnerabilities_step(docker_image): for key in images: cmds = cmds + ["trivy --exit-code 1 --severity HIGH,CRITICAL " + images[key]] else: - cmds = ["trivy --exit-code 1 --severity HIGH,CRITICAL " + docker_image] + cmds = ["trivy image --exit-code 1 --severity HIGH,CRITICAL " + docker_image] return { "name": "scan-high-critical-vulnerabilities", "image": aquasec_trivy_image, "commands": cmds, "depends_on": ["authenticate-gcr"], - "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + "environment": { + "GOOGLE_APPLICATION_CREDENTIALS": from_secret("gcr_credentials_json"), + }, + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}, {"name": "config", "path": "/root/.docker/"}], } def slack_job_failed_step(channel, image): return { "name": "slack-notify-failure", - "image": images["plugins_slack_image"], + "image": images["plugins_slack"], "settings": { "webhook": from_secret("slack_webhook_backend"), "channel": channel, @@ -167,7 +169,7 @@ def slack_job_failed_step(channel, image): def post_to_grafana_com_step(): return { "name": "post-to-grafana-com", - "image": images["publish_image"], + "image": images["publish"], "environment": { "GRAFANA_COM_API_KEY": from_secret("grafana_api_key"), "GCP_KEY": from_secret("gcp_key"), @@ -175,13 +177,3 @@ def post_to_grafana_com_step(): "depends_on": ["compile-build-cmd"], "commands": ["./bin/build publish grafana-com --edition oss"], } - -def grafana_com_nightly_pipeline(): - return cron_job_pipeline( - cronName = "grafana-com-nightly", - name = "grafana-com-nightly", - steps = [ - compile_build_cmd(), - post_to_grafana_com_step(), - ], - ) diff --git a/scripts/drone/events/main.star b/scripts/drone/events/main.star index b37b1893229..ee47392ee5c 100644 --- a/scripts/drone/events/main.star +++ b/scripts/drone/events/main.star @@ -3,40 +3,18 @@ This module returns all the pipelines used in the event of pushes to the main br """ load( - "scripts/drone/utils/utils.star", - "drone_change_template", - "failure_template", - "notify_pipeline", + "scripts/drone/pipelines/build.star", + "build_e2e", ) load( "scripts/drone/pipelines/docs.star", "docs_pipelines", "trigger_docs_main", ) -load( - "scripts/drone/pipelines/test_frontend.star", - "test_frontend", -) -load( - "scripts/drone/pipelines/test_backend.star", - "test_backend", -) load( "scripts/drone/pipelines/integration_tests.star", "integration_tests", ) -load( - "scripts/drone/pipelines/build.star", - "build_e2e", -) -load( - "scripts/drone/pipelines/windows.star", - "windows", -) -load( - "scripts/drone/pipelines/trigger_downstream.star", - "enterprise_downstream_pipeline", -) load( "scripts/drone/pipelines/lint_backend.star", "lint_backend_pipeline", @@ -45,6 +23,27 @@ load( "scripts/drone/pipelines/lint_frontend.star", "lint_frontend_pipeline", ) +load( + "scripts/drone/pipelines/test_backend.star", + "test_backend", +) +load( + "scripts/drone/pipelines/test_frontend.star", + "test_frontend", +) +load( + "scripts/drone/pipelines/trigger_downstream.star", + "enterprise_downstream_pipeline", +) +load( + "scripts/drone/pipelines/windows.star", + "windows", +) +load( + "scripts/drone/utils/utils.star", + "failure_template", + "notify_pipeline", +) ver_mode = "main" trigger = { @@ -65,24 +64,6 @@ trigger = { } def main_pipelines(): - drone_change_trigger = { - "event": [ - "push", - ], - "branch": "main", - "repo": [ - "grafana/grafana", - ], - "paths": { - "include": [ - ".drone.yml", - ], - "exclude": [ - "exclude", - ], - }, - } - pipelines = [ docs_pipelines(ver_mode, trigger_docs_main()), test_frontend(trigger, ver_mode), @@ -92,13 +73,6 @@ def main_pipelines(): build_e2e(trigger, ver_mode), integration_tests(trigger, prefix = ver_mode, ver_mode = ver_mode), windows(trigger, ver_mode = ver_mode), - notify_pipeline( - name = "notify-drone-changes", - slack_channel = "slack-webhooks-test", - trigger = drone_change_trigger, - template = drone_change_template, - secret = "drone-changes-webhook", - ), enterprise_downstream_pipeline(), notify_pipeline( name = "main-notify", diff --git a/scripts/drone/events/pr.star b/scripts/drone/events/pr.star index 5acf2cd2bb6..cbd425efcf6 100644 --- a/scripts/drone/events/pr.star +++ b/scripts/drone/events/pr.star @@ -4,37 +4,21 @@ It also includes a function generating a PR trigger from a list of included and """ load( - "scripts/drone/pipelines/test_frontend.star", - "test_frontend", -) -load( - "scripts/drone/pipelines/test_backend.star", - "test_backend", -) -load( - "scripts/drone/pipelines/integration_tests.star", - "integration_tests", + "scripts/drone/pipelines/benchmarks.star", + "integration_benchmarks", ) load( "scripts/drone/pipelines/build.star", "build_e2e", ) -load( - "scripts/drone/pipelines/verify_drone.star", - "verify_drone", -) -load( - "scripts/drone/pipelines/verify_starlark.star", - "verify_starlark", -) load( "scripts/drone/pipelines/docs.star", "docs_pipelines", "trigger_docs_pr", ) load( - "scripts/drone/pipelines/shellcheck.star", - "shellcheck_pipeline", + "scripts/drone/pipelines/integration_tests.star", + "integration_tests", ) load( "scripts/drone/pipelines/lint_backend.star", @@ -45,8 +29,24 @@ load( "lint_frontend_pipeline", ) load( - "scripts/drone/pipelines/benchmarks.star", - "integration_benchmarks", + "scripts/drone/pipelines/shellcheck.star", + "shellcheck_pipeline", +) +load( + "scripts/drone/pipelines/test_backend.star", + "test_backend", +) +load( + "scripts/drone/pipelines/test_frontend.star", + "test_frontend", +) +load( + "scripts/drone/pipelines/verify_drone.star", + "verify_drone", +) +load( + "scripts/drone/pipelines/verify_starlark.star", + "verify_starlark", ) ver_mode = "pr" diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index 295e4a922fa..5e71f1eed04 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -2,55 +2,36 @@ This module returns all the pipelines used in the event of a release along with supporting functions. """ -load( - "scripts/drone/steps/lib.star", - "build_backend_step", - "build_docker_images_step", - "build_frontend_package_step", - "build_frontend_step", - "build_plugins_step", - "build_storybook_step", - "compile_build_cmd", - "copy_packages_for_docker_step", - "download_grabpl_step", - "e2e_tests_artifacts", - "e2e_tests_step", - "get_windows_steps", - "grafana_server_step", - "identify_runner_step", - "memcached_integration_tests_step", - "mysql_integration_tests_step", - "package_step", - "postgres_integration_tests_step", - "publish_grafanacom_step", - "publish_linux_packages_step", - "redis_integration_tests_step", - "store_storybook_step", - "trigger_oss", - "upload_cdn_step", - "upload_packages_step", - "verify_gen_cue_step", - "verify_gen_jsonnet_step", - "wire_install_step", - "yarn_install_step", -) load( "scripts/drone/services/services.star", "integration_test_services", "integration_test_services_volumes", ) +load( + "scripts/drone/steps/lib.star", + "compile_build_cmd", + "download_grabpl_step", + "identify_runner_step", + "memcached_integration_tests_steps", + "mysql_integration_tests_steps", + "postgres_integration_tests_steps", + "publish_grafanacom_step", + "publish_linux_packages_step", + "redis_integration_tests_steps", + "remote_alertmanager_integration_tests_steps", + "verify_gen_cue_step", + "verify_gen_jsonnet_step", + "wire_install_step", + "yarn_install_step", +) +load( + "scripts/drone/utils/images.star", + "images", +) load( "scripts/drone/utils/utils.star", "pipeline", ) -load( - "scripts/drone/pipelines/test_frontend.star", - "test_frontend", -) -load( - "scripts/drone/pipelines/test_backend.star", - "test_backend", -) load( "scripts/drone/vault.star", "from_secret", @@ -59,51 +40,13 @@ load( "prerelease_bucket", "rgm_gcp_key_base64", ) -load( - "scripts/drone/utils/images.star", - "images", -) -load( - "scripts/drone/pipelines/whats_new_checker.star", - "whats_new_checker_pipeline", -) ver_mode = "release" -release_trigger = { - "event": { - "exclude": [ - "promote", - ], - }, - "ref": { - "include": [ - "refs/tags/v*", - ], - "exclude": [ - "refs/tags/*-cloud*", - ], - }, -} - -def store_npm_packages_step(): - return { - "name": "store-npm-packages", - "image": images["build_image"], - "depends_on": [ - "compile-build-cmd", - "build-frontend-packages", - ], - "environment": { - "GCP_KEY": from_secret(gcp_grafanauploads_base64), - "PRERELEASE_BUCKET": from_secret(prerelease_bucket), - }, - "commands": ["./bin/build artifacts npm store --tag ${DRONE_TAG}"], - } def retrieve_npm_packages_step(): return { "name": "retrieve-npm-packages", - "image": images["publish_image"], + "image": images["publish"], "depends_on": [ "compile-build-cmd", "yarn-install", @@ -119,7 +62,7 @@ def retrieve_npm_packages_step(): def release_npm_packages_step(): return { "name": "release-npm-packages", - "image": images["build_image"], + "image": images["go"], "depends_on": [ "compile-build-cmd", "retrieve-npm-packages", @@ -131,146 +74,10 @@ def release_npm_packages_step(): "commands": ["./bin/build artifacts npm release --tag ${DRONE_TAG}"], } -def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger): - """Generates all pipelines used for Grafana OSS. - - Args: - ver_mode: controls which steps are included in the pipeline. - Defaults to 'release'. - trigger: controls which events can trigger the pipeline execution. - Defaults to tag events for tags with a 'v' prefix. - - Returns: - List of Drone pipelines. - """ - - environment = {"EDITION": "oss"} - - services = integration_test_services() - volumes = integration_test_services_volumes() - - init_steps = [ - identify_runner_step(), - download_grabpl_step(), - verify_gen_cue_step(), - wire_install_step(), - yarn_install_step(), - compile_build_cmd(), - ] - - build_steps = [ - build_backend_step(ver_mode = ver_mode), - build_frontend_step(ver_mode = ver_mode), - build_frontend_package_step(ver_mode = ver_mode), - build_plugins_step(ver_mode = ver_mode), - package_step(ver_mode = ver_mode), - copy_packages_for_docker_step(), - build_docker_images_step(publish = True), - build_docker_images_step( - publish = True, - ubuntu = True, - ), - grafana_server_step(), - e2e_tests_step("dashboards-suite", tries = 3), - e2e_tests_step("smoke-tests-suite", tries = 3), - e2e_tests_step("panels-suite", tries = 3), - e2e_tests_step("various-suite", tries = 3), - e2e_tests_artifacts(), - build_storybook_step(ver_mode = ver_mode), - ] - - publish_steps = [] - - if ver_mode in ( - "release", - "release-branch", - ): - publish_steps.extend( - [ - upload_cdn_step(ver_mode = ver_mode, trigger = trigger_oss), - upload_packages_step( - ver_mode = ver_mode, - trigger = trigger_oss, - ), - ], - ) - - if ver_mode in ("release",): - publish_steps.extend( - [ - store_storybook_step(ver_mode = ver_mode), - store_npm_packages_step(), - ], - ) - - integration_test_steps = [ - postgres_integration_tests_step(), - mysql_integration_tests_step("mysql57", "5.7"), - mysql_integration_tests_step("mysql80", "8.0"), - redis_integration_tests_step(), - memcached_integration_tests_step(), - ] - - pipelines = [] - - # We don't need to run integration tests at release time since they have - # been run multiple times before: - if ver_mode in ("release"): - pipelines.append(whats_new_checker_pipeline(release_trigger)) - integration_test_steps = [] - volumes = [] - - windows_pipeline_dependencies = [ - "{}-build-e2e-publish".format(ver_mode), - "{}-test-frontend".format(ver_mode), - ] - pipelines.extend([ - pipeline( - name = "{}-build-e2e-publish".format(ver_mode), - trigger = trigger, - services = [], - steps = init_steps + build_steps + publish_steps, - environment = environment, - volumes = volumes, - ), - test_frontend(trigger, ver_mode), - test_backend(trigger, ver_mode), - ]) - - if ver_mode not in ("release"): - pipelines.append(pipeline( - name = "{}-integration-tests".format(ver_mode), - trigger = trigger, - services = services, - steps = [ - download_grabpl_step(), - identify_runner_step(), - verify_gen_cue_step(), - verify_gen_jsonnet_step(), - wire_install_step(), - ] + - integration_test_steps, - environment = environment, - volumes = volumes, - )) - - windows_pipeline = pipeline( - name = "{}-windows".format(ver_mode), - trigger = trigger, - steps = get_windows_steps(ver_mode = ver_mode), - platform = "windows", - depends_on = windows_pipeline_dependencies, - environment = environment, - ) - - pipelines.append(windows_pipeline) - - return pipelines - def publish_artifacts_step(): return { "name": "publish-artifacts", - "image": images["publish_image"], + "image": images["publish"], "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -284,7 +91,7 @@ def publish_artifacts_step(): def publish_static_assets_step(): return { "name": "publish-static-assets", - "image": images["publish_image"], + "image": images["publish"], "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -299,7 +106,7 @@ def publish_static_assets_step(): def publish_storybook_step(): return { "name": "publish-storybook", - "image": images["publish_image"], + "image": images["publish"], "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -411,13 +218,12 @@ def integration_test_pipelines(): } pipelines = [] volumes = integration_test_services_volumes() - integration_test_steps = [ - postgres_integration_tests_step(), - mysql_integration_tests_step("mysql57", "5.7"), - mysql_integration_tests_step("mysql80", "8.0"), - redis_integration_tests_step(), - memcached_integration_tests_step(), - ] + integration_test_steps = postgres_integration_tests_steps() + \ + mysql_integration_tests_steps("mysql57", "5.7") + \ + mysql_integration_tests_steps("mysql80", "8.0") + \ + redis_integration_tests_steps() + \ + memcached_integration_tests_steps() + \ + remote_alertmanager_integration_tests_steps() pipelines.append(pipeline( name = "integration-tests", @@ -442,7 +248,7 @@ def verify_release_pipeline( bucket = from_secret(prerelease_bucket), gcp_key = from_secret(rgm_gcp_key_base64), version = "${DRONE_TAG}", - trigger = release_trigger, + trigger = {}, depends_on = [ "release-build-e2e-publish", "release-windows", @@ -456,7 +262,7 @@ def verify_release_pipeline( step = { "name": "gsutil-stat", "depends_on": ["clone"], - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "environment": { "BUCKET": bucket, "GCP_KEY": gcp_key, diff --git a/scripts/drone/pipelines/benchmarks.star b/scripts/drone/pipelines/benchmarks.star index 5e1fc6a3ec1..ae5a9611675 100644 --- a/scripts/drone/pipelines/benchmarks.star +++ b/scripts/drone/pipelines/benchmarks.star @@ -2,6 +2,11 @@ This module returns the pipeline used for integration benchmarks. """ +load( + "scripts/drone/services/services.star", + "integration_test_services", + "integration_test_services_volumes", +) load( "scripts/drone/steps/lib.star", "compile_build_cmd", @@ -11,11 +16,6 @@ load( "verify_gen_jsonnet_step", "wire_install_step", ) -load( - "scripts/drone/services/services.star", - "integration_test_services", - "integration_test_services_volumes", -) load( "scripts/drone/utils/utils.star", "pipeline", @@ -52,22 +52,20 @@ def integration_benchmarks(prefix): wire_install_step(), ] - benchmark_steps = [ - integration_benchmarks_step("sqlite"), - integration_benchmarks_step("postgres", { - "PGPASSWORD": "grafanatest", - "GRAFANA_TEST_DB": "postgres", - "POSTGRES_HOST": "postgres", - }), - integration_benchmarks_step("mysql-5.7", { - "GRAFANA_TEST_DB": "mysql", - "MYSQL_HOST": "mysql57", - }), - integration_benchmarks_step("mysql-8.0", { - "GRAFANA_TEST_DB": "mysql", - "MYSQL_HOST": "mysql80", - }), - ] + benchmark_steps = integration_benchmarks_step("sqlite") + \ + integration_benchmarks_step("postgres", { + "PGPASSWORD": "grafanatest", + "GRAFANA_TEST_DB": "postgres", + "POSTGRES_HOST": "postgres", + }) + \ + integration_benchmarks_step("mysql-5.7", { + "GRAFANA_TEST_DB": "mysql", + "MYSQL_HOST": "mysql57", + }) + \ + integration_benchmarks_step("mysql-8.0", { + "GRAFANA_TEST_DB": "mysql", + "MYSQL_HOST": "mysql80", + }) return pipeline( name = "{}-integration-benchmarks".format(prefix), diff --git a/scripts/drone/pipelines/build.star b/scripts/drone/pipelines/build.star index 7085a7f9d42..88b5ed8e52a 100644 --- a/scripts/drone/pipelines/build.star +++ b/scripts/drone/pipelines/build.star @@ -2,15 +2,10 @@ load( "scripts/drone/steps/lib.star", - "build_backend_step", - "build_docker_images_step", "build_frontend_package_step", - "build_frontend_step", - "build_plugins_step", "build_storybook_step", "cloud_plugins_e2e_tests_step", "compile_build_cmd", - "copy_packages_for_docker_step", "download_grabpl_step", "e2e_tests_artifacts", "e2e_tests_step", @@ -18,12 +13,12 @@ load( "frontend_metrics_step", "grafana_server_step", "identify_runner_step", - "package_step", "publish_images_step", "release_canary_npm_packages_step", "store_storybook_step", "test_a11y_frontend_step", "trigger_oss", + "update_package_json_version", "upload_cdn_step", "upload_packages_step", "verify_gen_cue_step", @@ -31,6 +26,15 @@ load( "wire_install_step", "yarn_install_step", ) +load( + "scripts/drone/steps/rgm.star", + "rgm_build_docker_step", + "rgm_package_step", +) +load( + "scripts/drone/utils/images.star", + "images", +) load( "scripts/drone/utils/utils.star", "pipeline", @@ -42,7 +46,7 @@ def build_e2e(trigger, ver_mode): Args: trigger: controls which events can trigger the pipeline execution. - ver_mode: used in the naming of the pipeline. + ver_mode: used in the naming of the pipeline. Either 'pr' or 'main'. Returns: Drone pipeline. @@ -64,17 +68,19 @@ def build_e2e(trigger, ver_mode): if ver_mode == "pr": build_steps.extend( [ + build_frontend_package_step(), enterprise_downstream_step(ver_mode = ver_mode), ], ) + else: + build_steps.extend([ + update_package_json_version(), + build_frontend_package_step(depends_on = ["update-package-json-version"]), + ]) build_steps.extend( [ - build_backend_step(ver_mode = ver_mode), - build_frontend_step(ver_mode = ver_mode), - build_frontend_package_step(ver_mode = ver_mode), - build_plugins_step(ver_mode = ver_mode), - package_step(ver_mode = ver_mode), + rgm_package_step(distros = "linux/amd64,linux/arm64", file = "packages.txt"), grafana_server_step(), e2e_tests_step("dashboards-suite"), e2e_tests_step("smoke-tests-suite"), @@ -87,7 +93,6 @@ def build_e2e(trigger, ver_mode): ), e2e_tests_artifacts(), build_storybook_step(ver_mode = ver_mode), - copy_packages_for_docker_step(), test_a11y_frontend_step(ver_mode = ver_mode), ], ) @@ -97,12 +102,12 @@ def build_e2e(trigger, ver_mode): [ store_storybook_step(trigger = trigger_oss, ver_mode = ver_mode), frontend_metrics_step(trigger = trigger_oss), - build_docker_images_step( - publish = False, - ), - build_docker_images_step( - publish = False, - ubuntu = True, + rgm_build_docker_step( + "packages.txt", + images["ubuntu"], + images["alpine"], + tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}", + ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}", ), publish_images_step( docker_repo = "grafana", @@ -128,16 +133,12 @@ def build_e2e(trigger, ver_mode): elif ver_mode == "pr": build_steps.extend( [ - build_docker_images_step( - archs = [ - "amd64", - ], - ), - build_docker_images_step( - archs = [ - "amd64", - ], - ubuntu = True, + rgm_build_docker_step( + "packages.txt", + images["ubuntu"], + images["alpine"], + tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}", + ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}", ), publish_images_step( docker_repo = "grafana", diff --git a/scripts/drone/pipelines/ci_images.star b/scripts/drone/pipelines/ci_images.star index 65df476754b..e8180d6d0ef 100644 --- a/scripts/drone/pipelines/ci_images.star +++ b/scripts/drone/pipelines/ci_images.star @@ -3,21 +3,21 @@ This module contains steps and pipelines relating to creating CI Docker images. """ load( - "scripts/drone/utils/utils.star", - "pipeline", + "scripts/drone/utils/images.star", + "images", ) load( - "scripts/drone/vault.star", - "from_secret", - "gcp_download_build_container_assets_key", + "scripts/drone/utils/utils.star", + "pipeline", ) load( "scripts/drone/utils/windows_images.star", "windows_images", ) load( - "scripts/drone/utils/images.star", - "images", + "scripts/drone/vault.star", + "from_secret", + "gcp_download_build_container_assets_key", ) def publish_ci_windows_test_image_pipeline(): @@ -32,7 +32,7 @@ def publish_ci_windows_test_image_pipeline(): steps = [ { "name": "clone", - "image": windows_images["wix_image"], + "image": windows_images["wix"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, @@ -43,7 +43,7 @@ def publish_ci_windows_test_image_pipeline(): }, { "name": "build-and-publish", - "image": windows_images["windows_server_core_image"], + "image": windows_images["windows_server_core"], "environment": { "DOCKER_USERNAME": from_secret("docker_username"), "DOCKER_PASSWORD": from_secret("docker_password"), @@ -81,14 +81,14 @@ def publish_ci_build_container_image_pipeline(): steps = [ { "name": "validate-version", - "image": images["alpine_image"], + "image": images["alpine"], "commands": [ "if [ -z \"${BUILD_CONTAINER_VERSION}\" ]; then echo Missing BUILD_CONTAINER_VERSION; false; fi", ], }, { "name": "download-macos-sdk", - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "environment": { "GCP_KEY": from_secret(gcp_download_build_container_assets_key), }, @@ -100,7 +100,7 @@ def publish_ci_build_container_image_pipeline(): }, { "name": "build-and-publish", # Consider splitting the build and the upload task. - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], "environment": { "DOCKER_USERNAME": from_secret("docker_username"), diff --git a/scripts/drone/pipelines/docs.star b/scripts/drone/pipelines/docs.star index cdf944e43ec..536749b9e04 100644 --- a/scripts/drone/pipelines/docs.star +++ b/scripts/drone/pipelines/docs.star @@ -50,7 +50,7 @@ def docs_pipelines(ver_mode, trigger): def lint_docs(): return { "name": "lint-docs", - "image": images["build_image"], + "image": images["node"], "depends_on": [ "yarn-install", ], diff --git a/scripts/drone/pipelines/integration_tests.star b/scripts/drone/pipelines/integration_tests.star index c72456b45cf..e51b55bf48f 100644 --- a/scripts/drone/pipelines/integration_tests.star +++ b/scripts/drone/pipelines/integration_tests.star @@ -2,25 +2,26 @@ This module returns the pipeline used for integration tests. """ +load( + "scripts/drone/services/services.star", + "integration_test_services", + "integration_test_services_volumes", +) load( "scripts/drone/steps/lib.star", "compile_build_cmd", "download_grabpl_step", "enterprise_setup_step", "identify_runner_step", - "memcached_integration_tests_step", - "mysql_integration_tests_step", - "postgres_integration_tests_step", - "redis_integration_tests_step", + "memcached_integration_tests_steps", + "mysql_integration_tests_steps", + "postgres_integration_tests_steps", + "redis_integration_tests_steps", + "remote_alertmanager_integration_tests_steps", "verify_gen_cue_step", "verify_gen_jsonnet_step", "wire_install_step", ) -load( - "scripts/drone/services/services.star", - "integration_test_services", - "integration_test_services_volumes", -) load( "scripts/drone/utils/utils.star", "pipeline", @@ -60,13 +61,13 @@ def integration_tests(trigger, prefix, ver_mode = "pr"): wire_install_step(), ] - test_steps = [ - postgres_integration_tests_step(), - mysql_integration_tests_step("mysql57", "5.7"), - mysql_integration_tests_step("mysql80", "8.0"), - redis_integration_tests_step(), - memcached_integration_tests_step(), - ] + # test_steps = [a, b] + [c, d] + [e, f]... + test_steps = postgres_integration_tests_steps() + \ + mysql_integration_tests_steps("mysql57", "5.7") + \ + mysql_integration_tests_steps("mysql80", "8.0") + \ + redis_integration_tests_steps() + \ + memcached_integration_tests_steps() + \ + remote_alertmanager_integration_tests_steps() return pipeline( name = "{}-integration-tests".format(prefix), diff --git a/scripts/drone/pipelines/lint_backend.star b/scripts/drone/pipelines/lint_backend.star index 53877f889ba..d675436cdc2 100644 --- a/scripts/drone/pipelines/lint_backend.star +++ b/scripts/drone/pipelines/lint_backend.star @@ -46,6 +46,8 @@ def lint_backend_pipeline(trigger, ver_mode): test_steps = [ lint_backend_step(), validate_modfile_step(), + # OpenAPI spec validation is disabled for versions below 10.2.x. + # validate_openapi_spec_step(), ] if ver_mode == "main": diff --git a/scripts/drone/pipelines/shellcheck.star b/scripts/drone/pipelines/shellcheck.star index 5b8d481094c..7f56c553976 100644 --- a/scripts/drone/pipelines/shellcheck.star +++ b/scripts/drone/pipelines/shellcheck.star @@ -3,14 +3,14 @@ This module returns a Drone step and pipeline for linting with shellcheck. """ load("scripts/drone/steps/lib.star", "compile_build_cmd") -load( - "scripts/drone/utils/utils.star", - "pipeline", -) load( "scripts/drone/utils/images.star", "images", ) +load( + "scripts/drone/utils/utils.star", + "pipeline", +) trigger = { "event": [ @@ -29,12 +29,10 @@ trigger = { def shellcheck_step(): return { "name": "shellcheck", - "image": images["build_image"], - "depends_on": [ - "compile-build-cmd", - ], + "image": images["ubuntu"], "commands": [ - "./bin/build shellcheck", + "apt-get update -yq && apt-get install shellcheck", + "shellcheck -e SC1071 -e SC2162 scripts/**/*.sh", ], } diff --git a/scripts/drone/pipelines/test_backend.star b/scripts/drone/pipelines/test_backend.star index ad1394196f3..c5bb47da11e 100644 --- a/scripts/drone/pipelines/test_backend.star +++ b/scripts/drone/pipelines/test_backend.star @@ -4,7 +4,6 @@ This module returns the pipeline used for testing backend code. load( "scripts/drone/steps/lib.star", - "compile_build_cmd", "enterprise_setup_step", "identify_runner_step", "test_backend_integration_step", @@ -41,7 +40,6 @@ def test_backend(trigger, ver_mode): steps += [ identify_runner_step(), - compile_build_cmd(), verify_step, verify_jsonnet_step, wire_install_step(), diff --git a/scripts/drone/pipelines/test_frontend.star b/scripts/drone/pipelines/test_frontend.star index e9e353720bd..9a01f101c58 100644 --- a/scripts/drone/pipelines/test_frontend.star +++ b/scripts/drone/pipelines/test_frontend.star @@ -5,7 +5,6 @@ This module returns the pipeline used for testing backend code. load( "scripts/drone/steps/lib.star", "betterer_frontend_step", - "download_grabpl_step", "enterprise_setup_step", "identify_runner_step", "test_frontend_step", @@ -30,7 +29,6 @@ def test_frontend(trigger, ver_mode): steps = [ identify_runner_step(), - download_grabpl_step(), yarn_install_step(), betterer_frontend_step(), ] diff --git a/scripts/drone/pipelines/whats_new_checker.star b/scripts/drone/pipelines/whats_new_checker.star index e15a8d7a291..2c56be573ce 100644 --- a/scripts/drone/pipelines/whats_new_checker.star +++ b/scripts/drone/pipelines/whats_new_checker.star @@ -2,14 +2,14 @@ This module contains logic for checking if the package.json whats new url matches with the in-flight tag. """ -load( - "scripts/drone/utils/images.star", - "images", -) load( "scripts/drone/steps/lib.star", "compile_build_cmd", ) +load( + "scripts/drone/utils/images.star", + "images", +) load( "scripts/drone/utils/utils.star", "pipeline", @@ -18,7 +18,7 @@ load( def whats_new_checker_step(): return { "name": "whats-new-checker", - "image": images["go_image"], + "image": images["go"], "depends_on": [ "compile-build-cmd", ], diff --git a/scripts/drone/pipelines/windows.star b/scripts/drone/pipelines/windows.star index fd5b99c9bf0..59c12a19536 100644 --- a/scripts/drone/pipelines/windows.star +++ b/scripts/drone/pipelines/windows.star @@ -3,15 +3,15 @@ This module returns the pipeline used for building Grafana on Windows. """ load( - "scripts/drone/utils/utils.star", - "pipeline", + "scripts/drone/steps/lib_windows.star", + "clone_step_windows", + "get_windows_steps", + "test_backend_step_windows", + "wire_install_step_windows", ) load( - "scripts/drone/steps/lib.star", - "get_windows_steps", - "windows_clone_step", - "windows_test_backend_step", - "windows_wire_install_step", + "scripts/drone/utils/utils.star", + "pipeline", ) load( "scripts/drone/utils/windows_images.star", @@ -30,19 +30,19 @@ def windows_test_backend(trigger, edition, ver_mode): """ environment = {"EDITION": edition} steps = [ - windows_clone_step(), + clone_step_windows(), ] steps.extend([{ "name": "windows-init", - "image": windows_images["windows_go_image"], + "image": windows_images["go"], "depends_on": ["clone"], "commands": [], }]) steps.extend([ - windows_wire_install_step(edition), - windows_test_backend_step(), + wire_install_step_windows(edition), + test_backend_step_windows(), ]) pl = pipeline( name = "{}-test-backend-windows".format(ver_mode), diff --git a/scripts/drone/rgm.star b/scripts/drone/rgm.star index 0721f76b32b..cd4df9b9de8 100644 --- a/scripts/drone/rgm.star +++ b/scripts/drone/rgm.star @@ -4,50 +4,52 @@ rgm uses 'github.com/grafana/grafana-build' to build Grafana on the following ev * A tag that begins with a 'v' """ -load( - "scripts/drone/steps/lib.star", - "get_windows_steps", -) -load( - "scripts/drone/utils/utils.star", - "ignore_failure", - "pipeline", -) load( "scripts/drone/events/release.star", "verify_release_pipeline", ) -load( - "scripts/drone/pipelines/test_frontend.star", - "test_frontend", -) load( "scripts/drone/pipelines/test_backend.star", "test_backend", ) +load( + "scripts/drone/pipelines/test_frontend.star", + "test_frontend", +) load( "scripts/drone/pipelines/whats_new_checker.star", "whats_new_checker_pipeline", ) +load( + "scripts/drone/steps/lib_windows.star", + "get_windows_steps", +) +load( + "scripts/drone/utils/images.star", + "images", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", + "with_deps", +) +load( + "scripts/drone/variables.star", + "golang_version", +) load( "scripts/drone/vault.star", "from_secret", + "npm_token", + "rgm_cdn_destination", "rgm_dagger_token", "rgm_destination", + "rgm_downloads_destination", "rgm_gcp_key_base64", "rgm_github_token", + "rgm_storybook_destination", ) -rgm_env_secrets = { - "GCP_KEY_BASE64": from_secret(rgm_gcp_key_base64), - "DESTINATION": from_secret(rgm_destination), - "GITHUB_TOKEN": from_secret(rgm_github_token), - "_EXPERIMENTAL_DAGGER_CLOUD_TOKEN": from_secret(rgm_dagger_token), - "GPG_PRIVATE_KEY": from_secret("packages_gpg_private_key"), - "GPG_PUBLIC_KEY": from_secret("packages_gpg_public_key"), - "GPG_PASSPHRASE": from_secret("packages_gpg_passphrase"), -} - docs_paths = { "exclude": [ "*.md", @@ -73,75 +75,233 @@ tag_trigger = { }, } -def rgm_build(script = "drone_publish_main.sh", canFail = True): - rgm_build_step = { - "name": "rgm-build", +main_trigger = { + "event": [ + "push", + ], + "branch": "main", + "paths": docs_paths, + "repo": [ + "grafana/grafana", + ], +} + +nightly_trigger = { + "event": { + "include": [ + "cron", + ], + }, + "cron": { + "include": [ + "nightly-release", + ], + }, +} + +version_branch_trigger = {"ref": ["refs/heads/v[0-9]*"]} + +def rgm_env_secrets(env): + """Adds the rgm secret ENV variables to the given env arg + + Args: + env: A map of environment varables. This function will adds the necessary secrets to it (and potentially overwrite them). + Returns: + Drone step. + """ + env["DESTINATION"] = from_secret(rgm_destination) + env["STORYBOOK_DESTINATION"] = from_secret(rgm_storybook_destination) + env["CDN_DESTINATION"] = from_secret(rgm_cdn_destination) + env["DOWNLOADS_DESTINATION"] = from_secret(rgm_downloads_destination) + + env["GCP_KEY_BASE64"] = from_secret(rgm_gcp_key_base64) + env["GITHUB_TOKEN"] = from_secret(rgm_github_token) + env["_EXPERIMENTAL_DAGGER_CLOUD_TOKEN"] = from_secret(rgm_dagger_token) + env["GPG_PRIVATE_KEY"] = from_secret("packages_gpg_private_key") + env["GPG_PUBLIC_KEY"] = from_secret("packages_gpg_public_key") + env["GPG_PASSPHRASE"] = from_secret("packages_gpg_passphrase") + env["DOCKER_USERNAME"] = from_secret("docker_username") + env["DOCKER_PASSWORD"] = from_secret("docker_password") + env["NPM_TOKEN"] = from_secret(npm_token) + env["GCOM_API_KEY"] = from_secret("grafana_api_key") + return env + +def rgm_run(name, script): + """Returns a pipeline that does a full build & package of Grafana. + + Args: + name: The name of the pipeline step. + script: The script in the container to run. + Returns: + Drone step. + """ + env = { + "GO_VERSION": golang_version, + } + rgm_run_step = { + "name": name, "image": "grafana/grafana-build:main", + "pull": "always", "commands": [ "export GRAFANA_DIR=$$(pwd)", "cd /src && ./scripts/{}".format(script), ], - "environment": rgm_env_secrets, + "environment": rgm_env_secrets(env), # The docker socket is a requirement for running dagger programs # In the future we should find a way to use dagger without mounting the docker socket. "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], } - if canFail: - rgm_build_step["failure"] = "ignore" return [ - rgm_build_step, + rgm_run_step, ] -def rgm_main(): - trigger = { - "event": [ - "push", - ], - "branch": "main", - "paths": docs_paths, - "repo": [ - "grafana/grafana", - ], +def rgm_copy(src, dst): + """Copies file from/to GCS. + + Args: + src: source of the files. + dst: destination of the files. + + Returns: + Drone steps. + """ + commands = [ + "printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json", + "gcloud auth activate-service-account --key-file=/tmp/key.json", + "gcloud storage cp -r {} {}".format(src, dst), + ] + + if not dst.startswith("gs://"): + commands.insert(0, "mkdir -p {}".format(dst)) + + rgm_copy_step = { + "name": "rgm-copy", + "image": "google/cloud-sdk:alpine", + "commands": commands, + "environment": rgm_env_secrets({}), } + return [ + rgm_copy_step, + ] + +def rgm_publish_packages(bucket = "grafana-packages"): + """Publish deb and rpm packages. + + Args: + bucket: target bucket to publish the packages. + + Returns: + Drone steps. + """ + steps = [] + for package_manager in ["deb", "rpm"]: + steps.append({ + "name": "publish-{}".format(package_manager), + # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image + "image": images["package_publish"], + "privileged": True, + "settings": { + "access_key_id": from_secret("packages_access_key_id"), + "secret_access_key": from_secret("packages_secret_access_key"), + "service_account_json": from_secret("packages_service_account"), + "target_bucket": bucket, + "gpg_passphrase": from_secret("packages_gpg_passphrase"), + "gpg_public_key": from_secret("packages_gpg_public_key"), + "gpg_private_key": from_secret("packages_gpg_private_key"), + "package_path": "file:///drone/src/dist/*.{}".format(package_manager), + }, + }) + + return steps + +def rgm_main(): + # Runs a package / build process (with some distros) when commits are merged to main return pipeline( name = "rgm-main-prerelease", - trigger = trigger, - steps = rgm_build(canFail = True), + trigger = main_trigger, + steps = rgm_run("rgm-build", "drone_publish_main.sh"), depends_on = ["main-test-backend", "main-test-frontend"], ) def rgm_tag(): + # Runs a package / build process (with all distros) when a tag is made return pipeline( name = "rgm-tag-prerelease", trigger = tag_trigger, - steps = rgm_build(script = "drone_publish_tag_grafana.sh", canFail = False), + steps = rgm_run("rgm-build", "drone_publish_tag_grafana.sh"), depends_on = ["release-test-backend", "release-test-frontend"], ) -def rgm_windows(): +def rgm_tag_windows(): return pipeline( name = "rgm-tag-prerelease-windows", trigger = tag_trigger, - steps = ignore_failure( - get_windows_steps( - ver_mode = "release", - bucket = "grafana-prerelease", - ), + steps = get_windows_steps( + ver_mode = "release", + bucket = "grafana-prerelease", ), depends_on = ["rgm-tag-prerelease"], platform = "windows", ) -def rgm(): +def rgm_version_branch(): + # Runs a package / build proces (with all distros) when a commit lands on a version branch + return pipeline( + name = "rgm-version-branch-prerelease", + trigger = version_branch_trigger, + steps = rgm_run("rgm-build", "drone_publish_tag_grafana.sh"), + depends_on = ["release-test-backend", "release-test-frontend"], + ) + +def rgm_nightly_build(): + src = "$${DRONE_WORKSPACE}/dist/*" + dst = "$${DESTINATION}/$${DRONE_BUILD_EVENT}" + + copy_steps = with_deps(rgm_copy(src, dst), ["rgm-build"]) + + return pipeline( + name = "rgm-nightly-build", + trigger = nightly_trigger, + steps = rgm_run("rgm-build", "drone_build_nightly_grafana.sh") + copy_steps, + depends_on = ["nightly-test-backend", "nightly-test-frontend"], + ) + +def rgm_nightly_publish(): + """Nightly publish pipeline. + + Returns: + Drone pipeline. + """ + src = "$${DESTINATION}/$${DRONE_BUILD_EVENT}/*_$${DRONE_BUILD_NUMBER}_*" + dst = "$${DRONE_WORKSPACE}/dist" + + publish_steps = with_deps(rgm_run("rgm-publish", "drone_publish_nightly_grafana.sh"), ["rgm-copy"]) + package_steps = with_deps(rgm_publish_packages(), ["rgm-publish"]) + + return pipeline( + name = "rgm-nightly-publish", + trigger = nightly_trigger, + steps = rgm_copy(src, dst) + publish_steps + package_steps, + depends_on = ["rgm-nightly-build"], + ) + +def rgm_nightly_pipeline(): + return [ + test_frontend(nightly_trigger, "nightly"), + test_backend(nightly_trigger, "nightly"), + rgm_nightly_build(), + rgm_nightly_publish(), + ] + +def rgm_tag_pipeline(): return [ whats_new_checker_pipeline(tag_trigger), test_frontend(tag_trigger, "release"), test_backend(tag_trigger, "release"), - rgm_main(), rgm_tag(), - rgm_windows(), + rgm_tag_windows(), verify_release_pipeline( trigger = tag_trigger, name = "rgm-tag-verify-prerelease-assets", @@ -152,3 +312,29 @@ def rgm(): ], ), ] + +def rgm_version_branch_pipeline(): + return [ + rgm_version_branch(), + verify_release_pipeline( + trigger = version_branch_trigger, + name = "rgm-prerelease-verify-prerelease-assets", + bucket = "grafana-prerelease", + depends_on = [ + "rgm-version-branch-prerelease", + ], + ), + ] + +def rgm_main_pipeline(): + return [ + rgm_main(), + ] + +def rgm(): + return ( + rgm_main_pipeline() + + rgm_tag_pipeline() + + rgm_version_branch_pipeline() + + rgm_nightly_pipeline() + ) diff --git a/scripts/drone/services/services.star b/scripts/drone/services/services.star index ae2d939cf4b..911aacfb000 100644 --- a/scripts/drone/services/services.star +++ b/scripts/drone/services/services.star @@ -18,7 +18,7 @@ def integration_test_services(): services = [ { "name": "postgres", - "image": images["postgres_alpine_image"], + "image": images["postgres_alpine"], "environment": { "POSTGRES_USER": "grafanatest", "POSTGRES_PASSWORD": "grafanatest", @@ -31,7 +31,7 @@ def integration_test_services(): }, { "name": "mysql57", - "image": images["mysql5_image"], + "image": images["mysql5"], "environment": { "MYSQL_ROOT_PASSWORD": "rootpass", "MYSQL_DATABASE": "grafana_tests", @@ -43,7 +43,7 @@ def integration_test_services(): }, { "name": "mysql80", - "image": images["mysql8_image"], + "image": images["mysql8"], "environment": { "MYSQL_ROOT_PASSWORD": "rootpass", "MYSQL_DATABASE": "grafana_tests", @@ -53,14 +53,20 @@ def integration_test_services(): "volumes": [{"name": "mysql80", "path": "/var/lib/mysql"}], "commands": ["docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password"], }, + { + "name": "mimir_backend", + "image": images["mimir"], + "environment": {}, + "commands": ["/bin/mimir -target=backend"], + }, { "name": "redis", - "image": images["redis_alpine_image"], + "image": images["redis_alpine"], "environment": {}, }, { "name": "memcached", - "image": images["memcached_alpine_image"], + "image": images["memcached_alpine"], "environment": {}, }, ] @@ -70,7 +76,7 @@ def integration_test_services(): def ldap_service(): return { "name": "ldap", - "image": images["openldap_image"], + "image": images["openldap"], "environment": { "LDAP_ADMIN_PASSWORD": "grafana", "LDAP_DOMAIN": "grafana.org", diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 0f2a1ba7123..c723d84dd61 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -2,6 +2,18 @@ This module is a library of Drone steps and other pipeline components. """ +load( + "scripts/drone/steps/rgm.star", + "rgm_build_backend_step", +) +load( + "scripts/drone/utils/images.star", + "images", +) +load( + "scripts/drone/variables.star", + "grabpl_version", +) load( "scripts/drone/vault.star", "from_secret", @@ -11,16 +23,6 @@ load( "npm_token", "prerelease_bucket", ) -load( - "scripts/drone/utils/images.star", - "images", -) -load( - "scripts/drone/utils/windows_images.star", - "windows_images", -) - -grabpl_version = "v3.0.41" trigger_oss = { "repo": [ @@ -28,21 +30,10 @@ trigger_oss = { ], } -def slack_step(channel, template, secret): - return { - "name": "slack", - "image": images["plugins_slack_image"], - "settings": { - "webhook": from_secret(secret), - "channel": channel, - "template": template, - }, - } - def yarn_install_step(): return { "name": "yarn-install", - "image": images["build_image"], + "image": images["node"], "commands": [ "yarn install --immutable", ], @@ -52,8 +43,9 @@ def yarn_install_step(): def wire_install_step(): return { "name": "wire-install", - "image": images["build_image"], + "image": images["go"], "commands": [ + "apk add --update make", "make gen-go", ], "depends_on": [ @@ -61,37 +53,15 @@ def wire_install_step(): ], } -def windows_wire_install_step(edition): +def identify_runner_step(): return { - "name": "wire-install", - "image": windows_images["windows_go_image"], + "name": "identify-runner", + "image": images["alpine"], "commands": [ - "go install github.com/google/wire/cmd/wire@v0.5.0", - "wire gen -tags {} ./pkg/server".format(edition), - ], - "depends_on": [ - "windows-init", + "echo $DRONE_RUNNER_NAME", ], } -def identify_runner_step(platform = "linux"): - if platform == "linux": - return { - "name": "identify-runner", - "image": images["alpine_image"], - "commands": [ - "echo $DRONE_RUNNER_NAME", - ], - } - else: - return { - "name": "identify-runner", - "image": windows_images["1809_image"], - "commands": [ - "echo $env:DRONE_RUNNER_NAME", - ], - } - def enterprise_setup_step(source = "${DRONE_SOURCE_BRANCH}", canFail = True, isPromote = False): """Setup the enterprise source into the ./grafana-enterprise directory. @@ -135,11 +105,12 @@ def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFai step = { "name": "clone-enterprise", - "image": images["build_image"], + "image": images["git"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, "commands": [ + "apk add --update curl jq bash", ] + check + [ 'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" ' + location, "cd {}".format(location), @@ -152,22 +123,10 @@ def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFai return step -def download_grabpl_step(platform = "linux"): - if platform == "windows": - return { - "name": "grabpl", - "image": windows_images["wix_image"], - "commands": [ - '$$ProgressPreference = "SilentlyContinue"', - "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( - grabpl_version, - ), - ], - } - +def download_grabpl_step(): return { "name": "grabpl", - "image": images["curl_image"], + "image": images["curl"], "commands": [ "mkdir -p bin", "curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/grabpl".format( @@ -180,7 +139,7 @@ def download_grabpl_step(platform = "linux"): def lint_drone_step(): return { "name": "lint-drone", - "image": images["curl_image"], + "image": images["curl"], "commands": [ "./bin/build verify-drone", ], @@ -192,9 +151,10 @@ def lint_drone_step(): def lint_starlark_step(): return { "name": "lint-starlark", - "image": images["build_image"], + "image": images["go"], "commands": [ - "./bin/build verify-starlark .", + "go install github.com/bazelbuild/buildtools/buildifier@latest", + "buildifier --lint=warn -mode=check -r .", ], "depends_on": [ "compile-build-cmd", @@ -219,7 +179,7 @@ def enterprise_downstream_step(ver_mode): step = { "name": "trigger-enterprise-downstream", - "image": images["drone_downstream_image"], + "image": images["drone_downstream"], "settings": { "server": "https://drone.grafana.net", "token": from_secret("drone_token"), @@ -242,8 +202,7 @@ def enterprise_downstream_step(ver_mode): def lint_backend_step(): return { "name": "lint-backend", - # TODO: build_image or go_image? - "image": images["go_image"], + "image": images["go"], "environment": { # We need CGO because of go-sqlite3 "CGO_ENABLED": "1", @@ -252,7 +211,7 @@ def lint_backend_step(): "wire-install", ], "commands": [ - "apt-get update && apt-get install make", + "apk add --update make build-base", # Don't use Make since it will re-download the linters "make lint-go", ], @@ -261,33 +220,38 @@ def lint_backend_step(): def validate_modfile_step(): return { "name": "validate-modfile", - "image": images["go_image"], - "failure": "ignore", + "image": images["go"], "commands": [ "go run scripts/modowners/modowners.go check go.mod", ], } -def benchmark_ldap_step(): +def validate_openapi_spec_step(): return { - "name": "benchmark-ldap", - "image": images["build_image"], - "environment": { - "LDAP_HOSTNAME": "ldap", - }, + "name": "validate-openapi-spec", + "image": images["go"], "commands": [ - "dockerize -wait tcp://ldap:389 -timeout 120s", - 'go test -benchmem -run=^$ ./pkg/extensions/ldapsync -bench "^(Benchmark50Users)$"', + "apk add --update make", + "make swagger-validate", + ], + } + +def dockerize_step(name, hostname, port): + return { + "name": name, + "image": images["dockerize"], + "commands": [ + "dockerize -wait tcp://{}:{} -timeout 120s".format(hostname, port), ], } def build_storybook_step(ver_mode): return { "name": "build-storybook", - "image": images["build_image"], + "image": images["node"], "depends_on": [ # Best to ensure that this step doesn't mess with what's getting built and packaged - "build-frontend", + "rgm-package", "build-frontend-packages", ], "environment": { @@ -328,7 +292,7 @@ def store_storybook_step(ver_mode, trigger = None): step = { "name": "store-storybook", - "image": images["publish_image"], + "image": images["publish"], "depends_on": [ "build-storybook", ] + @@ -358,7 +322,7 @@ def store_storybook_step(ver_mode, trigger = None): def e2e_tests_artifacts(): return { "name": "e2e-tests-artifacts-upload", - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "depends_on": [ "end-to-end-tests-dashboards-suite", "end-to-end-tests-panels-suite", @@ -406,7 +370,7 @@ def upload_cdn_step(ver_mode, trigger = None): step = { "name": "upload-cdn-assets", - "image": images["publish_image"], + "image": images["publish"], "depends_on": [ "grafana-server", ], @@ -422,75 +386,27 @@ def upload_cdn_step(ver_mode, trigger = None): step = dict(step, when = trigger) return step -def build_backend_step(ver_mode, variants = None): +def build_backend_step(distros = "linux/amd64,linux/arm64"): """Build the backend code using the Grafana build tool. Args: - ver_mode: if ver_mode != 'release', pass the DRONE_BUILD_NUMBER environment - variable as the value for the --build-id option. - TODO: is this option actually used by the build-backend subcommand? - variants: a list of variants be passed to the build-backend subcommand - using the --variants option. - Defaults to None. + distros: a list of distributes to be built. For a full list, see `go tool dist list`. Returns: Drone step. """ - variants_str = "" - if variants: - variants_str = " --variants {}".format(",".join(variants)) - # TODO: Convert number of jobs to percentage - if ver_mode == "release": - cmds = [ - "./bin/build build-backend --jobs 8 --edition oss ${DRONE_TAG}", - ] - else: - build_no = "${DRONE_BUILD_NUMBER}" - cmds = [ - "./bin/build build-backend --jobs 8 --edition oss --build-id {}{}".format( - build_no, - variants_str, - ), - ] + return rgm_build_backend_step(distros) - return { - "name": "build-backend", - "image": images["build_image"], - "depends_on": [ - "wire-install", - "compile-build-cmd", - ], - "commands": cmds, - } - -def build_frontend_step(ver_mode): - """Build the frontend code using the Grafana build tool. - - Args: - ver_mode: if ver_mode != 'release', use the DRONE_BUILD_NUMBER environment - variable as a build identifier. +def build_frontend_step(): + """Build the frontend code to ensure it's compilable Returns: Drone step. """ - build_no = "${DRONE_BUILD_NUMBER}" - - # TODO: Use percentage for num jobs - if ver_mode == "release": - cmds = [ - "./bin/build build-frontend --jobs 8 " + - "--edition oss ${DRONE_TAG}", - ] - else: - cmds = [ - "./bin/build build-frontend --jobs 8 --edition oss" + - "--build-id {}".format(build_no), - ] - return { "name": "build-frontend", - "image": images["build_image"], + "image": images["node"], "environment": { "NODE_OPTIONS": "--max_old_space_size=8192", }, @@ -498,45 +414,60 @@ def build_frontend_step(ver_mode): "compile-build-cmd", "yarn-install", ], - "commands": cmds, + "commands": [ + "yarn build", + ], } -def build_frontend_package_step(ver_mode): +def update_package_json_version(): + """Updates the packages/ to use a version that has the build ID in it: 10.0.0pre -> 10.0.0-5432pre + + Returns: + Drone step that updates the 'version' key in package.json + """ + + return { + "name": "update-package-json-version", + "image": images["node"], + "depends_on": [ + "yarn-install", + ], + "commands": [ + "apk add --update jq", + "new_version=$(cat package.json | jq .version | sed s/pre/${DRONE_BUILD_NUMBER}/g)", + "echo \"New version: $new_version\"", + "yarn run lerna version $new_version --exact --no-git-tag-version --no-push --force-publish -y", + "yarn install --mode=update-lockfile", + ], + } + +def build_frontend_package_step(depends_on = []): """Build the frontend packages using the Grafana build tool. Args: - ver_mode: if ver_mode != 'release', use the DRONE_BUILD_NUMBER environment - variable as a build identifier. + depends_on: a list of step names (strings) that must complete before this step runs. Returns: Drone step. """ - build_no = "${DRONE_BUILD_NUMBER}" - # TODO: Use percentage for num jobs - if ver_mode == "release": - cmds = [ - "./bin/build build-frontend-packages --jobs 8 " + - "--edition oss ${DRONE_TAG}", - ] - else: - cmds = [ - "./bin/build build-frontend-packages --jobs 8 --edition oss" + - "--build-id {}".format(build_no), - "yarn packages:pack", - "./scripts/validate-npm-packages.sh", - ] + cmds = [ + "apk add --update jq bash", # bash is needed for the validate-npm-packages.sh script since it has a 'bash' + # shebang. + "yarn packages:build", + "yarn packages:pack", + "./scripts/validate-npm-packages.sh", + ] return { "name": "build-frontend-packages", - "image": images["build_image"], + "image": images["node"], "environment": { "NODE_OPTIONS": "--max_old_space_size=8192", }, "depends_on": [ - "compile-build-cmd", "yarn-install", - ], + ] + depends_on, "commands": cmds, } @@ -549,42 +480,41 @@ def build_plugins_step(ver_mode): env = None return { "name": "build-plugins", - "image": images["build_image"], + "image": images["node"], "environment": env, "depends_on": [ - "compile-build-cmd", "yarn-install", ], "commands": [ - # TODO: Use percentage for num jobs - "./bin/build build-plugins --jobs 8 --edition oss", + "apk add --update findutils", # Replaces the busybox 'find' with the GNU one. + "yarn plugins:build", ], } -def test_backend_step(image = images["build_image"]): +def test_backend_step(): return { "name": "test-backend", - "image": image, + "image": images["go"], "depends_on": [ "wire-install", ], "commands": [ + # shared-mime-info and shared-mime-info-lang is used for exactly 1 test for the + # mime.TypeByExtension function. + "apk add --update build-base shared-mime-info shared-mime-info-lang", "go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/...", ], } -def windows_test_backend_step(): - step = test_backend_step(image = windows_images["windows_go_image"]) - return step - def test_backend_integration_step(): return { "name": "test-backend-integration", - "image": images["build_image"], + "image": images["go"], "depends_on": [ "wire-install", ], "commands": [ + "apk add --update build-base", "go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\\(.*\\)/' | sort -u)", ], } @@ -598,11 +528,12 @@ def betterer_frontend_step(): return { "name": "betterer-frontend", - "image": images["build_image"], + "image": images["node"], "depends_on": [ "yarn-install", ], "commands": [ + "apk add --update git bash", "yarn betterer ci", ], } @@ -616,7 +547,7 @@ def test_frontend_step(): return { "name": "test-frontend", - "image": images["build_image"], + "image": images["node"], "environment": { "TEST_MAX_WORKERS": "50%", }, @@ -631,7 +562,7 @@ def test_frontend_step(): def lint_frontend_step(): return { "name": "lint-frontend", - "image": images["build_image"], + "image": images["node"], "environment": { "TEST_MAX_WORKERS": "50%", }, @@ -650,11 +581,12 @@ def verify_i18n_step(): uncommited_error_message = "\nTranslation extraction has not been committed. Please run 'yarn i18n:extract', commit the changes and push again." return { "name": "verify-i18n", - "image": images["build_image"], + "image": images["node"], "depends_on": [ "yarn-install", ], "commands": [ + "apk add --update git", "yarn run i18n:extract || (echo \"{}\" && false)".format(extract_error_message), # Verify that translation extraction has been committed ''' @@ -702,7 +634,7 @@ def test_a11y_frontend_step(ver_mode, port = 3001): return { "name": "test-a11y-frontend", # TODO which image should be used? - "image": images["docker_puppeteer_image"], + "image": images["docker_puppeteer"], "depends_on": [ "grafana-server", ], @@ -727,7 +659,7 @@ def frontend_metrics_step(trigger = None): """ step = { "name": "publish-frontend-metrics", - "image": images["build_image"], + "image": images["node"], "depends_on": [ "test-a11y-frontend", ], @@ -736,7 +668,8 @@ def frontend_metrics_step(trigger = None): }, "failure": "ignore", "commands": [ - "./scripts/ci-frontend-metrics.sh | ./bin/build publish-metrics $${GRAFANA_MISC_STATS_API_KEY}", + "apk add --update bash grep git", + "./scripts/ci-frontend-metrics.sh ./grafana/public/build | ./bin/build publish-metrics $$GRAFANA_MISC_STATS_API_KEY", ], } if trigger: @@ -746,92 +679,38 @@ def frontend_metrics_step(trigger = None): def codespell_step(): return { "name": "codespell", - "image": images["build_image"], + "image": images["python"], "commands": [ + "pip3 install codespell", "codespell -I .codespellignore docs/", ], } -def package_step(ver_mode): - """Packages Grafana with the Grafana build tool. - - Args: - ver_mode: controls whether the packages are signed for a release. - If ver_mode != 'release', use the DRONE_BUILD_NUMBER environment - variable as a build identifier. - - Returns: - Drone step. - """ - deps = [ - "build-plugins", - "build-backend", - "build-frontend", - "build-frontend-packages", - ] - - if ver_mode in ("main", "release", "release-branch"): - sign_args = " --sign" - env = { - "GRAFANA_API_KEY": from_secret("grafana_api_key"), - "GPG_PRIV_KEY": from_secret("packages_gpg_private_key"), - "GPG_PUB_KEY": from_secret("packages_gpg_public_key"), - "GPG_KEY_PASSWORD": from_secret("packages_gpg_passphrase"), - } - test_args = "" - else: - sign_args = "" - env = None - - # TODO: env vars no longer needed by build if not signing - test_args = ". scripts/build/gpg-test-vars.sh && " - - # TODO: Use percentage for jobs - if ver_mode == "release": - cmds = [ - "{}./bin/build package --jobs 8 --edition oss ".format(test_args) + - "{} $${{DRONE_TAG}}".format(sign_args), - ] - else: - build_no = "${DRONE_BUILD_NUMBER}" - cmds = [ - "{}./bin/build package --jobs 8 --edition oss ".format(test_args) + - "--build-id {}{}".format(build_no, sign_args), - ] - - return { - "name": "package", - "image": images["build_image"], - "depends_on": deps, - "environment": env, - "commands": cmds, - } - -def grafana_server_step(port = 3001): +def grafana_server_step(): """Runs the grafana-server binary as a service. - Args: - port: port to listen on. - Defaults to 3001. - Returns: Drone step. """ - environment = {"PORT": port, "ARCH": "linux-amd64"} + environment = { + "GF_SERVER_HTTP_PORT": "3001", + "GF_SERVER_ROUTER_LOGGING": "1", + "GF_APP_MODE": "development", + } return { "name": "grafana-server", - "image": images["build_image"], + "image": images["alpine"], "detach": True, "depends_on": [ - "build-plugins", - "build-backend", - "build-frontend", - "build-frontend-packages", + "rgm-package", ], "environment": environment, "commands": [ - "./scripts/grafana-server/start-server", + "apk add --update tar bash", + "mkdir grafana", + "tar --strip-components=1 -xvf ./dist/*amd64.tar.gz -C grafana", + "cp -r devenv scripts tools grafana && cd grafana && ./scripts/grafana-server/start-server", ], } @@ -841,7 +720,7 @@ def e2e_tests_step(suite, port = 3001, tries = None): cmd += " --tries {}".format(tries) return { "name": "end-to-end-tests-{}".format(suite), - "image": images["cypress_image"], + "image": images["cypress"], "depends_on": [ "grafana-server", ], @@ -849,7 +728,6 @@ def e2e_tests_step(suite, port = 3001, tries = None): "HOST": "grafana-server", }, "commands": [ - "apt-get install -y netcat", cmd, ], } @@ -893,7 +771,7 @@ def cloud_plugins_e2e_tests_step(suite, cloud, trigger = None): branch = "${DRONE_SOURCE_BRANCH}".replace("/", "-") step = { "name": "end-to-end-tests-{}-{}".format(suite, cloud), - "image": images["cloud_datasources_e2e_image"], + "image": "us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:3.0.0", "depends_on": [ "grafana-server", ], @@ -907,73 +785,19 @@ def build_docs_website_step(): return { "name": "build-docs-website", # Use latest revision here, since we want to catch if it breaks - "image": images["docs_image"], + "image": images["docs"], "commands": [ "mkdir -p /hugo/content/docs/grafana/latest", + "echo -e '---\\nredirectURL: /docs/grafana/latest/\\ntype: redirect\\nversioned: true\\n---\\n' > /hugo/content/docs/grafana/_index.md", "cp -r docs/sources/* /hugo/content/docs/grafana/latest/", "cd /hugo && make prod", ], } -def copy_packages_for_docker_step(): - return { - "name": "copy-packages-for-docker", - "image": images["build_image"], - "depends_on": [ - "package", - ], - "commands": [ - "ls dist/*.tar.gz*", - "cp dist/*.tar.gz* packaging/docker/", - ], - } - -def build_docker_images_step(archs = None, ubuntu = False, publish = False): - """Build Docker images using the Grafana build tool. - - Args: - archs: a list of architectures to build the image for. - Defaults to None. - ubuntu: controls whether the final image is built from an Ubuntu base image. - Defaults to False. - publish: controls whether the built image is saved to a pre-release repository. - Defaults to False. - - Returns: - Drone step. - """ - cmd = "./bin/build build-docker --edition oss" - if publish: - cmd += " --shouldSave" - - ubuntu_sfx = "" - if ubuntu: - ubuntu_sfx = "-ubuntu" - cmd += " --ubuntu" - - if archs: - cmd += " -archs {}".format(",".join(archs)) - - environment = { - "GCP_KEY": from_secret(gcp_grafanauploads), - } - - return { - "name": "build-docker-images" + ubuntu_sfx, - "image": images["cloudsdk_image"], - "depends_on": [ - "copy-packages-for-docker", - "compile-build-cmd", - ], - "commands": [cmd], - "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], - "environment": environment, - } - def fetch_images_step(): return { "name": "fetch-images", - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "environment": { "GCP_KEY": from_secret(gcp_grafanauploads), "DOCKER_USER": from_secret("docker_username"), @@ -1014,7 +838,7 @@ def publish_images_step(ver_mode, docker_repo, trigger = None): docker_repo, ) - deps = ["build-docker-images", "build-docker-images-ubuntu"] + deps = ["rgm-build-docker"] if ver_mode == "release": deps = ["fetch-images"] cmd += " --version-tag ${DRONE_TAG}" @@ -1030,7 +854,7 @@ def publish_images_step(ver_mode, docker_repo, trigger = None): step = { "name": "publish-images-{}".format(name), - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "environment": environment, "commands": [cmd], "depends_on": deps, @@ -1043,18 +867,47 @@ def publish_images_step(ver_mode, docker_repo, trigger = None): return step -def integration_tests_step(name, cmds, environment = None): +def integration_tests_steps(name, cmds, hostname = None, port = None, environment = None): + """Integration test steps + + Args: + name: the name of the step. + cmds: the commands to run to perform the integration tests. + hostname: the hostname where the remote server is available. + port: the port where the remote server is available. + environment: Any extra environment variables needed to run the integration tests. + + Returns: + A list of drone steps. If a hostname / port were provided, then a step to wait for the remove server to be + available is also returned. + """ + dockerize_name = "wait-for-{}".format(name) + + depends = [ + "wire-install", + ] + step = { "name": "{}-integration-tests".format(name), - "image": images["build_image"], - "depends_on": ["wire-install"], - "commands": cmds, + "image": images["go"], + "depends_on": depends, + "commands": [ + "apk add --update build-base", + ] + cmds, } if environment: step["environment"] = environment - return step + if hostname == None: + return [step] + + depends = depends.append(dockerize_name) + + return [ + dockerize_step(dockerize_name, hostname, port), + step, + ] def integration_benchmarks_step(name, environment = None): cmds = [ @@ -1062,13 +915,11 @@ def integration_benchmarks_step(name, environment = None): "go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES}", ] - return integration_tests_step("{}-benchmark".format(name), cmds, environment) + return integration_tests_steps("{}-benchmark".format(name), cmds, environment = environment) -def postgres_integration_tests_step(): +def postgres_integration_tests_steps(): cmds = [ - "apt-get update", - "apt-get install -yq postgresql-client", - "dockerize -wait tcp://postgres:5432 -timeout 120s", + "apk add --update postgresql-client", "psql -p 5432 -h postgres -U grafanatest -d grafanatest -f " + "devenv/docker/blocks/postgres_tests/setup.sql", "go clean -testcache", @@ -1081,13 +932,11 @@ def postgres_integration_tests_step(): "POSTGRES_HOST": "postgres", } - return integration_tests_step("postgres", cmds, environment) + return integration_tests_steps("postgres", cmds, "postgres", "5432", environment) -def mysql_integration_tests_step(hostname, version): +def mysql_integration_tests_steps(hostname, version): cmds = [ - "apt-get update", - "apt-get install -yq default-mysql-client", - "dockerize -wait tcp://{}:3306 -timeout 120s".format(hostname), + "apk add --update mysql-client", "cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h {} -P 3306 -u root -prootpass".format(hostname), "go clean -testcache", "go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\\(.*\\)/' | sort -u)", @@ -1098,11 +947,10 @@ def mysql_integration_tests_step(hostname, version): "MYSQL_HOST": hostname, } - return integration_tests_step("mysql-{}".format(version), cmds, environment) + return integration_tests_steps("mysql-{}".format(version), cmds, hostname, "3306", environment) -def redis_integration_tests_step(): +def redis_integration_tests_steps(): cmds = [ - "dockerize -wait tcp://redis:6379/0 -timeout 120s", "go clean -testcache", "go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...", ] @@ -1111,11 +959,24 @@ def redis_integration_tests_step(): "REDIS_URL": "redis://redis:6379/0", } - return integration_tests_step("redis", cmds, environment) + return integration_tests_steps("redis", cmds, "redis", "6379", environment = environment) -def memcached_integration_tests_step(): +def remote_alertmanager_integration_tests_steps(): + cmds = [ + "go clean -testcache", + "go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/...", + ] + + environment = { + "AM_TENANT_ID": "test", + "AM_PASSWORD": "test", + "AM_URL": "http://mimir_backend:8080", + } + + return integration_tests_steps("remote-alertmanager", cmds, "mimir_backend", "8080", environment = environment) + +def memcached_integration_tests_steps(): cmds = [ - "dockerize -wait tcp://memcached:11211 -timeout 120s", "go clean -testcache", "go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...", ] @@ -1124,7 +985,7 @@ def memcached_integration_tests_step(): "MEMCACHED_HOSTS": "memcached:11211", } - return integration_tests_step("memcached", cmds, environment) + return integration_tests_steps("memcached", cmds, "memcached", "11211", environment) def release_canary_npm_packages_step(trigger = None): """Releases canary NPM packages. @@ -1138,15 +999,17 @@ def release_canary_npm_packages_step(trigger = None): """ step = { "name": "release-canary-npm-packages", - "image": images["build_image"], - "depends_on": end_to_end_tests_deps(), + "image": images["node"], + "depends_on": end_to_end_tests_deps() + ["build-frontend-packages"], "environment": { "NPM_TOKEN": from_secret(npm_token), }, "commands": [ + "apk add --update bash", "./scripts/publish-npm-packages.sh --dist-tag 'canary' --registry 'https://registry.npmjs.org'", ], } + if trigger: step = dict( step, @@ -1159,6 +1022,7 @@ def release_canary_npm_packages_step(trigger = None): }, ), ) + return step def upload_packages_step(ver_mode, trigger = None): @@ -1175,7 +1039,7 @@ def upload_packages_step(ver_mode, trigger = None): """ step = { "name": "upload-packages", - "image": images["publish_image"], + "image": images["publish"], "depends_on": end_to_end_tests_deps(), "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), @@ -1213,7 +1077,7 @@ def publish_grafanacom_step(ver_mode): return { "name": "publish-grafanacom", - "image": images["publish_image"], + "image": images["publish"], "depends_on": [ "publish-linux-packages-deb", "publish-linux-packages-rpm", @@ -1231,7 +1095,7 @@ def publish_linux_packages_step(package_manager = "deb"): return { "name": "publish-linux-packages-{}".format(package_manager), # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image - "image": images["package_publish_image"], + "image": images["package_publish"], "depends_on": ["compile-build-cmd"], "privileged": True, "settings": { @@ -1249,122 +1113,15 @@ def publish_linux_packages_step(package_manager = "deb"): }, } -def windows_clone_step(): - return { - "name": "clone", - "image": windows_images["wix_image"], - "environment": { - "GITHUB_TOKEN": from_secret("github_token"), - }, - "commands": [ - 'git clone "https://$$env:GITHUB_TOKEN@github.com/$$env:DRONE_REPO.git" .', - "git checkout -f $$env:DRONE_COMMIT", - ], - } - -def get_windows_steps(ver_mode, bucket = "%PRERELEASE_BUCKET%"): - """Generate the list of Windows steps. - - Args: - ver_mode: used to differentiate steps for different version modes. - bucket: used to override prerelease bucket. - - Returns: - List of Drone steps. - """ - steps = [ - identify_runner_step("windows"), - ] - - init_cmds = [ - '$$ProgressPreference = "SilentlyContinue"', - "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( - grabpl_version, - ), - ] - - steps.extend( - [ - { - "name": "windows-init", - "image": windows_images["wix_image"], - "commands": init_cmds, - }, - ], - ) - - if ver_mode in ( - "release", - "release-branch", - "main", - ): - gcp_bucket = "{}/artifacts/downloads".format(bucket) - if ver_mode == "release": - ver_part = "${DRONE_TAG}" - dir = "release" - else: - dir = "main" - gcp_bucket = "grafana-downloads" - build_no = "DRONE_BUILD_NUMBER" - ver_part = "--build-id $$env:{}".format(build_no) - installer_commands = [ - "$$gcpKey = $$env:GCP_KEY", - "[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json", - # gcloud fails to read the file unless converted with dos2unix - "dos2unix gcpkey.json", - "gcloud auth activate-service-account --key-file=gcpkey.json", - "rm gcpkey.json", - "cp C:\\App\\nssm-2.24.zip .", - ] - - if ver_mode == "release": - version = "${DRONE_TAG:1}" - installer_commands.extend( - [ - ".\\grabpl.exe windows-installer --target {} --edition oss {}".format("gs://{}/{}/oss/{}/grafana-{}.windows-amd64.zip".format(gcp_bucket, ver_part, ver_mode, version), ver_part), - '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', - 'gsutil cp "$$fname" gs://{}/{}/oss/{}/'.format(gcp_bucket, ver_part, dir), - 'gsutil cp "$$fname.sha256" gs://{}/{}/oss/{}/'.format(gcp_bucket, ver_part, dir), - ], - ) - if ver_mode in ("main"): - installer_commands.extend( - [ - ".\\grabpl.exe windows-installer --edition oss --build-id $$env:DRONE_BUILD_NUMBER", - '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', - 'gsutil cp "$$fname" gs://{}/oss/{}/'.format(gcp_bucket, dir), - 'gsutil cp "$$fname.sha256" gs://{}/oss/{}/'.format( - gcp_bucket, - dir, - ), - ], - ) - steps.append( - { - "name": "build-windows-installer", - "image": windows_images["wix_image"], - "depends_on": [ - "windows-init", - ], - "environment": { - "GCP_KEY": from_secret("gcp_key"), - "PRERELEASE_BUCKET": from_secret(prerelease_bucket), - "GITHUB_TOKEN": from_secret("github_token"), - }, - "commands": installer_commands, - }, - ) - - return steps - def verify_gen_cue_step(): return { "name": "verify-gen-cue", - "image": images["build_image"], + "image": images["go"], "depends_on": [], "commands": [ "# It is required that code generated from Thema/CUE be committed and in sync with its inputs.", "# The following command will fail if running code generators produces any diff in output.", + "apk add --update make", "CODEGEN_VERIFY=1 make gen-cue", ], } @@ -1372,11 +1129,12 @@ def verify_gen_cue_step(): def verify_gen_jsonnet_step(): return { "name": "verify-gen-jsonnet", - "image": images["build_image"], + "image": images["go"], "depends_on": [], "commands": [ "# It is required that generated jsonnet is committed and in sync with its inputs.", "# The following command will fail if running code generators produces any diff in output.", + "apk add --update make", "CODEGEN_VERIFY=1 make gen-jsonnet", ], } @@ -1394,7 +1152,7 @@ def compile_build_cmd(): return { "name": "compile-build-cmd", - "image": images["go_image"], + "image": images["go"], "commands": [ "go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd", ], @@ -1425,3 +1183,14 @@ def get_trigger_storybook(ver_mode): }, } return trigger_storybook + +def slack_step(channel, template, secret): + return { + "name": "slack", + "image": images["plugins_slack"], + "settings": { + "webhook": from_secret(secret), + "channel": channel, + "template": template, + }, + } diff --git a/scripts/drone/steps/lib_windows.star b/scripts/drone/steps/lib_windows.star new file mode 100644 index 00000000000..f9b11bcf85e --- /dev/null +++ b/scripts/drone/steps/lib_windows.star @@ -0,0 +1,187 @@ +""" +This module is a library of Drone steps that exclusively run on windows machines. +""" + +load( + "scripts/drone/utils/windows_images.star", + "windows_images", +) +load( + "scripts/drone/variables.star", + "grabpl_version", +) +load( + "scripts/drone/vault.star", + "from_secret", + "gcp_grafanauploads_base64", + "prerelease_bucket", +) + +def identify_runner_step_windows(): + return { + "name": "identify-runner", + "image": windows_images["1809"], + "commands": [ + "echo $env:DRONE_RUNNER_NAME", + ], + } + +def get_windows_steps(ver_mode, bucket = "%PRERELEASE_BUCKET%"): + """Generate the list of Windows steps. + + Args: + ver_mode: used to differentiate steps for different version modes. + bucket: used to override prerelease bucket. + + Returns: + List of Drone steps. + """ + steps = [ + identify_runner_step_windows(), + ] + + init_cmds = [ + '$$ProgressPreference = "SilentlyContinue"', + "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( + grabpl_version, + ), + ] + + steps.extend( + [ + { + "name": "windows-init", + "image": windows_images["wix"], + "commands": init_cmds, + }, + ], + ) + + if ver_mode in ( + "release", + "release-branch", + ): + gcp_bucket = "{}/artifacts/downloads".format(bucket) + if ver_mode == "release": + ver_part = "${DRONE_TAG}" + dir = "release" + else: + dir = "main" + gcp_bucket = "grafana-downloads" + build_no = "DRONE_BUILD_NUMBER" + ver_part = "--build-id $$env:{}".format(build_no) + installer_commands = [ + "$$gcpKey = $$env:GCP_KEY", + "[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json", + # gcloud fails to read the file unless converted with dos2unix + "dos2unix gcpkey.json", + "gcloud auth activate-service-account --key-file=gcpkey.json", + "rm gcpkey.json", + "cp C:\\App\\nssm-2.24.zip .", + ] + + if ver_mode in ("release",): + version = "${DRONE_TAG:1}" + installer_commands.extend( + [ + ".\\grabpl.exe windows-installer --target {} --edition oss {}".format( + "gs://{}/{}/oss/{}/grafana-{}.windows-amd64.zip".format(gcp_bucket, ver_part, ver_mode, version), + ver_part, + ), + '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', + ], + ) + if ver_mode == "main": + installer_commands.extend( + [ + "gsutil cp $$fname gs://{}/oss/{}/".format(gcp_bucket, dir), + 'gsutil cp "$$fname.sha256" gs://{}/oss/{}/'.format( + gcp_bucket, + dir, + ), + ], + ) + else: + installer_commands.extend( + [ + "gsutil cp $$fname gs://{}/{}/oss/{}/".format( + gcp_bucket, + ver_part, + dir, + ), + 'gsutil cp "$$fname.sha256" gs://{}/{}/oss/{}/'.format( + gcp_bucket, + ver_part, + dir, + ), + ], + ) + steps.append( + { + "name": "build-windows-installer", + "image": windows_images["wix"], + "depends_on": [ + "windows-init", + ], + "environment": { + "GCP_KEY": from_secret(gcp_grafanauploads_base64), + "PRERELEASE_BUCKET": from_secret(prerelease_bucket), + "GITHUB_TOKEN": from_secret("github_token"), + }, + "commands": installer_commands, + }, + ) + + return steps + +def download_grabpl_step_windows(): + return { + "name": "grabpl", + "image": windows_images["wix"], + "commands": [ + '$$ProgressPreference = "SilentlyContinue"', + "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( + grabpl_version, + ), + ], + } + +def test_backend_step_windows(): + # TODO: This is mostly a duplicate of "test_backend_step" in lib.star; but this file can't import that one, + # otherwise it creates an import cycle. + return { + "name": "test-backend", + "image": windows_images["go"], + "depends_on": [ + "wire-install", + ], + "commands": [ + "go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/...", + ], + } + +def clone_step_windows(): + return { + "name": "clone", + "image": windows_images["wix"], + "environment": { + "GITHUB_TOKEN": from_secret("github_token"), + }, + "commands": [ + 'git clone "https://$$env:GITHUB_TOKEN@github.com/$$env:DRONE_REPO.git" .', + "git checkout -f $$env:DRONE_COMMIT", + ], + } + +def wire_install_step_windows(edition): + return { + "name": "wire-install", + "image": windows_images["go"], + "commands": [ + "go install github.com/google/wire/cmd/wire@v0.5.0", + "wire gen -tags {} ./pkg/server".format(edition), + ], + "depends_on": [ + "windows-init", + ], + } diff --git a/scripts/drone/steps/rgm.star b/scripts/drone/steps/rgm.star new file mode 100644 index 00000000000..495b443e4a0 --- /dev/null +++ b/scripts/drone/steps/rgm.star @@ -0,0 +1,61 @@ +""" +Individual steps that use 'grafana-build' to replace existing individual steps. +These aren't used in releases. +""" + +load( + "scripts/drone/variables.star", + "golang_version", +) + +# rgm_package_step will create a tar.gz for use in e2e tests or other PR testing related activities.. +def rgm_package_step(distros = "linux/amd64,linux/arm64", file = "packages.txt"): + return { + "name": "rgm-package", + "image": "grafana/grafana-build:main", + "pull": "always", + "depends_on": ["yarn-install"], + "commands": [ + "/src/grafana-build package --distro={} ".format(distros) + + "--go-version={} ".format(golang_version) + + "--yarn-cache=$$YARN_CACHE_FOLDER " + + "--build-id=$$DRONE_BUILD_NUMBER " + + "--grafana-dir=$$PWD > {}".format(file), + ], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + } + +# rgm_build_backend will create compile the grafana backend for various platforms. It's preferred to use +# 'rgm_package_step' if you creating a "usable" artifact. This should really only be used to verify that the code is +# compilable. +def rgm_build_backend_step(distros = "linux/amd64,linux/arm64"): + return { + "name": "rgm-package", + "image": "grafana/grafana-build:main", + "pull": "always", + "commands": [ + "/src/grafana-build build " + + "--go-version={} ".format(golang_version) + + "--distro={} --grafana-dir=$$PWD".format(distros), + ], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + } + +def rgm_build_docker_step(packages, ubuntu, alpine, depends_on = ["rgm-package"], file = "docker.txt", tag_format = "{{ .version }}-{{ .arch }}", ubuntu_tag_format = "{{ .version }}-ubuntu-{{ .arch }}"): + return { + "name": "rgm-build-docker", + "image": "grafana/grafana-build:main", + "pull": "always", + "commands": [ + "docker run --privileged --rm tonistiigi/binfmt --install all", + "/src/grafana-build docker " + + "$(cat {} | grep tar.gz | grep -v docker | grep -v sha256 | awk '{{print \"--package=\" $0}}') ".format(packages) + + "--ubuntu-base={} ".format(ubuntu) + + "--alpine-base={} ".format(alpine) + + "--tag-format='{}' ".format(tag_format) + + "--ubuntu-tag-format='{}' > {}".format(ubuntu_tag_format, file), + "find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i", + ], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + "depends_on": depends_on, + } diff --git a/scripts/drone/utils/images.star b/scripts/drone/utils/images.star index b621583366a..7f5b40c0e1e 100644 --- a/scripts/drone/utils/images.star +++ b/scripts/drone/utils/images.star @@ -2,24 +2,35 @@ This module contains all the docker images that are used to build test and publish Grafana. """ +load( + "scripts/drone/variables.star", + "golang_version", + "nodejs_version", +) + images = { - "cloudsdk_image": "google/cloud-sdk:431.0.0", - "build_image": "grafana/build-container:1.7.5", - "publish_image": "grafana/grafana-ci-deploy:1.3.3", - "alpine_image": "alpine:3.18.3", - "curl_image": "byrnedo/alpine-curl:0.1.8", - "go_image": "golang:1.20.10", - "plugins_slack_image": "plugins/slack", - "postgres_alpine_image": "postgres:12.3-alpine", - "mysql5_image": "mysql:5.7.39", - "mysql8_image": "mysql:8.0.32", - "redis_alpine_image": "redis:6.2.11-alpine", - "memcached_alpine_image": "memcached:1.6.9-alpine", - "package_publish_image": "us.gcr.io/kubernetes-dev/package-publish:latest", - "openldap_image": "osixia/openldap:1.4.0", - "drone_downstream_image": "grafana/drone-downstream", - "docker_puppeteer_image": "grafana/docker-puppeteer:1.1.0", - "docs_image": "grafana/docs-base:dbd975af06", - "cypress_image": "cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97", - "cloud_datasources_e2e_image": "us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest", + "git": "alpine/git:2.40.1", + "go": "golang:{}-alpine".format(golang_version), + "node": "node:{}-alpine".format(nodejs_version), + "cloudsdk": "google/cloud-sdk:431.0.0", + "publish": "grafana/grafana-ci-deploy:1.3.3", + "alpine": "alpine:3.18.3", + "ubuntu": "ubuntu:22.04", + "curl": "byrnedo/alpine-curl:0.1.8", + "plugins_slack": "plugins/slack", + "python": "python:3.8", + "postgres_alpine": "postgres:12.3-alpine", + "mimir": "grafana/mimir:latest", + "mysql5": "mysql:5.7.39", + "mysql8": "mysql:8.0.32", + "redis_alpine": "redis:6.2.11-alpine", + "memcached_alpine": "memcached:1.6.9-alpine", + "package_publish": "us.gcr.io/kubernetes-dev/package-publish:latest", + "openldap": "osixia/openldap:1.4.0", + "drone_downstream": "grafana/drone-downstream", + "docker_puppeteer": "grafana/docker-puppeteer:1.1.0", + "docs": "grafana/docs-base:dbd975af06", + "cypress": "cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97", + "dockerize": "jwilder/dockerize:0.6.1", + "shellcheck": "koalaman/shellcheck:stable", } diff --git a/scripts/drone/utils/utils.star b/scripts/drone/utils/utils.star index 82cc0dca592..aea832018cf 100644 --- a/scripts/drone/utils/utils.star +++ b/scripts/drone/utils/utils.star @@ -9,7 +9,6 @@ load( load("scripts/drone/vault.star", "pull_secret") failure_template = "Build {{build.number}} failed for commit: : {{build.link}}\nBranch: \nAuthor: {{build.author}}" -drone_change_template = "`.drone.yml` and `starlark` files have been changed on the OSS repo, by: {{build.author}}. \nBranch: \nCommit hash: " def pipeline( name, diff --git a/scripts/drone/utils/windows_images.star b/scripts/drone/utils/windows_images.star index 2a605727ba8..a25c637a11e 100644 --- a/scripts/drone/utils/windows_images.star +++ b/scripts/drone/utils/windows_images.star @@ -4,9 +4,14 @@ All the windows images needed to be in a different file than the other images, s by trivy. Related issue: https://github.com/aquasecurity/trivy/issues/1392 """ +load( + "scripts/drone/variables.star", + "golang_version", +) + windows_images = { - "1809_image": "mcr.microsoft.com/windows:1809", - "wix_image": "grafana/ci-wix:0.1.1", - "windows_server_core_image": "docker:windowsservercore-1809", - "windows_go_image": "grafana/grafana-ci-windows-test:0.1.0", + "1809": "mcr.microsoft.com/windows:1809", + "wix": "grafana/ci-wix:0.1.1", + "windows_server_core": "docker:windowsservercore-1809", + "go": "golang:{}-windowsservercore-1809".format(golang_version), } diff --git a/scripts/drone/variables.star b/scripts/drone/variables.star new file mode 100644 index 00000000000..f4d1b97b322 --- /dev/null +++ b/scripts/drone/variables.star @@ -0,0 +1,9 @@ +""" +global variables +""" + +grabpl_version = "v3.0.42" +golang_version = "1.20.10" + +# nodejs_version should match what's in ".nvmrc", but without the v prefix. +nodejs_version = "18.12.0" diff --git a/scripts/drone/vault.star b/scripts/drone/vault.star index 12e92ddd026..6a27cf51785 100644 --- a/scripts/drone/vault.star +++ b/scripts/drone/vault.star @@ -14,6 +14,9 @@ azure_tenant = "azure_tenant" rgm_gcp_key_base64 = "gcp_key_base64" rgm_destination = "destination" +rgm_storybook_destination = "rgm_storybook_destination" +rgm_cdn_destination = "rgm_cdn_destination" +rgm_downloads_destination = "rgm_downloads_destination" rgm_github_token = "github_token" rgm_dagger_token = "dagger_token" @@ -122,6 +125,21 @@ def secrets(): "infra/data/ci/grafana-release-eng/rgm", "destination_prod", ), + vault_secret( + rgm_storybook_destination, + "infra/data/ci/grafana-release-eng/rgm", + "storybook_destination", + ), + vault_secret( + rgm_cdn_destination, + "infra/data/ci/grafana-release-eng/rgm", + "cdn_destination", + ), + vault_secret( + rgm_downloads_destination, + "infra/data/ci/grafana-release-eng/rgm", + "downloads_destination", + ), vault_secret( rgm_dagger_token, "infra/data/ci/grafana-release-eng/rgm", diff --git a/scripts/drone/version.star b/scripts/drone/version.star deleted file mode 100644 index 140cd4d35d8..00000000000 --- a/scripts/drone/version.star +++ /dev/null @@ -1,16 +0,0 @@ -""" -This module returns the pipeline used for version branches. -""" - -load( - "scripts/drone/events/release.star", - "oss_pipelines", -) - -ver_mode = "release-branch" -trigger = {"ref": ["refs/heads/v[0-9]*"]} - -def version_branch_pipelines(): - return ( - oss_pipelines(ver_mode = ver_mode, trigger = trigger) - ) diff --git a/scripts/validate-npm-packages.sh b/scripts/validate-npm-packages.sh index a07ebe5a662..4e2a27184fa 100755 --- a/scripts/validate-npm-packages.sh +++ b/scripts/validate-npm-packages.sh @@ -8,7 +8,7 @@ ARTIFACTS_DIR="./npm-artifacts" for file in "$ARTIFACTS_DIR"/*.tgz; do echo "🔍 Checking NPM package: $file" # get filename then strip everything after package name. - dir_name=$(basename "$file" .tgz | sed 's/^@\(.*\)-[0-9]*[.]*[0-9]*[.]*[0-9]*-\([0-9]*[a-zA-Z]*\)/\1/') + dir_name=$(basename "$file" .tgz | sed -E 's/@([a-zA-Z0-9-]+)-[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9-]+)?/\1/') mkdir -p "./npm-artifacts/$dir_name" tar -xzf "$file" -C "./npm-artifacts/$dir_name" --strip-components=1