diff --git a/.drone.star b/.drone.star index ce31860e844..a14146a7c14 100644 --- a/.drone.star +++ b/.drone.star @@ -7,8 +7,9 @@ This module returns a Drone configuration including pipelines and secrets. """ -load("scripts/drone/events/pr.star", "pr_pipelines") +load("scripts/drone/events/cron.star", "cronjobs") load("scripts/drone/events/main.star", "main_pipelines") +load("scripts/drone/events/pr.star", "pr_pipelines") load( "scripts/drone/events/release.star", "integration_test_pipelines", @@ -17,24 +18,22 @@ load( "publish_packages_pipeline", ) load( - "scripts/drone/rgm.star", - "rgm", + "scripts/drone/pipelines/ci_images.star", + "publish_ci_build_container_image_pipeline", + "publish_ci_windows_test_image_pipeline", ) load( "scripts/drone/pipelines/publish_images.star", "publish_image_pipelines_public", ) -load( - "scripts/drone/pipelines/ci_images.star", - "publish_ci_build_container_image_pipeline", - "publish_ci_windows_test_image_pipeline", -) load( "scripts/drone/pipelines/windows.star", "windows_test_backend", ) -load("scripts/drone/version.star", "version_branch_pipelines") -load("scripts/drone/events/cron.star", "cronjobs") +load( + "scripts/drone/rgm.star", + "rgm", +) load("scripts/drone/vault.star", "secrets") def main(_ctx): @@ -50,7 +49,6 @@ def main(_ctx): "event": ["promote"], "target": ["test-windows"], }, "oss", "testing")] + - version_branch_pipelines() + integration_test_pipelines() + publish_ci_windows_test_image_pipeline() + publish_ci_build_container_image_pipeline() + diff --git a/.drone.yml b/.drone.yml index 580f5bc59bd..2ab8f702f95 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,7 +24,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - ./bin/build verify-drone @@ -74,13 +74,14 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - - ./bin/build verify-starlark . + - go install github.com/bazelbuild/buildtools/buildifier@latest + - buildifier --lint=warn -mode=check -r . depends_on: - compile-build-cmd - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: lint-starlark trigger: event: @@ -118,24 +119,20 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: yarn-install - commands: + - apk add --update git bash - yarn betterer ci depends_on: - yarn-install - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: betterer-frontend - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -153,7 +150,7 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.4 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - yarn run ci:test-frontend @@ -161,7 +158,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: test-frontend trigger: event: @@ -198,6 +195,7 @@ platform: services: [] steps: - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -215,7 +213,7 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.4 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - echo $DRONE_RUNNER_NAME @@ -224,18 +222,17 @@ steps: - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: yarn-install - commands: - yarn run prettier:check - yarn run lint - - yarn run i18n:compile - yarn run typecheck depends_on: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: lint-frontend trigger: event: @@ -272,6 +269,7 @@ platform: services: [] steps: - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -289,53 +287,53 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.4 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: wire-install - commands: + - apk add --update build-base shared-mime-info shared-mime-info-lang - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: test-backend - commands: - - go test -run Integration -covermode=atomic -timeout=5m ./pkg/... + - apk add --update build-base + - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: test-backend-integration trigger: event: @@ -384,9 +382,10 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -404,21 +403,22 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.4 + image: alpine/git:2.40.1 name: clone-enterprise - commands: + - apk add --update make - make gen-go depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: wire-install - commands: - - apt-get update && apt-get install make + - apk add --update make build-base - make lint-go depends_on: - wire-install environment: CGO_ENABLED: "1" - image: golang:1.20.10 + image: golang:1.20.10-alpine name: lint-backend trigger: event: @@ -465,7 +465,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -474,36 +474,50 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: wire-install - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: yarn-install +- commands: + - apk add --update jq bash + - yarn packages:build + - yarn packages:pack + - ./scripts/validate-npm-packages.sh + depends_on: + - yarn-install + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: node:18.12.0-alpine + name: build-frontend-packages - failure: ignore image: grafana/drone-downstream name: trigger-enterprise-downstream @@ -518,64 +532,32 @@ steps: token: from_secret: drone_token - commands: - - ./bin/build build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - /src/grafana-build package --distro=linux/amd64,linux/arm64 --go-version=1.20.10 + --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD + > packages.txt depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.7.4 - name: build-backend -- commands: - - ./bin/build build-frontend --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.4 - name: build-frontend + image: grafana/grafana-build:main + name: rgm-package + pull: always + volumes: + - name: docker + path: /var/run/docker.sock - commands: - - ./bin/build build-frontend-packages --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} + - apk add --update tar bash + - mkdir grafana + - tar --strip-components=1 -xvf ./dist/*amd64.tar.gz -C grafana + - cp -r devenv scripts tools grafana && cd grafana && ./scripts/grafana-server/start-server depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.4 - name: build-frontend-packages -- commands: - - ./bin/build build-plugins --jobs 8 --edition oss - depends_on: - - compile-build-cmd - - yarn-install - environment: null - image: grafana/build-container:1.7.4 - name: build-plugins -- commands: - - . scripts/build/gpg-test-vars.sh && ./bin/build package --jobs 8 --edition oss - --build-id ${DRONE_BUILD_NUMBER} - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - environment: null - image: grafana/build-container:1.7.4 - name: package -- commands: - - ./scripts/grafana-server/start-server - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages + - rgm-package detach: true environment: - ARCH: linux-amd64 - PORT: 3001 - image: grafana/build-container:1.7.4 + GF_APP_MODE: development + GF_SERVER_HTTP_PORT: "3001" + GF_SERVER_ROUTER_LOGGING: "1" + image: alpine:3.18.3 name: grafana-server - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite dashboards-suite depends_on: - grafana-server @@ -584,7 +566,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-dashboards-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite depends_on: - grafana-server @@ -593,7 +574,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-smoke-tests-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite panels-suite depends_on: - grafana-server @@ -602,7 +582,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-panels-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite various-suite depends_on: - grafana-server @@ -626,13 +605,13 @@ steps: GITHUB_TOKEN: from_secret: github_token HOST: grafana-server - image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:3.0.0 name: end-to-end-tests-cloud-plugins-suite-azure when: paths: include: - pkg/tsdb/azuremonitor/** - - public/app/plugins/datasource/grafana-azure-monitor-datasource/** + - public/app/plugins/datasource/azuremonitor/** - e2e/cloud-plugins-suite/azure-monitor.spec.ts repo: - grafana/grafana @@ -671,23 +650,16 @@ steps: - yarn storybook:build - ./bin/build verify-storybook depends_on: - - build-frontend + - rgm-package - build-frontend-packages environment: NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: build-storybook when: paths: include: - packages/grafana-ui/** -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.7.4 - name: copy-packages-for-docker - commands: - yarn wait-on http://$HOST:$PORT - pa11y-ci --config .pa11yci-pr.conf.js @@ -702,36 +674,24 @@ steps: image: grafana/docker-puppeteer:1.1.0 name: test-a11y-frontend - commands: - - ./bin/build build-docker --edition oss -archs amd64 + - docker run --privileged --rm tonistiigi/binfmt --install all + - /src/grafana-build docker $(cat packages.txt | grep tar.gz | grep -v docker | + grep -v sha256 | awk '{print "--package=" $0}') --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.18.3 + --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --ubuntu-tag-format='{{ + .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > docker.txt + - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build build-docker --edition oss --ubuntu -archs amd64 - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images-ubuntu + - rgm-package + image: grafana/grafana-build:main + name: rgm-build-docker + pull: always volumes: - name: docker path: /var/run/docker.sock - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana depends_on: - - build-docker-images - - build-docker-images-ubuntu + - rgm-build-docker environment: DOCKER_PASSWORD: from_secret: docker_password @@ -788,18 +748,44 @@ services: volumes: - name: postgres path: /var/lib/postgresql/data/pgdata -- environment: +- commands: + - docker-entrypoint.sh mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + environment: MYSQL_DATABASE: grafana_tests MYSQL_PASSWORD: password MYSQL_ROOT_PASSWORD: rootpass MYSQL_USER: grafana image: mysql:5.7.39 - name: mysql + name: mysql57 volumes: - - name: mysql + - name: mysql57 path: /var/lib/mysql +- commands: + - docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password + environment: + MYSQL_DATABASE: grafana_tests + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_USER: grafana + image: mysql:8.0.32 + name: mysql80 + volumes: + - name: mysql80 + path: /var/lib/mysql +- commands: + - /bin/mimir -target=backend + environment: {} + image: grafana/mimir:latest + name: mimir_backend +- environment: {} + image: redis:6.2.11-alpine + name: redis +- environment: {} + image: memcached:1.6.9-alpine + name: memcached steps: - commands: + - apk add --update curl jq bash - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork) - if [ "$is_fork" != false ]; then return 1; fi @@ -817,11 +803,11 @@ steps: GITHUB_TOKEN: from_secret: github_token failure: ignore - image: grafana/build-container:1.7.4 + image: alpine/git:2.40.1 name: clone-enterprise - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -830,7 +816,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME @@ -841,56 +827,137 @@ steps: with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: wire-install - commands: - - apt-get update - - apt-get install -yq postgresql-client - dockerize -wait tcp://postgres:5432 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-postgres +- commands: + - apk add --update build-base + - apk add --update postgresql-client - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' + - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-postgres environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: postgres-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root + - dockerize -wait tcp://mysql57:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-5.7 +- commands: + - apk add --update build-base + - apk add --update mysql-client + - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql57 -P 3306 -u root -prootpass - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' + - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-5.7 environment: GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql - image: grafana/build-container:1.7.4 - name: mysql-integration-tests + MYSQL_HOST: mysql57 + image: golang:1.20.10-alpine + name: mysql-5.7-integration-tests +- commands: + - dockerize -wait tcp://mysql80:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-8.0 +- commands: + - apk add --update build-base + - apk add --update mysql-client + - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql80 -P 3306 -u root + -prootpass + - go clean -testcache + - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) + depends_on: + - wire-install + - wait-for-mysql-8.0 + environment: + GRAFANA_TEST_DB: mysql + MYSQL_HOST: mysql80 + image: golang:1.20.10-alpine + name: mysql-8.0-integration-tests +- commands: + - dockerize -wait tcp://redis:6379 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-redis +- commands: + - apk add --update build-base + - go clean -testcache + - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... + depends_on: + - wire-install + - wait-for-redis + environment: + REDIS_URL: redis://redis:6379/0 + image: golang:1.20.10-alpine + name: redis-integration-tests +- commands: + - dockerize -wait tcp://memcached:11211 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-memcached +- commands: + - apk add --update build-base + - go clean -testcache + - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... + depends_on: + - wire-install + - wait-for-memcached + environment: + MEMCACHED_HOSTS: memcached:11211 + image: golang:1.20.10-alpine + name: memcached-integration-tests +- commands: + - dockerize -wait tcp://mimir_backend:8080 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-remote-alertmanager +- commands: + - apk add --update build-base + - go clean -testcache + - go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/... + depends_on: + - wire-install + - wait-for-remote-alertmanager + environment: + AM_PASSWORD: test + AM_TENANT_ID: test + AM_URL: http://mimir_backend:8080 + image: golang:1.20.10-alpine + name: remote-alertmanager-integration-tests trigger: event: - pull_request @@ -914,7 +981,10 @@ volumes: - name: postgres temp: medium: memory -- name: mysql +- name: mysql57 + temp: + medium: memory +- name: mysql80 temp: medium: memory --- @@ -941,19 +1011,12 @@ steps: - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: yarn-install - commands: - - |- - echo -e "unknwon - referer - errorstring - eror - iam - wan" > words_to_ignore.txt - - codespell -I words_to_ignore.txt docs/ - - rm words_to_ignore.txt - image: grafana/build-container:1.7.4 + - pip3 install codespell + - codespell -I .codespellignore docs/ + image: python:3.8 name: codespell - commands: - yarn run prettier:checkDocs @@ -961,14 +1024,26 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: lint-docs - commands: - - mkdir -p /hugo/content/docs/grafana + - mkdir -p /hugo/content/docs/grafana/latest + - 'echo -e ''---\nredirectURL: /docs/grafana/latest/\ntype: redirect\nversioned: + true\n---\n'' > /hugo/content/docs/grafana/_index.md' - cp -r docs/sources/* /hugo/content/docs/grafana/latest/ - cd /hugo && make prod image: grafana/docs-base:dbd975af06 name: build-docs-website +- commands: + - '# It is required that code generated from Thema/CUE be committed and in sync + with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - apk add --update make + - CODEGEN_VERIFY=1 make gen-cue + depends_on: [] + image: golang:1.20.10-alpine + name: verify-gen-cue trigger: event: - pull_request @@ -978,6 +1053,8 @@ trigger: - docs/** - packages/**/*.md - latest.json + repo: + - grafana/grafana type: docker volumes: - host: @@ -1005,13 +1082,12 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - - ./bin/build shellcheck - depends_on: - - compile-build-cmd - image: grafana/build-container:1.7.4 + - apt-get update -yq && apt-get install shellcheck + - shellcheck -e SC1071 -e SC2162 scripts/**/*.sh + image: ubuntu:22.04 name: shellcheck trigger: event: @@ -1037,6 +1113,182 @@ environment: image_pull_secrets: - dockerconfigjson kind: pipeline +name: pr-integration-benchmarks +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: +- environment: + PGDATA: /var/lib/postgresql/data/pgdata + POSTGRES_DB: grafanatest + POSTGRES_PASSWORD: grafanatest + POSTGRES_USER: grafanatest + image: postgres:12.3-alpine + name: postgres + volumes: + - name: postgres + path: /var/lib/postgresql/data/pgdata +- commands: + - docker-entrypoint.sh mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + environment: + MYSQL_DATABASE: grafana_tests + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_USER: grafana + image: mysql:5.7.39 + name: mysql57 + volumes: + - name: mysql57 + path: /var/lib/mysql +- commands: + - docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password + environment: + MYSQL_DATABASE: grafana_tests + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_USER: grafana + image: mysql:8.0.32 + name: mysql80 + volumes: + - name: mysql80 + path: /var/lib/mysql +- commands: + - /bin/mimir -target=backend + environment: {} + image: grafana/mimir:latest + name: mimir_backend +- environment: {} + image: redis:6.2.11-alpine + name: redis +- environment: {} + image: memcached:1.6.9-alpine + name: memcached +steps: +- commands: + - apk add --update curl jq bash + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + ../grafana-enterprise + - cd ../grafana-enterprise + - if git checkout ${DRONE_SOURCE_BRANCH}; then echo "checked out ${DRONE_SOURCE_BRANCH}"; + elif git checkout ${DRONE_TARGET_BRANCH}; then echo "git checkout ${DRONE_TARGET_BRANCH}"; + else git checkout main; fi + - cd ../ + - ln -s src grafana + - cd ./grafana-enterprise + - ./build.sh + environment: + GITHUB_TOKEN: + from_secret: github_token + failure: ignore + image: alpine/git:2.40.1 + name: clone-enterprise +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] + environment: + CGO_ENABLED: 0 + image: golang:1.20.10-alpine + name: compile-build-cmd +- commands: + - '# It is required that code generated from Thema/CUE be committed and in sync + with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - apk add --update make + - CODEGEN_VERIFY=1 make gen-cue + depends_on: + - clone-enterprise + image: golang:1.20.10-alpine + name: verify-gen-cue +- commands: + - '# It is required that generated jsonnet is committed and in sync with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - apk add --update make + - CODEGEN_VERIFY=1 make gen-jsonnet + depends_on: + - clone-enterprise + image: golang:1.20.10-alpine + name: verify-gen-jsonnet +- commands: + - apk add --update make + - make gen-go + depends_on: + - verify-gen-cue + image: golang:1.20.10-alpine + name: wire-install +- commands: + - apk add --update build-base + - if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi + - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} + depends_on: + - wire-install + image: golang:1.20.10-alpine + name: sqlite-benchmark-integration-tests +- commands: + - apk add --update build-base + - if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi + - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} + depends_on: + - wire-install + environment: + GRAFANA_TEST_DB: postgres + PGPASSWORD: grafanatest + POSTGRES_HOST: postgres + image: golang:1.20.10-alpine + name: postgres-benchmark-integration-tests +- commands: + - apk add --update build-base + - if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi + - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} + depends_on: + - wire-install + environment: + GRAFANA_TEST_DB: mysql + MYSQL_HOST: mysql57 + image: golang:1.20.10-alpine + name: mysql-5.7-benchmark-integration-tests +- commands: + - apk add --update build-base + - if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi + - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} + depends_on: + - wire-install + environment: + GRAFANA_TEST_DB: mysql + MYSQL_HOST: mysql80 + image: golang:1.20.10-alpine + name: mysql-8.0-benchmark-integration-tests +trigger: + event: + - promote + target: + - gobenchmarks +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +- name: postgres + temp: + medium: memory +- name: mysql57 + temp: + medium: memory +- name: mysql80 + temp: + medium: memory +--- +clone: + retries: 3 +depends_on: [] +environment: + EDITION: oss +image_pull_secrets: +- dockerconfigjson +kind: pipeline name: main-docs node: type: no-parallel @@ -1052,19 +1304,12 @@ steps: - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: yarn-install - commands: - - |- - echo -e "unknwon - referer - errorstring - eror - iam - wan" > words_to_ignore.txt - - codespell -I words_to_ignore.txt docs/ - - rm words_to_ignore.txt - image: grafana/build-container:1.7.4 + - pip3 install codespell + - codespell -I .codespellignore docs/ + image: python:3.8 name: codespell - commands: - yarn run prettier:checkDocs @@ -1072,14 +1317,26 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: lint-docs - commands: - - mkdir -p /hugo/content/docs/grafana + - mkdir -p /hugo/content/docs/grafana/latest + - 'echo -e ''---\nredirectURL: /docs/grafana/latest/\ntype: redirect\nversioned: + true\n---\n'' > /hugo/content/docs/grafana/_index.md' - cp -r docs/sources/* /hugo/content/docs/grafana/latest/ - cd /hugo && make prod image: grafana/docs-base:dbd975af06 name: build-docs-website +- commands: + - '# It is required that code generated from Thema/CUE be committed and in sync + with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - apk add --update make + - CODEGEN_VERIFY=1 make gen-cue + depends_on: [] + image: golang:1.20.10-alpine + name: verify-gen-cue trigger: branch: main event: @@ -1090,6 +1347,8 @@ trigger: - docs/** - packages/**/*.md - latest.json + repo: + - grafana/grafana type: docker volumes: - host: @@ -1116,22 +1375,17 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: yarn-install - commands: + - apk add --update git bash - yarn betterer ci depends_on: - yarn-install - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: betterer-frontend - commands: - yarn run ci:test-frontend @@ -1139,7 +1393,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: test-frontend trigger: branch: main @@ -1150,6 +1404,8 @@ trigger: - '*.md' - docs/** - latest.json + repo: + - grafana/grafana type: docker volumes: - host: @@ -1179,18 +1435,17 @@ steps: - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: yarn-install - commands: - yarn run prettier:check - yarn run lint - - yarn run i18n:compile - yarn run typecheck depends_on: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: lint-frontend trigger: branch: main @@ -1201,6 +1456,8 @@ trigger: - '*.md' - docs/** - latest.json + repo: + - grafana/grafana type: docker volumes: - host: @@ -1227,47 +1484,47 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: wire-install - commands: + - apk add --update build-base shared-mime-info shared-mime-info-lang - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: test-backend - commands: - - go test -run Integration -covermode=atomic -timeout=5m ./pkg/... + - apk add --update build-base + - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: test-backend-integration trigger: branch: main @@ -1278,6 +1535,8 @@ trigger: - '*.md' - docs/** - latest.json + repo: + - grafana/grafana type: docker volumes: - host: @@ -1309,21 +1568,22 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: + - apk add --update make - make gen-go depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: wire-install - commands: - - apt-get update && apt-get install make + - apk add --update make build-base - make lint-go depends_on: - wire-install environment: CGO_ENABLED: "1" - image: golang:1.20.10 + image: golang:1.20.10-alpine name: lint-backend - commands: - ./bin/build verify-drone @@ -1340,6 +1600,8 @@ trigger: - '*.md' - docs/** - latest.json + repo: + - grafana/grafana type: docker volumes: - host: @@ -1368,7 +1630,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1377,104 +1639,89 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: wire-install - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: yarn-install - commands: - - ./bin/build build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - apk add --update jq + - new_version=$(cat package.json | jq .version | sed s/pre/${DRONE_BUILD_NUMBER}/g) + - 'echo "New version: $new_version"' + - yarn run lerna version $new_version --exact --no-git-tag-version --no-push --force-publish + -y + - yarn install --mode=update-lockfile depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.7.4 - name: build-backend -- commands: - - ./bin/build build-frontend --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - yarn-install + image: node:18.12.0-alpine + name: update-package-json-version +- commands: + - apk add --update jq bash + - yarn packages:build + - yarn packages:pack + - ./scripts/validate-npm-packages.sh + depends_on: + - yarn-install + - update-package-json-version environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.4 - name: build-frontend -- commands: - - ./bin/build build-frontend-packages --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: build-frontend-packages - commands: - - ./bin/build build-plugins --jobs 8 --edition oss + - /src/grafana-build package --distro=linux/amd64,linux/arm64 --go-version=1.20.10 + --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD + > packages.txt depends_on: - - compile-build-cmd - yarn-install - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.4 - name: build-plugins + image: grafana/grafana-build:main + name: rgm-package + pull: always + volumes: + - name: docker + path: /var/run/docker.sock - commands: - - ./bin/build package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --sign + - apk add --update tar bash + - mkdir grafana + - tar --strip-components=1 -xvf ./dist/*amd64.tar.gz -C grafana + - cp -r devenv scripts tools grafana && cd grafana && ./scripts/grafana-server/start-server depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - environment: - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.4 - name: package -- commands: - - ./scripts/grafana-server/start-server - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages + - rgm-package detach: true environment: - ARCH: linux-amd64 - PORT: 3001 - image: grafana/build-container:1.7.4 + GF_APP_MODE: development + GF_SERVER_HTTP_PORT: "3001" + GF_SERVER_ROUTER_LOGGING: "1" + image: alpine:3.18.3 name: grafana-server - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite dashboards-suite depends_on: - grafana-server @@ -1483,7 +1730,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-dashboards-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite depends_on: - grafana-server @@ -1492,7 +1738,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-smoke-tests-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite panels-suite depends_on: - grafana-server @@ -1501,7 +1746,6 @@ steps: image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-panels-suite - commands: - - apt-get install -y netcat - ./bin/build e2e-tests --port 3001 --suite various-suite depends_on: - grafana-server @@ -1525,13 +1769,13 @@ steps: GITHUB_TOKEN: from_secret: github_token HOST: grafana-server - image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:3.0.0 name: end-to-end-tests-cloud-plugins-suite-azure when: paths: include: - pkg/tsdb/azuremonitor/** - - public/app/plugins/datasource/grafana-azure-monitor-datasource/** + - public/app/plugins/datasource/azuremonitor/** - e2e/cloud-plugins-suite/azure-monitor.spec.ts repo: - grafana/grafana @@ -1570,23 +1814,16 @@ steps: - yarn storybook:build - ./bin/build verify-storybook depends_on: - - build-frontend + - rgm-package - build-frontend-packages environment: NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: build-storybook when: paths: include: - packages/grafana-ui/** -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.7.4 - name: copy-packages-for-docker - commands: - yarn wait-on http://$HOST:$PORT - pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json @@ -1622,49 +1859,39 @@ steps: repo: - grafana/grafana - commands: - - ./scripts/ci-frontend-metrics.sh | ./bin/build publish-metrics $${GRAFANA_MISC_STATS_API_KEY} + - apk add --update bash grep git + - ./scripts/ci-frontend-metrics.sh ./grafana/public/build | ./bin/build publish-metrics + $$GRAFANA_MISC_STATS_API_KEY depends_on: - test-a11y-frontend environment: GRAFANA_MISC_STATS_API_KEY: from_secret: grafana_misc_stats_api_key failure: ignore - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: publish-frontend-metrics when: repo: - grafana/grafana - commands: - - ./bin/build build-docker --edition oss + - docker run --privileged --rm tonistiigi/binfmt --install all + - /src/grafana-build docker $(cat packages.txt | grep tar.gz | grep -v docker | + grep -v sha256 | awk '{print "--package=" $0}') --ubuntu-base=ubuntu:22.04 --alpine-base=alpine:3.18.3 + --tag-format='{{ .version_base }}-{{ .buildID }}-{{ .arch }}' --ubuntu-tag-format='{{ + .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}' > docker.txt + - find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build build-docker --edition oss --ubuntu - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images-ubuntu + - rgm-package + image: grafana/grafana-build:main + name: rgm-build-docker + pull: always volumes: - name: docker path: /var/run/docker.sock - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana depends_on: - - build-docker-images - - build-docker-images-ubuntu + - rgm-build-docker environment: DOCKER_PASSWORD: from_secret: docker_password @@ -1689,8 +1916,7 @@ steps: - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana-oss depends_on: - - build-docker-images - - build-docker-images-ubuntu + - rgm-build-docker environment: DOCKER_PASSWORD: from_secret: docker_password @@ -1713,18 +1939,23 @@ steps: repo: - grafana/grafana - commands: - - ./scripts/circle-release-canary-packages.sh + - apk add --update bash + - ./scripts/publish-npm-packages.sh --dist-tag 'canary' --registry 'https://registry.npmjs.org' depends_on: - end-to-end-tests-dashboards-suite - end-to-end-tests-panels-suite - end-to-end-tests-smoke-tests-suite - end-to-end-tests-various-suite + - build-frontend-packages environment: NPM_TOKEN: from_secret: npm_token - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: release-canary-npm-packages when: + paths: + include: + - packages/** repo: - grafana/grafana - commands: @@ -1767,6 +1998,8 @@ trigger: - '*.md' - docs/** - latest.json + repo: + - grafana/grafana type: docker volumes: - host: @@ -1798,20 +2031,45 @@ services: volumes: - name: postgres path: /var/lib/postgresql/data/pgdata -- environment: +- commands: + - docker-entrypoint.sh mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + environment: MYSQL_DATABASE: grafana_tests MYSQL_PASSWORD: password MYSQL_ROOT_PASSWORD: rootpass MYSQL_USER: grafana image: mysql:5.7.39 - name: mysql + name: mysql57 volumes: - - name: mysql + - name: mysql57 path: /var/lib/mysql +- commands: + - docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password + environment: + MYSQL_DATABASE: grafana_tests + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_USER: grafana + image: mysql:8.0.32 + name: mysql80 + volumes: + - name: mysql80 + path: /var/lib/mysql +- commands: + - /bin/mimir -target=backend + environment: {} + image: grafana/mimir:latest + name: mimir_backend +- environment: {} + image: redis:6.2.11-alpine + name: redis +- environment: {} + image: memcached:1.6.9-alpine + name: memcached steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1820,7 +2078,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME @@ -1831,56 +2089,137 @@ steps: with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: wire-install - commands: - - apt-get update - - apt-get install -yq postgresql-client - dockerize -wait tcp://postgres:5432 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-postgres +- commands: + - apk add --update build-base + - apk add --update postgresql-client - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' + - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-postgres environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: postgres-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root + - dockerize -wait tcp://mysql57:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-5.7 +- commands: + - apk add --update build-base + - apk add --update mysql-client + - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql57 -P 3306 -u root -prootpass - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' + - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-5.7 environment: GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql - image: grafana/build-container:1.7.4 - name: mysql-integration-tests + MYSQL_HOST: mysql57 + image: golang:1.20.10-alpine + name: mysql-5.7-integration-tests +- commands: + - dockerize -wait tcp://mysql80:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-8.0 +- commands: + - apk add --update build-base + - apk add --update mysql-client + - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql80 -P 3306 -u root + -prootpass + - go clean -testcache + - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) + depends_on: + - wire-install + - wait-for-mysql-8.0 + environment: + GRAFANA_TEST_DB: mysql + MYSQL_HOST: mysql80 + image: golang:1.20.10-alpine + name: mysql-8.0-integration-tests +- commands: + - dockerize -wait tcp://redis:6379 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-redis +- commands: + - apk add --update build-base + - go clean -testcache + - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... + depends_on: + - wire-install + - wait-for-redis + environment: + REDIS_URL: redis://redis:6379/0 + image: golang:1.20.10-alpine + name: redis-integration-tests +- commands: + - dockerize -wait tcp://memcached:11211 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-memcached +- commands: + - apk add --update build-base + - go clean -testcache + - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... + depends_on: + - wire-install + - wait-for-memcached + environment: + MEMCACHED_HOSTS: memcached:11211 + image: golang:1.20.10-alpine + name: memcached-integration-tests +- commands: + - dockerize -wait tcp://mimir_backend:8080 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-remote-alertmanager +- commands: + - apk add --update build-base + - go clean -testcache + - go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/... + depends_on: + - wire-install + - wait-for-remote-alertmanager + environment: + AM_PASSWORD: test + AM_TENANT_ID: test + AM_URL: http://mimir_backend:8080 + image: golang:1.20.10-alpine + name: remote-alertmanager-integration-tests trigger: branch: main event: @@ -1890,6 +2229,8 @@ trigger: - '*.md' - docs/** - latest.json + repo: + - grafana/grafana type: docker volumes: - host: @@ -1898,7 +2239,10 @@ volumes: - name: postgres temp: medium: memory -- name: mysql +- name: mysql57 + temp: + medium: memory +- name: mysql80 temp: medium: memory --- @@ -1927,33 +2271,10 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: windows-init -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - - .\grabpl.exe windows-installer --edition oss --build-id $$env:DRONE_BUILD_NUMBER - - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp "$$fname" gs://grafana-downloads/oss/main/ - - gsutil cp "$$fname.sha256" gs://grafana-downloads/oss/main/ - depends_on: - - windows-init - environment: - GCP_KEY: - from_secret: gcp_grafanauploads_base64 - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer trigger: branch: main event: @@ -1971,38 +2292,6 @@ volumes: path: //./pipe/docker_engine/ name: docker --- -clone: - retries: 3 -depends_on: [] -kind: pipeline -name: notify-drone-changes -platform: - arch: amd64 - os: linux -steps: -- image: plugins/slack - name: slack - settings: - channel: slack-webhooks-test - template: "`.drone.yml` and `starlark` files have been changed on the OSS repo, - by: {{build.author}}. \nBranch: \nCommit hash: " - webhook: - from_secret: drone-changes-webhook -trigger: - branch: main - event: - - push - paths: - exclude: - - exclude - include: - - .drone.yml - repo: - - grafana/grafana -type: docker ---- clone: retries: 3 depends_on: @@ -2082,6 +2371,8 @@ trigger: - '*.md' - docs/** - latest.json + repo: + - grafana/grafana status: - failure type: docker @@ -2108,7 +2399,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2117,7 +2408,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - ./bin/build artifacts docker fetch --edition oss @@ -2213,7 +2504,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - ./bin/build artifacts packages --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET} @@ -2282,12 +2573,12 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: yarn-install - commands: - ./bin/build artifacts npm retrieve --tag ${DRONE_TAG} @@ -2311,7 +2602,7 @@ steps: NPM_TOKEN: from_secret: npm_token failure: ignore - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: release-npm-packages trigger: event: @@ -2347,7 +2638,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - depends_on: - compile-build-cmd @@ -2414,6 +2705,79 @@ volumes: path: /var/run/docker.sock name: docker --- +clone: + retries: 3 +depends_on: +- main-test-backend +- main-test-frontend +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-main-prerelease +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_publish_main.sh + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.20.10 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: grafana/grafana-build:main + name: rgm-build + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + branch: main + event: + - push + paths: + exclude: + - '*.md' + - docs/** + - packages/**/*.md + - latest.json + repo: + - grafana/grafana +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- clone: retries: 3 depends_on: [] @@ -2435,13 +2799,13 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.10 + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - ./bin/build whatsnew-checker depends_on: - compile-build-cmd - image: golang:1.20.10 + image: golang:1.20.10-alpine name: whats-new-checker trigger: event: @@ -2478,22 +2842,17 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: yarn-install - commands: + - apk add --update git bash - yarn betterer ci depends_on: - yarn-install - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: betterer-frontend - commands: - yarn run ci:test-frontend @@ -2501,7 +2860,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.4 + image: node:18.12.0-alpine name: test-frontend trigger: event: @@ -2538,47 +2897,47 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.18.3 name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: wire-install - commands: + - apk add --update build-base shared-mime-info shared-mime-info-lang - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: test-backend - commands: - - go test -run Integration -covermode=atomic -timeout=5m ./pkg/... + - apk add --update build-base + - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: test-backend-integration trigger: event: @@ -2598,64 +2957,6 @@ volumes: clone: retries: 3 depends_on: -- main-test-backend -- main-test-frontend -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: rgm-main-prerelease -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - export GRAFANA_DIR=$$(pwd) - - cd /src && ./scripts/drone_publish_main.sh - environment: - _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: - from_secret: dagger_token - DESTINATION: - from_secret: destination - GCP_KEY_BASE64: - from_secret: gcp_key_base64 - GITHUB_TOKEN: - from_secret: github_token - GPG_PASSPHRASE: - from_secret: packages_gpg_passphrase - GPG_PRIVATE_KEY: - from_secret: packages_gpg_private_key - GPG_PUBLIC_KEY: - from_secret: packages_gpg_public_key - failure: ignore - image: grafana/grafana-build:main - name: rgm-build - volumes: - - name: docker - path: /var/run/docker.sock -trigger: - branch: main - event: - - push - paths: - exclude: - - '*.md' - - docs/** - - packages/**/*.md - - latest.json - repo: - - grafana/grafana -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: - release-test-backend - release-test-frontend image_pull_secrets: @@ -2675,20 +2976,36 @@ steps: environment: _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination DESTINATION: from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key GCP_KEY_BASE64: from_secret: gcp_key_base64 GITHUB_TOKEN: from_secret: github_token + GO_VERSION: 1.20.10 GPG_PASSPHRASE: from_secret: packages_gpg_passphrase GPG_PRIVATE_KEY: from_secret: packages_gpg_private_key GPG_PUBLIC_KEY: from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination image: grafana/grafana-build:main name: rgm-build + pull: always volumes: - name: docker path: /var/run/docker.sock @@ -2723,14 +3040,12 @@ services: [] steps: - commands: - echo $env:DRONE_RUNNER_NAME - failure: ignore image: mcr.microsoft.com/windows:1809 name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/windows/grabpl.exe -OutFile grabpl.exe - failure: ignore image: grafana/ci-wix:0.1.1 name: windows-init - commands: @@ -2744,7 +3059,7 @@ steps: - .\grabpl.exe windows-installer --target gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/grafana-${DRONE_TAG:1}.windows-amd64.zip --edition oss ${DRONE_TAG} - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp "$$fname" gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/ + - gsutil cp $$fname gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/ - gsutil cp "$$fname.sha256" gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/ depends_on: - windows-init @@ -2755,7 +3070,6 @@ steps: from_secret: github_token PRERELEASE_BUCKET: from_secret: prerelease_bucket - failure: ignore image: grafana/ci-wix:0.1.1 name: build-windows-installer trigger: @@ -2818,6 +3132,488 @@ volumes: path: /var/run/docker.sock name: docker --- +clone: + retries: 3 +depends_on: +- release-test-backend +- release-test-frontend +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-version-branch-prerelease +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_publish_tag_grafana.sh + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.20.10 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: grafana/grafana-build:main + name: rgm-build + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + ref: + - refs/heads/v[0-9]* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: +- rgm-version-branch-prerelease +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-prerelease-verify-prerelease-assets +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - apt-get update && apt-get install -yq gettext + - printenv GCP_KEY | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - ./scripts/list-release-artifacts.sh ${DRONE_TAG} | xargs -n1 gsutil stat >> /tmp/stat.log + - '! cat /tmp/stat.log | grep "No URLs matched"' + depends_on: + - clone + environment: + BUCKET: grafana-prerelease + GCP_KEY: + from_secret: gcp_key_base64 + image: google/cloud-sdk:431.0.0 + name: gsutil-stat +trigger: + ref: + - refs/heads/v[0-9]* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: [] +environment: + EDITION: oss +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: nightly-test-frontend +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.18.3 + name: identify-runner +- commands: + - yarn install --immutable + depends_on: [] + image: node:18.12.0-alpine + name: yarn-install +- commands: + - apk add --update git bash + - yarn betterer ci + depends_on: + - yarn-install + image: node:18.12.0-alpine + name: betterer-frontend +- commands: + - yarn run ci:test-frontend + depends_on: + - yarn-install + environment: + TEST_MAX_WORKERS: 50% + image: node:18.12.0-alpine + name: test-frontend +trigger: + cron: + include: + - nightly-release + event: + include: + - cron +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: [] +environment: + EDITION: oss +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: nightly-test-backend +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.18.3 + name: identify-runner +- commands: + - '# It is required that code generated from Thema/CUE be committed and in sync + with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - apk add --update make + - CODEGEN_VERIFY=1 make gen-cue + depends_on: [] + image: golang:1.20.10-alpine + name: verify-gen-cue +- commands: + - '# It is required that generated jsonnet is committed and in sync with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - apk add --update make + - CODEGEN_VERIFY=1 make gen-jsonnet + depends_on: [] + image: golang:1.20.10-alpine + name: verify-gen-jsonnet +- commands: + - apk add --update make + - make gen-go + depends_on: + - verify-gen-cue + image: golang:1.20.10-alpine + name: wire-install +- commands: + - apk add --update build-base shared-mime-info shared-mime-info-lang + - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... + depends_on: + - wire-install + image: golang:1.20.10-alpine + name: test-backend +- commands: + - apk add --update build-base + - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) + depends_on: + - wire-install + image: golang:1.20.10-alpine + name: test-backend-integration +trigger: + cron: + include: + - nightly-release + event: + include: + - cron +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: +- nightly-test-backend +- nightly-test-frontend +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-nightly-build +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_build_nightly_grafana.sh + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.20.10 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: grafana/grafana-build:main + name: rgm-build + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - mkdir -p $${DESTINATION}/$${DRONE_BUILD_EVENT} + - printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - gcloud storage cp -r $${DRONE_WORKSPACE}/dist/* $${DESTINATION}/$${DRONE_BUILD_EVENT} + depends_on: + - rgm-build + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: google/cloud-sdk:alpine + name: rgm-copy +trigger: + cron: + include: + - nightly-release + event: + include: + - cron +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: +- rgm-nightly-build +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: rgm-nightly-publish +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p $${DRONE_WORKSPACE}/dist + - printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - gcloud storage cp -r $${DESTINATION}/$${DRONE_BUILD_EVENT}/*_$${DRONE_BUILD_NUMBER}_* + $${DRONE_WORKSPACE}/dist + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: google/cloud-sdk:alpine + name: rgm-copy +- commands: + - export GRAFANA_DIR=$$(pwd) + - cd /src && ./scripts/drone_publish_nightly_grafana.sh + depends_on: + - rgm-copy + environment: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: + from_secret: dagger_token + CDN_DESTINATION: + from_secret: rgm_cdn_destination + DESTINATION: + from_secret: destination + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + DOWNLOADS_DESTINATION: + from_secret: rgm_downloads_destination + GCOM_API_KEY: + from_secret: grafana_api_key + GCP_KEY_BASE64: + from_secret: gcp_key_base64 + GITHUB_TOKEN: + from_secret: github_token + GO_VERSION: 1.20.10 + GPG_PASSPHRASE: + from_secret: packages_gpg_passphrase + GPG_PRIVATE_KEY: + from_secret: packages_gpg_private_key + GPG_PUBLIC_KEY: + from_secret: packages_gpg_public_key + NPM_TOKEN: + from_secret: npm_token + STORYBOOK_DESTINATION: + from_secret: rgm_storybook_destination + image: grafana/grafana-build:main + name: rgm-publish + pull: always + volumes: + - name: docker + path: /var/run/docker.sock +- depends_on: + - rgm-publish + image: us.gcr.io/kubernetes-dev/package-publish:latest + name: publish-deb + privileged: true + settings: + access_key_id: + from_secret: packages_access_key_id + gpg_passphrase: + from_secret: packages_gpg_passphrase + gpg_private_key: + from_secret: packages_gpg_private_key + gpg_public_key: + from_secret: packages_gpg_public_key + package_path: file:///drone/src/dist/*.deb + secret_access_key: + from_secret: packages_secret_access_key + service_account_json: + from_secret: packages_service_account + target_bucket: grafana-packages +- depends_on: + - rgm-publish + image: us.gcr.io/kubernetes-dev/package-publish:latest + name: publish-rpm + privileged: true + settings: + access_key_id: + from_secret: packages_access_key_id + gpg_passphrase: + from_secret: packages_gpg_passphrase + gpg_private_key: + from_secret: packages_gpg_private_key + gpg_public_key: + from_secret: packages_gpg_public_key + package_path: file:///drone/src/dist/*.rpm + secret_access_key: + from_secret: packages_secret_access_key + service_account_json: + from_secret: packages_service_account + target_bucket: grafana-packages +trigger: + cron: + include: + - nightly-release + event: + include: + - cron +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- clone: disable: true depends_on: [] @@ -2844,20 +3640,20 @@ steps: - commands: [] depends_on: - clone - image: grafana/grafana-ci-windows-test:0.1.0 + image: golang:1.20.10-windowsservercore-1809 name: windows-init - commands: - go install github.com/google/wire/cmd/wire@v0.5.0 - wire gen -tags oss ./pkg/server depends_on: - windows-init - image: grafana/grafana-ci-windows-test:0.1.0 + image: golang:1.20.10-windowsservercore-1809 name: wire-install - commands: - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/grafana-ci-windows-test:0.1.0 + image: golang:1.20.10-windowsservercore-1809 name: test-backend trigger: event: @@ -2878,581 +3674,6 @@ environment: image_pull_secrets: - dockerconfigjson kind: pipeline -name: release-branch-build-e2e-publish -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 - name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: [] - image: grafana/build-container:1.7.4 - name: verify-gen-cue -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.7.4 - name: wire-install -- commands: - - yarn install --immutable - depends_on: [] - image: grafana/build-container:1.7.4 - name: yarn-install -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd -- commands: - - ./bin/build build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} - depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.7.4 - name: build-backend -- commands: - - ./bin/build build-frontend --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.4 - name: build-frontend -- commands: - - ./bin/build build-frontend-packages --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.4 - name: build-frontend-packages -- commands: - - ./bin/build build-plugins --jobs 8 --edition oss - depends_on: - - compile-build-cmd - - yarn-install - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.4 - name: build-plugins -- commands: - - ./bin/build package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --sign - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - environment: - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.4 - name: package -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.7.4 - name: copy-packages-for-docker -- commands: - - ./bin/build build-docker --edition oss --shouldSave - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build build-docker --edition oss --shouldSave --ubuntu - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - image: google/cloud-sdk:431.0.0 - name: build-docker-images-ubuntu - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./scripts/grafana-server/start-server - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - detach: true - environment: - ARCH: linux-amd64 - PORT: 3001 - image: grafana/build-container:1.7.4 - name: grafana-server -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite dashboards-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-dashboards-suite -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-smoke-tests-suite -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite panels-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-panels-suite -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite various-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-various-suite -- commands: - - apt-get update - - apt-get install -yq zip - - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ - - gsutil cp e2e/videos.zip gs://$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - export E2E_ARTIFACTS_VIDEO_ZIP=https://storage.googleapis.com/$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - 'echo "E2E Test artifacts uploaded to: $${E2E_ARTIFACTS_VIDEO_ZIP}"' - - 'curl -X POST https://api.github.com/repos/${DRONE_REPO}/statuses/${DRONE_COMMIT_SHA} - -H "Authorization: token $${GITHUB_TOKEN}" -d "{\"state\":\"success\",\"target_url\":\"$${E2E_ARTIFACTS_VIDEO_ZIP}\", - \"description\": \"Click on the details to download e2e recording videos\", \"context\": - \"e2e_artifacts\"}"' - depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - E2E_TEST_ARTIFACTS_BUCKET: releng-pipeline-artifacts-dev - GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY: - from_secret: gcp_upload_artifacts_key - GITHUB_TOKEN: - from_secret: github_token - failure: ignore - image: google/cloud-sdk:431.0.0 - name: e2e-tests-artifacts-upload - when: - status: - - success - - failure -- commands: - - yarn storybook:build - - ./bin/build verify-storybook - depends_on: - - build-frontend - - build-frontend-packages - environment: - NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.7.4 - name: build-storybook - when: - paths: - include: - - packages/grafana-ui/** -- commands: - - ./bin/build upload-cdn --edition oss - depends_on: - - grafana-server - environment: - GCP_KEY: - from_secret: gcp_grafanauploads - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-cdn-assets - when: - repo: - - grafana/grafana -- commands: - - ./bin/build upload-packages --edition oss - depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - GCP_KEY: - from_secret: gcp_grafanauploads_base64 - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-packages - when: - repo: - - grafana/grafana -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql - temp: - medium: memory ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-test-frontend -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 - name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - yarn install --immutable - depends_on: [] - image: grafana/build-container:1.7.4 - name: yarn-install -- commands: - - yarn betterer ci - depends_on: - - yarn-install - image: grafana/build-container:1.7.4 - name: betterer-frontend -- commands: - - yarn run ci:test-frontend - depends_on: - - yarn-install - environment: - TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.4 - name: test-frontend -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-test-backend -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 - name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: [] - image: grafana/build-container:1.7.4 - name: verify-gen-cue -- commands: - - '# It is required that generated jsonnet is committed and in sync with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-jsonnet - depends_on: [] - image: grafana/build-container:1.7.4 - name: verify-gen-jsonnet -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.7.4 - name: wire-install -- commands: - - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... - depends_on: - - wire-install - image: grafana/build-container:1.7.4 - name: test-backend -- commands: - - go test -run Integration -covermode=atomic -timeout=5m ./pkg/... - depends_on: - - wire-install - image: grafana/build-container:1.7.4 - name: test-backend-integration -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-integration-tests -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: -- environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_DB: grafanatest - POSTGRES_PASSWORD: grafanatest - POSTGRES_USER: grafanatest - image: postgres:12.3-alpine - name: postgres - volumes: - - name: postgres - path: /var/lib/postgresql/data/pgdata -- environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:5.7.39 - name: mysql - volumes: - - name: mysql - path: /var/lib/mysql -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.18.3 - name: identify-runner -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: [] - image: grafana/build-container:1.7.4 - name: verify-gen-cue -- commands: - - '# It is required that generated jsonnet is committed and in sync with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-jsonnet - depends_on: [] - image: grafana/build-container:1.7.4 - name: verify-gen-jsonnet -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.7.4 - name: wire-install -- commands: - - apt-get update - - apt-get install -yq postgresql-client - - dockerize -wait tcp://postgres:5432 -timeout 120s - - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' - depends_on: - - wire-install - environment: - GRAFANA_TEST_DB: postgres - PGPASSWORD: grafanatest - POSTGRES_HOST: postgres - image: grafana/build-container:1.7.4 - name: postgres-integration-tests -- commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root - -prootpass - - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' - depends_on: - - wire-install - environment: - GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql - image: grafana/build-container:1.7.4 - name: mysql-integration-tests -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql - temp: - medium: memory ---- -clone: - retries: 3 -depends_on: -- release-branch-build-e2e-publish -- release-branch-test-frontend -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-windows -platform: - arch: amd64 - os: windows - version: "1809" -services: [] -steps: -- commands: - - echo $env:DRONE_RUNNER_NAME - image: mcr.microsoft.com/windows:1809 - name: identify-runner -- commands: - - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/windows/grabpl.exe - -OutFile grabpl.exe - image: grafana/ci-wix:0.1.1 - name: windows-init -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - depends_on: - - windows-init - environment: - GCP_KEY: - from_secret: gcp_grafanauploads_base64 - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: //./pipe/docker_engine/ - name: docker ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- dockerconfigjson -kind: pipeline name: integration-tests node: type: no-parallel @@ -3470,20 +3691,45 @@ services: volumes: - name: postgres path: /var/lib/postgresql/data/pgdata -- environment: +- commands: + - docker-entrypoint.sh mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + environment: MYSQL_DATABASE: grafana_tests MYSQL_PASSWORD: password MYSQL_ROOT_PASSWORD: rootpass MYSQL_USER: grafana image: mysql:5.7.39 - name: mysql + name: mysql57 volumes: - - name: mysql + - name: mysql57 path: /var/lib/mysql +- commands: + - docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password + environment: + MYSQL_DATABASE: grafana_tests + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_USER: grafana + image: mysql:8.0.32 + name: mysql80 + volumes: + - name: mysql80 + path: /var/lib/mysql +- commands: + - /bin/mimir -target=backend + environment: {} + image: grafana/mimir:latest + name: mimir_backend +- environment: {} + image: redis:6.2.11-alpine + name: redis +- environment: {} + image: memcached:1.6.9-alpine + name: memcached steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.41/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.42/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -3496,56 +3742,137 @@ steps: with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' - '# The following command will fail if running code generators produces any diff in output.' + - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: + - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: wire-install - commands: - - apt-get update - - apt-get install -yq postgresql-client - dockerize -wait tcp://postgres:5432 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-postgres +- commands: + - apk add --update build-base + - apk add --update postgresql-client - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' + - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-postgres environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.7.4 + image: golang:1.20.10-alpine name: postgres-integration-tests - commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root + - dockerize -wait tcp://mysql57:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-5.7 +- commands: + - apk add --update build-base + - apk add --update mysql-client + - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql57 -P 3306 -u root -prootpass - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' + - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install + - wait-for-mysql-5.7 environment: GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql - image: grafana/build-container:1.7.4 - name: mysql-integration-tests + MYSQL_HOST: mysql57 + image: golang:1.20.10-alpine + name: mysql-5.7-integration-tests +- commands: + - dockerize -wait tcp://mysql80:3306 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-mysql-8.0 +- commands: + - apk add --update build-base + - apk add --update mysql-client + - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql80 -P 3306 -u root + -prootpass + - go clean -testcache + - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find + ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' + | grep -o '\(.*\)/' | sort -u) + depends_on: + - wire-install + - wait-for-mysql-8.0 + environment: + GRAFANA_TEST_DB: mysql + MYSQL_HOST: mysql80 + image: golang:1.20.10-alpine + name: mysql-8.0-integration-tests +- commands: + - dockerize -wait tcp://redis:6379 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-redis +- commands: + - apk add --update build-base + - go clean -testcache + - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... + depends_on: + - wire-install + - wait-for-redis + environment: + REDIS_URL: redis://redis:6379/0 + image: golang:1.20.10-alpine + name: redis-integration-tests +- commands: + - dockerize -wait tcp://memcached:11211 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-memcached +- commands: + - apk add --update build-base + - go clean -testcache + - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... + depends_on: + - wire-install + - wait-for-memcached + environment: + MEMCACHED_HOSTS: memcached:11211 + image: golang:1.20.10-alpine + name: memcached-integration-tests +- commands: + - dockerize -wait tcp://mimir_backend:8080 -timeout 120s + image: jwilder/dockerize:0.6.1 + name: wait-for-remote-alertmanager +- commands: + - apk add --update build-base + - go clean -testcache + - go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/... + depends_on: + - wire-install + - wait-for-remote-alertmanager + environment: + AM_PASSWORD: test + AM_TENANT_ID: test + AM_URL: http://mimir_backend:8080 + image: golang:1.20.10-alpine + name: remote-alertmanager-integration-tests trigger: event: - promote @@ -3558,7 +3885,10 @@ volumes: - name: postgres temp: medium: memory -- name: mysql +- name: mysql57 + temp: + medium: memory +- name: mysql80 temp: medium: memory --- @@ -3684,7 +4014,7 @@ steps: - name: config path: /root/.docker/ - commands: - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest + - trivy image --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -3692,15 +4022,22 @@ steps: volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - commands: - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest + - trivy image --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest depends_on: - authenticate-gcr + environment: + GOOGLE_APPLICATION_CREDENTIALS: + from_secret: gcr_credentials_json image: aquasec/trivy:0.21.0 name: scan-high-critical-vulnerabilities volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - image: plugins/slack name: slack-notify-failure settings: @@ -3718,6 +4055,8 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: config + temp: {} --- clone: retries: 3 @@ -3740,7 +4079,7 @@ steps: - name: config path: /root/.docker/ - commands: - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main + - trivy image --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -3748,15 +4087,22 @@ steps: volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - commands: - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main + - trivy image --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main depends_on: - authenticate-gcr + environment: + GOOGLE_APPLICATION_CREDENTIALS: + from_secret: gcr_credentials_json image: aquasec/trivy:0.21.0 name: scan-high-critical-vulnerabilities volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - image: plugins/slack name: slack-notify-failure settings: @@ -3774,6 +4120,8 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: config + temp: {} --- clone: retries: 3 @@ -3796,7 +4144,7 @@ steps: - name: config path: /root/.docker/ - commands: - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest-ubuntu + - trivy image --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest-ubuntu depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -3804,15 +4152,22 @@ steps: volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - commands: - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest-ubuntu + - trivy image --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest-ubuntu depends_on: - authenticate-gcr + environment: + GOOGLE_APPLICATION_CREDENTIALS: + from_secret: gcr_credentials_json image: aquasec/trivy:0.21.0 name: scan-high-critical-vulnerabilities volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - image: plugins/slack name: slack-notify-failure settings: @@ -3831,6 +4186,8 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: config + temp: {} --- clone: retries: 3 @@ -3853,7 +4210,7 @@ steps: - name: config path: /root/.docker/ - commands: - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main-ubuntu + - trivy image --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main-ubuntu depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -3861,15 +4218,22 @@ steps: volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - commands: - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main-ubuntu + - trivy image --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main-ubuntu depends_on: - authenticate-gcr + environment: + GOOGLE_APPLICATION_CREDENTIALS: + from_secret: gcr_credentials_json image: aquasec/trivy:0.21.0 name: scan-high-critical-vulnerabilities volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - image: plugins/slack name: slack-notify-failure settings: @@ -3888,6 +4252,8 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: config + temp: {} --- clone: retries: 3 @@ -3910,14 +4276,18 @@ steps: - name: config path: /root/.docker/ - commands: + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine/git:2.40.1 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM golang:1.20.10-alpine + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM node:18.12.0-alpine - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM google/cloud-sdk:431.0.0 - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/build-container:1.7.4 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana-ci-deploy:1.3.3 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine:3.18.3 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM ubuntu:22.04 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM byrnedo/alpine-curl:0.1.8 - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM golang:1.20.10 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM plugins/slack + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM python:3.8 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM postgres:12.3-alpine + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/mimir:latest - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:5.7.39 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:8.0.32 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM redis:6.2.11-alpine @@ -3928,7 +4298,8 @@ steps: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docker-puppeteer:1.1.0 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docs-base:dbd975af06 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM jwilder/dockerize:0.6.1 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM koalaman/shellcheck:stable depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -3936,15 +4307,21 @@ steps: volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - commands: + - trivy --exit-code 1 --severity HIGH,CRITICAL alpine/git:2.40.1 + - trivy --exit-code 1 --severity HIGH,CRITICAL golang:1.20.10-alpine + - trivy --exit-code 1 --severity HIGH,CRITICAL node:18.12.0-alpine - trivy --exit-code 1 --severity HIGH,CRITICAL google/cloud-sdk:431.0.0 - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/build-container:1.7.4 - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana-ci-deploy:1.3.3 - trivy --exit-code 1 --severity HIGH,CRITICAL alpine:3.18.3 + - trivy --exit-code 1 --severity HIGH,CRITICAL ubuntu:22.04 - trivy --exit-code 1 --severity HIGH,CRITICAL byrnedo/alpine-curl:0.1.8 - - trivy --exit-code 1 --severity HIGH,CRITICAL golang:1.20.10 - trivy --exit-code 1 --severity HIGH,CRITICAL plugins/slack + - trivy --exit-code 1 --severity HIGH,CRITICAL python:3.8 - trivy --exit-code 1 --severity HIGH,CRITICAL postgres:12.3-alpine + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/mimir:latest - trivy --exit-code 1 --severity HIGH,CRITICAL mysql:5.7.39 - trivy --exit-code 1 --severity HIGH,CRITICAL mysql:8.0.32 - trivy --exit-code 1 --severity HIGH,CRITICAL redis:6.2.11-alpine @@ -3955,14 +4332,20 @@ steps: - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docker-puppeteer:1.1.0 - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docs-base:dbd975af06 - trivy --exit-code 1 --severity HIGH,CRITICAL cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - - trivy --exit-code 1 --severity HIGH,CRITICAL us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + - trivy --exit-code 1 --severity HIGH,CRITICAL jwilder/dockerize:0.6.1 + - trivy --exit-code 1 --severity HIGH,CRITICAL koalaman/shellcheck:stable depends_on: - authenticate-gcr + environment: + GOOGLE_APPLICATION_CREDENTIALS: + from_secret: gcr_credentials_json image: aquasec/trivy:0.21.0 name: scan-high-critical-vulnerabilities volumes: - name: docker path: /var/run/docker.sock + - name: config + path: /root/.docker/ - image: plugins/slack name: slack-notify-failure settings: @@ -3980,41 +4363,8 @@ volumes: - host: path: /var/run/docker.sock name: docker ---- -clone: - retries: 3 -kind: pipeline -name: grafana-com-nightly -platform: - arch: amd64 - os: linux -steps: -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.10 - name: compile-build-cmd -- commands: - - ./bin/build publish grafana-com --edition oss - depends_on: - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - GRAFANA_COM_API_KEY: - from_secret: grafana_api_key - image: grafana/grafana-ci-deploy:1.3.3 - name: post-to-grafana-com -trigger: - cron: grafana-com-nightly - event: cron -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker +- name: config + temp: {} --- get: name: credentials.json @@ -4160,6 +4510,24 @@ get: kind: secret name: destination --- +get: + name: storybook_destination + path: infra/data/ci/grafana-release-eng/rgm +kind: secret +name: rgm_storybook_destination +--- +get: + name: cdn_destination + path: infra/data/ci/grafana-release-eng/rgm +kind: secret +name: rgm_cdn_destination +--- +get: + name: downloads_destination + path: infra/data/ci/grafana-release-eng/rgm +kind: secret +name: rgm_downloads_destination +--- get: name: dagger_token path: infra/data/ci/grafana-release-eng/rgm @@ -4184,24 +4552,6 @@ get: kind: secret name: delivery-bot-app-private-key --- -get: - name: gcp_service_account_base64 - path: infra/data/ci/grafana-release-eng/rgm -kind: secret -name: gcp_key_base64 ---- -get: - name: destination - path: infra/data/ci/grafana-release-eng/rgm -kind: secret -name: destination ---- -get: - name: pat - path: infra/data/ci/github/grafanabot -kind: secret -name: github_token ---- get: name: service-account path: secret/data/common/gcr @@ -4209,6 +4559,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: 72417133186021b1fede12bd120397608d0a7267637da122d01d53ea196c79d9 +hmac: 4c68ef8aedb7202b86f077dbccc04504b81eaa1729276238d90a06672198487e ... diff --git a/Makefile b/Makefile index f062eb6594d..cf30d4b5b37 100644 --- a/Makefile +++ b/Makefile @@ -13,9 +13,12 @@ GO = go GO_FILES ?= ./pkg/... SH_FILES ?= $(shell find ./scripts -name *.sh) GO_BUILD_FLAGS += $(if $(GO_BUILD_DEV),-dev) -GO_BUILD_FLAGS += $(if $(GO_BUILD_DEV),-dev) GO_BUILD_FLAGS += $(if $(GO_BUILD_TAGS),-build-tags=$(GO_BUILD_TAGS)) +targets := $(shell echo '$(sources)' | tr "," " ") + +GO_INTEGRATION_TESTS := $(shell find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u) + all: deps build ##@ Dependencies @@ -33,35 +36,68 @@ node_modules: package.json yarn.lock ## Install node modules. ##@ Swagger SPEC_TARGET = public/api-spec.json -MERGED_SPEC_TARGET := public/api-merged.json +ENTERPRISE_SPEC_TARGET = public/api-enterprise-spec.json +MERGED_SPEC_TARGET = public/api-merged.json NGALERT_SPEC_TARGET = pkg/services/ngalert/api/tooling/api.json $(NGALERT_SPEC_TARGET): +$(MAKE) -C pkg/services/ngalert/api/tooling api.json -$(MERGED_SPEC_TARGET): $(SPEC_TARGET) $(NGALERT_SPEC_TARGET) $(SWAGGER) ## Merge generated and ngalert API specs +$(MERGED_SPEC_TARGET): swagger-oss-gen swagger-enterprise-gen $(NGALERT_SPEC_TARGET) $(SWAGGER) ## Merge generated and ngalert API specs # known conflicts DsPermissionType, AddApiKeyCommand, Json, Duration (identical models referenced by both specs) - $(SWAGGER) mixin $(SPEC_TARGET) $(NGALERT_SPEC_TARGET) --ignore-conflicts -o $(MERGED_SPEC_TARGET) + $(SWAGGER) mixin $(SPEC_TARGET) $(ENTERPRISE_SPEC_TARGET) $(NGALERT_SPEC_TARGET) --ignore-conflicts -o $(MERGED_SPEC_TARGET) -$(SPEC_TARGET): $(SWAGGER) ## Generate API Swagger specification +swagger-oss-gen: $(SWAGGER) ## Generate API Swagger specification + @echo "re-generating swagger for OSS" + rm -f $(SPEC_TARGET) SWAGGER_GENERATE_EXTENSION=false $(SWAGGER) generate spec -m -w pkg/server -o $(SPEC_TARGET) \ -x "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" \ -x "github.com/prometheus/alertmanager" \ -i pkg/api/swagger_tags.json \ - --exclude-tag=alpha + --exclude-tag=alpha \ + --exclude-tag=enterprise -swagger-api-spec: gen-go $(SPEC_TARGET) $(MERGED_SPEC_TARGET) validate-api-spec +# this file only exists if enterprise is enabled +ENTERPRISE_EXT_FILE = pkg/extensions/ext.go +ifeq ("$(wildcard $(ENTERPRISE_EXT_FILE))","") ## if enterprise is not enabled +swagger-enterprise-gen: + @echo "skipping re-generating swagger for enterprise: not enabled" +else +swagger-enterprise-gen: $(SWAGGER) ## Generate API Swagger specification + @echo "re-generating swagger for enterprise" + rm -f $(ENTERPRISE_SPEC_TARGET) + SWAGGER_GENERATE_EXTENSION=false $(SWAGGER) generate spec -m -w pkg/server -o $(ENTERPRISE_SPEC_TARGET) \ + -x "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" \ + -x "github.com/prometheus/alertmanager" \ + -i pkg/api/swagger_tags.json \ + --exclude-tag=alpha \ + --include-tag=enterprise +endif -validate-api-spec: $(MERGED_SPEC_TARGET) $(SWAGGER) ## Validate API spec +swagger-gen: gen-go $(MERGED_SPEC_TARGET) swagger-validate + +swagger-validate: $(MERGED_SPEC_TARGET) $(SWAGGER) ## Validate API spec $(SWAGGER) validate $(<) -clean-api-spec: - rm $(SPEC_TARGET) $(MERGED_SPEC_TARGET) $(OAPI_SPEC_TARGET) +swagger-clean: + rm -f $(SPEC_TARGET) $(MERGED_SPEC_TARGET) $(OAPI_SPEC_TARGET) + +.PHONY: cleanup-old-git-hooks +cleanup-old-git-hooks: + ./scripts/cleanup-husky.sh + +.PHONY: lefthook-install +lefthook-install: cleanup-old-git-hooks $(LEFTHOOK) # install lefthook for pre-commit hooks + $(LEFTHOOK) install -f + +.PHONY: lefthook-uninstall +lefthook-uninstall: $(LEFTHOOK) + $(LEFTHOOK) uninstall ##@ OpenAPI 3 OAPI_SPEC_TARGET = public/openapi3.json -openapi3-gen: swagger-api-spec ## Generates OpenApi 3 specs from the Swagger 2 already generated +openapi3-gen: swagger-gen ## Generates OpenApi 3 specs from the Swagger 2 already generated $(GO) run scripts/openapi3/openapi3conv.go $(MERGED_SPEC_TARGET) $(OAPI_SPEC_TARGET) ##@ Building @@ -72,7 +108,7 @@ gen-cue: ## Do all CUE/Thema code generation go generate ./public/app/plugins/gen.go go generate ./pkg/kindsys/report.go -gen-go: $(WIRE) gen-cue +gen-go: $(WIRE) @echo "generate go files" $(WIRE) gen -tags $(WIRE_TAGS) ./pkg/server ./pkg/cmd/grafana-cli/runner @@ -84,7 +120,7 @@ fix-cue: $(CUE) gen-jsonnet: go generate ./devenv/jsonnet -build-go: $(MERGED_SPEC_TARGET) gen-go ## Build all Go binaries. +build-go: gen-go ## Build all Go binaries. @echo "build go files" $(GO) run build.go $(GO_BUILD_FLAGS) build @@ -126,19 +162,39 @@ test-go-unit: ## Run unit tests for backend with flags. .PHONY: test-go-integration test-go-integration: ## Run integration tests for backend with flags. @echo "test backend integration tests" - $(GO) test -run Integration -covermode=atomic -timeout=30m ./pkg/... + $(GO) test -count=1 -run "^TestIntegration" -covermode=atomic -timeout=5m $(GO_INTEGRATION_TESTS) + +.PHONY: test-go-integration-alertmanager +test-go-integration-alertmanager: ## Run integration tests for the remote alertmanager (config taken from the mimir_backend block). + @echo "test remote alertmanager integration tests" + $(GO) clean -testcache + AM_URL=http://localhost:8080 AM_TENANT_ID=test AM_PASSWORD=test \ + $(GO) test -count=1 -run "^TestIntegrationRemoteAlertmanager" -covermode=atomic -timeout=5m ./pkg/services/ngalert/notifier/... .PHONY: test-go-integration-postgres test-go-integration-postgres: devenv-postgres ## Run integration tests for postgres backend with flags. @echo "test backend integration postgres tests" $(GO) clean -testcache - $(GO) list './pkg/...' | xargs -I {} sh -c 'GRAFANA_TEST_DB=postgres go test -run Integration -covermode=atomic -timeout=2m {}' + GRAFANA_TEST_DB=postgres \ + $(GO) test -p=1 -count=1 -run "^TestIntegration" -covermode=atomic -timeout=10m $(GO_INTEGRATION_TESTS) .PHONY: test-go-integration-mysql test-go-integration-mysql: devenv-mysql ## Run integration tests for mysql backend with flags. @echo "test backend integration mysql tests" + GRAFANA_TEST_DB=mysql \ + $(GO) test -p=1 -count=1 -run "^TestIntegration" -covermode=atomic -timeout=10m $(GO_INTEGRATION_TESTS) + +.PHONY: test-go-integration-redis +test-go-integration-redis: ## Run integration tests for redis cache. + @echo "test backend integration redis tests" $(GO) clean -testcache - $(GO) list './pkg/...' | xargs -I {} sh -c 'GRAFANA_TEST_DB=mysql go test -run Integration -covermode=atomic -timeout=2m {}' + REDIS_URL=localhost:6379 $(GO) test -run IntegrationRedis -covermode=atomic -timeout=2m $(GO_INTEGRATION_TESTS) + +.PHONY: test-go-integration-memcached +test-go-integration-memcached: ## Run integration tests for memcached cache. + @echo "test backend integration memcached tests" + $(GO) clean -testcache + MEMCACHED_HOSTS=localhost:11211 $(GO) test -run IntegrationMemcached -covermode=atomic -timeout=2m $(GO_INTEGRATION_TESTS) test-js: ## Run tests for frontend. @echo "test frontend" @@ -162,19 +218,36 @@ shellcheck: $(SH_FILES) ## Run checks for shell scripts. ##@ Docker +TAG_SUFFIX=$(if $(WIRE_TAGS)!=oss,-$(WIRE_TAGS)) +PLATFORM=linux/amd64 + build-docker-full: ## Build Docker image for development. @echo "build docker container" - DOCKER_BUILDKIT=1 \ - docker build \ - --tag grafana/grafana:dev . + tar -ch . | \ + docker buildx build - \ + --platform $(PLATFORM) \ + --build-arg BINGO=false \ + --build-arg GO_BUILD_TAGS=$(GO_BUILD_TAGS) \ + --build-arg WIRE_TAGS=$(WIRE_TAGS) \ + --build-arg COMMIT_SHA=$$(git rev-parse HEAD) \ + --build-arg BUILD_BRANCH=$$(git rev-parse --abbrev-ref HEAD) \ + --tag grafana/grafana$(TAG_SUFFIX):dev \ + $(DOCKER_BUILD_ARGS) build-docker-full-ubuntu: ## Build Docker image based on Ubuntu for development. @echo "build docker container" - DOCKER_BUILDKIT=1 \ - docker build \ - --build-arg BASE_IMAGE=ubuntu:20.04 \ + tar -ch . | \ + docker buildx build - \ + --platform $(PLATFORM) \ + --build-arg BINGO=false \ + --build-arg GO_BUILD_TAGS=$(GO_BUILD_TAGS) \ + --build-arg WIRE_TAGS=$(WIRE_TAGS) \ + --build-arg COMMIT_SHA=$$(git rev-parse HEAD) \ + --build-arg BUILD_BRANCH=$$(git rev-parse --abbrev-ref HEAD) \ + --build-arg BASE_IMAGE=ubuntu:22.04 \ --build-arg GO_IMAGE=golang:1.20.10 \ - --tag grafana/grafana:dev-ubuntu . + --tag grafana/grafana$(TAG_SUFFIX):dev-ubuntu \ + $(DOCKER_BUILD_ARGS) ##@ Services @@ -185,8 +258,6 @@ devenv: @printf 'You have to define sources for this command \nexample: make devenv sources=postgres,openldap\n' else devenv: devenv-down ## Start optional services, e.g. postgres, prometheus, and elasticsearch. - $(eval targets := $(shell echo '$(sources)' | tr "," " ")) - @cd devenv; \ ./create_docker_compose.sh $(targets) || \ (rm -rf {docker-compose.yaml,conf.tmp,.env}; exit 1) @@ -219,6 +290,9 @@ devenv-mysql: protobuf: ## Compile protobuf definitions bash scripts/protobuf-check.sh bash pkg/plugins/backendplugin/pluginextensionv2/generate.sh + bash pkg/plugins/backendplugin/secretsmanagerplugin/generate.sh + bash pkg/services/store/entity/generate.sh + bash pkg/infra/grn/generate.sh clean: ## Clean up intermediate build artifacts. @echo "cleaning" @@ -244,7 +318,7 @@ scripts/drone/TAGS: $(shell find scripts/drone -name '*.star') etags --lang none --regex="/def \(\w+\)[^:]+:/\1/" --regex="/\s*\(\w+\) =/\1/" $^ -o $@ format-drone: - buildifier -r scripts/drone + buildifier --lint=fix -r scripts/drone help: ## Display this help. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/pkg/build/cmd.go b/pkg/build/cmd.go index 93de968f7b4..fa98316e9c2 100644 --- a/pkg/build/cmd.go +++ b/pkg/build/cmd.go @@ -217,12 +217,32 @@ func ldflags(opts BuildOpts) (string, error) { return "", err } + commitSha := getGitSha() + if v := os.Getenv("COMMIT_SHA"); v != "" { + commitSha = v + } + + var enterpriseCommitSha string + if opts.enterprise { + enterpriseCommitSha = getGitEnterpriseSha() + if v := os.Getenv("ENTERPRISE_COMMIT_SHA"); v != "" { + enterpriseCommitSha = v + } + } + + buildBranch := getGitBranch() + if v := os.Getenv("BUILD_BRANCH"); v != "" { + buildBranch = v + } var b bytes.Buffer b.WriteString("-w") b.WriteString(fmt.Sprintf(" -X main.version=%s", opts.version)) - b.WriteString(fmt.Sprintf(" -X main.commit=%s", getGitSha())) + b.WriteString(fmt.Sprintf(" -X main.commit=%s", commitSha)) + if enterpriseCommitSha != "" { + b.WriteString(fmt.Sprintf(" -X main.enterpriseCommit=%s", enterpriseCommitSha)) + } b.WriteString(fmt.Sprintf(" -X main.buildstamp=%d", buildStamp)) - b.WriteString(fmt.Sprintf(" -X main.buildBranch=%s", getGitBranch())) + b.WriteString(fmt.Sprintf(" -X main.buildBranch=%s", buildBranch)) if v := os.Getenv("LDFLAGS"); v != "" { b.WriteString(fmt.Sprintf(" -extldflags \"%s\"", v)) } diff --git a/pkg/build/cmd/buildbackend.go b/pkg/build/cmd/buildbackend.go index 39f0d707fe4..40c4f90388a 100644 --- a/pkg/build/cmd/buildbackend.go +++ b/pkg/build/cmd/buildbackend.go @@ -4,12 +4,13 @@ import ( "fmt" "log" + "github.com/urfave/cli/v2" + "github.com/grafana/grafana/pkg/build/compilers" "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/errutil" "github.com/grafana/grafana/pkg/build/grafana" "github.com/grafana/grafana/pkg/build/syncutil" - "github.com/urfave/cli/v2" ) func BuildBackend(ctx *cli.Context) error { diff --git a/pkg/build/cmd/builddocker.go b/pkg/build/cmd/builddocker.go index 0af4b33f68f..f623057ee60 100644 --- a/pkg/build/cmd/builddocker.go +++ b/pkg/build/cmd/builddocker.go @@ -3,10 +3,11 @@ package main import ( "log" + "github.com/urfave/cli/v2" + "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/docker" "github.com/grafana/grafana/pkg/build/gcloud" - "github.com/urfave/cli/v2" ) func BuildDocker(c *cli.Context) error { diff --git a/pkg/build/cmd/buildfrontend.go b/pkg/build/cmd/buildfrontend.go index 3a1be84cac8..ef0fbc9a5fd 100644 --- a/pkg/build/cmd/buildfrontend.go +++ b/pkg/build/cmd/buildfrontend.go @@ -3,11 +3,12 @@ package main import ( "log" + "github.com/urfave/cli/v2" + "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/errutil" "github.com/grafana/grafana/pkg/build/frontend" "github.com/grafana/grafana/pkg/build/syncutil" - "github.com/urfave/cli/v2" ) func BuildFrontend(c *cli.Context) error { diff --git a/pkg/build/cmd/buildinternalplugins.go b/pkg/build/cmd/buildinternalplugins.go index fad80b818d5..15a1397158d 100644 --- a/pkg/build/cmd/buildinternalplugins.go +++ b/pkg/build/cmd/buildinternalplugins.go @@ -4,11 +4,12 @@ import ( "context" "log" + "github.com/urfave/cli/v2" + "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/errutil" "github.com/grafana/grafana/pkg/build/plugins" "github.com/grafana/grafana/pkg/build/syncutil" - "github.com/urfave/cli/v2" ) func BuildInternalPlugins(c *cli.Context) error { diff --git a/pkg/build/cmd/e2etests.go b/pkg/build/cmd/e2etests.go index f377b08f874..ec27fc9b28f 100644 --- a/pkg/build/cmd/e2etests.go +++ b/pkg/build/cmd/e2etests.go @@ -6,8 +6,9 @@ import ( "os" "os/exec" - "github.com/grafana/grafana/pkg/build/e2eutil" "github.com/urfave/cli/v2" + + "github.com/grafana/grafana/pkg/build/e2eutil" ) func EndToEndTests(c *cli.Context) error { diff --git a/pkg/build/cmd/enterprisecheck.go b/pkg/build/cmd/enterprisecheck.go index f2c36b0fc2e..3fb13f6c893 100644 --- a/pkg/build/cmd/enterprisecheck.go +++ b/pkg/build/cmd/enterprisecheck.go @@ -6,9 +6,10 @@ import ( "os" "strconv" + "github.com/urfave/cli/v2" + "github.com/grafana/grafana/pkg/build/env" "github.com/grafana/grafana/pkg/build/git" - "github.com/urfave/cli/v2" ) // checkOpts are options used to create a new GitHub check for the enterprise downstream test. diff --git a/pkg/build/cmd/exportversion.go b/pkg/build/cmd/exportversion.go index a3a54309c73..c4c4744ebef 100644 --- a/pkg/build/cmd/exportversion.go +++ b/pkg/build/cmd/exportversion.go @@ -4,8 +4,9 @@ import ( "os" "path/filepath" - "github.com/grafana/grafana/pkg/build/config" "github.com/urfave/cli/v2" + + "github.com/grafana/grafana/pkg/build/config" ) func ExportVersion(c *cli.Context) error { diff --git a/pkg/build/cmd/fetchimages.go b/pkg/build/cmd/fetchimages.go index bd1ec6a5102..b131cc9a73d 100644 --- a/pkg/build/cmd/fetchimages.go +++ b/pkg/build/cmd/fetchimages.go @@ -6,10 +6,11 @@ import ( "os/exec" "strings" + "github.com/urfave/cli/v2" + "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/docker" "github.com/grafana/grafana/pkg/build/gcloud" - "github.com/urfave/cli/v2" ) const ( diff --git a/pkg/build/cmd/grafanacom.go b/pkg/build/cmd/grafanacom.go index bafb3e41b8f..9d44b9298c9 100644 --- a/pkg/build/cmd/grafanacom.go +++ b/pkg/build/cmd/grafanacom.go @@ -9,7 +9,6 @@ import ( "net/http" "net/url" "os" - "path" "path/filepath" "strings" @@ -233,7 +232,7 @@ func getSHA256(u string) ([]byte, error) { return sha256, nil } -func postRequest(cfg packaging.PublishConfig, pth string, obj interface{}, descr string) error { +func postRequest(cfg packaging.PublishConfig, pth string, obj any, descr string) error { var sfx string switch cfg.Edition { case config.EditionOSS: diff --git a/pkg/build/cmd/main.go b/pkg/build/cmd/main.go index 473e87d1fb9..dff9db8f8b0 100644 --- a/pkg/build/cmd/main.go +++ b/pkg/build/cmd/main.go @@ -5,8 +5,9 @@ import ( "os" "strings" - "github.com/grafana/grafana/pkg/build/docker" "github.com/urfave/cli/v2" + + "github.com/grafana/grafana/pkg/build/docker" ) var additionalCommands []*cli.Command = make([]*cli.Command, 0, 5) diff --git a/pkg/build/cmd/npm.go b/pkg/build/cmd/npm.go index 07c2fe1365f..13994fb4e19 100644 --- a/pkg/build/cmd/npm.go +++ b/pkg/build/cmd/npm.go @@ -6,8 +6,9 @@ import ( "os/exec" "strings" - "github.com/grafana/grafana/pkg/build/npm" "github.com/urfave/cli/v2" + + "github.com/grafana/grafana/pkg/build/npm" ) func NpmRetrieveAction(c *cli.Context) error { diff --git a/pkg/build/cmd/package.go b/pkg/build/cmd/package.go index 7f1b2da25bc..6b961512ec1 100644 --- a/pkg/build/cmd/package.go +++ b/pkg/build/cmd/package.go @@ -5,11 +5,12 @@ import ( "log" "strings" + "github.com/urfave/cli/v2" + "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/gpg" "github.com/grafana/grafana/pkg/build/packaging" "github.com/grafana/grafana/pkg/build/syncutil" - "github.com/urfave/cli/v2" ) func Package(c *cli.Context) error { diff --git a/pkg/build/cmd/publishaws.go b/pkg/build/cmd/publishaws.go index e499c6b27fa..5fc98fbbc27 100644 --- a/pkg/build/cmd/publishaws.go +++ b/pkg/build/cmd/publishaws.go @@ -17,8 +17,9 @@ import ( "github.com/aws/aws-sdk-go/service/marketplacecatalog" "github.com/docker/docker/api/types" "github.com/docker/docker/client" - "github.com/grafana/grafana/pkg/build/config" "github.com/urfave/cli/v2" + + "github.com/grafana/grafana/pkg/build/config" ) const ( diff --git a/pkg/build/cmd/publishgithub.go b/pkg/build/cmd/publishgithub.go index e01a199b4d9..610b702d7c7 100644 --- a/pkg/build/cmd/publishgithub.go +++ b/pkg/build/cmd/publishgithub.go @@ -9,9 +9,10 @@ import ( "strings" "github.com/google/go-github/github" - "github.com/grafana/grafana/pkg/build/config" "github.com/urfave/cli/v2" "golang.org/x/oauth2" + + "github.com/grafana/grafana/pkg/build/config" ) type githubRepositoryService interface { diff --git a/pkg/build/cmd/publishimages_enterprise2.go b/pkg/build/cmd/publishimages_enterprise2.go index b76b2688d6f..94cd4b57c7d 100644 --- a/pkg/build/cmd/publishimages_enterprise2.go +++ b/pkg/build/cmd/publishimages_enterprise2.go @@ -6,10 +6,11 @@ import ( "os" "os/exec" + "github.com/urfave/cli/v2" + "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/docker" "github.com/grafana/grafana/pkg/build/gcloud" - "github.com/urfave/cli/v2" ) func Enterprise2(c *cli.Context) error { diff --git a/pkg/build/cmd/storestorybook.go b/pkg/build/cmd/storestorybook.go index 65adfa1fbb7..13970e6a32e 100644 --- a/pkg/build/cmd/storestorybook.go +++ b/pkg/build/cmd/storestorybook.go @@ -4,9 +4,10 @@ import ( "log" "path/filepath" + "github.com/urfave/cli/v2" + "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/gcloud/storage" - "github.com/urfave/cli/v2" ) // StoreStorybook implements the sub-command "store-storybook". diff --git a/pkg/build/cmd/uploadcdn.go b/pkg/build/cmd/uploadcdn.go index 8b8ddb0cd81..e18e1f65e16 100644 --- a/pkg/build/cmd/uploadcdn.go +++ b/pkg/build/cmd/uploadcdn.go @@ -6,9 +6,10 @@ import ( "os" "path/filepath" + "github.com/urfave/cli/v2" + "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/gcloud/storage" - "github.com/urfave/cli/v2" ) // UploadCDN implements the sub-command "upload-cdn". diff --git a/pkg/build/cmd/uploadpackages.go b/pkg/build/cmd/uploadpackages.go index 2c8f94d347f..da0dd6a607a 100644 --- a/pkg/build/cmd/uploadpackages.go +++ b/pkg/build/cmd/uploadpackages.go @@ -9,11 +9,12 @@ import ( "path/filepath" "strings" + "github.com/urfave/cli/v2" + "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/droneutil" "github.com/grafana/grafana/pkg/build/gcloud" "github.com/grafana/grafana/pkg/build/packaging" - "github.com/urfave/cli/v2" ) const releaseFolder = "release" diff --git a/pkg/build/cmd/uploadpackages_test.go b/pkg/build/cmd/uploadpackages_test.go index d8147187cbf..9d3bc7b52e1 100644 --- a/pkg/build/cmd/uploadpackages_test.go +++ b/pkg/build/cmd/uploadpackages_test.go @@ -5,9 +5,10 @@ import ( "fmt" "testing" - "github.com/grafana/grafana/pkg/build/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/build/config" ) func Test_getVersionFolder(t *testing.T) { diff --git a/pkg/build/cmd/verifydrone.go b/pkg/build/cmd/verifydrone.go index 19da6fb5a35..14596377110 100644 --- a/pkg/build/cmd/verifydrone.go +++ b/pkg/build/cmd/verifydrone.go @@ -10,9 +10,7 @@ import ( "github.com/drone/drone-cli/drone/lint" "github.com/drone/drone-cli/drone/starlark" - "github.com/google/go-cmp/cmp" - cliv1 "github.com/urfave/cli" "github.com/urfave/cli/v2" "gopkg.in/yaml.v3" @@ -70,7 +68,7 @@ func VerifyDrone(c *cli.Context) error { return nil } -func readConfig(fpath string) ([]map[string]interface{}, error) { +func readConfig(fpath string) ([]map[string]any, error) { //nolint:gosec f, err := os.Open(fpath) if err != nil { @@ -84,9 +82,9 @@ func readConfig(fpath string) ([]map[string]interface{}, error) { // The YAML stream may contain multiple pipeline configurations, read them all dec := yaml.NewDecoder(f) - var c []map[string]interface{} + var c []map[string]any for { - var m map[string]interface{} + var m map[string]any if err := dec.Decode(&m); err != nil { if errors.Is(err, io.EOF) { break diff --git a/pkg/build/config/genmetadata.go b/pkg/build/config/genmetadata.go index c68496d32c6..1ff0565bc17 100644 --- a/pkg/build/config/genmetadata.go +++ b/pkg/build/config/genmetadata.go @@ -5,8 +5,9 @@ import ( "os" "strings" - "github.com/grafana/grafana/pkg/build/droneutil" "github.com/urfave/cli/v2" + + "github.com/grafana/grafana/pkg/build/droneutil" ) func GenerateMetadata(c *cli.Context) (Metadata, error) { diff --git a/pkg/build/config/genmetadata_test.go b/pkg/build/config/genmetadata_test.go index 8810d4409c4..d8eeb06bb6a 100644 --- a/pkg/build/config/genmetadata_test.go +++ b/pkg/build/config/genmetadata_test.go @@ -72,10 +72,8 @@ func setUpEnv(t *testing.T, envMap map[string]string) { t.Helper() os.Clearenv() - err := os.Setenv("DRONE_COMMIT", "abcd12345") - require.NoError(t, err) + t.Setenv("DRONE_COMMIT", "abcd12345") for k, v := range envMap { - err := os.Setenv(k, v) - require.NoError(t, err) + t.Setenv(k, v) } } diff --git a/pkg/build/config/revision.go b/pkg/build/config/revision.go index aa104916652..70be21cdf18 100644 --- a/pkg/build/config/revision.go +++ b/pkg/build/config/revision.go @@ -3,6 +3,7 @@ package config import ( "context" "fmt" + "log" "strconv" "time" @@ -10,9 +11,10 @@ import ( ) type Revision struct { - Timestamp int64 - SHA256 string - Branch string + Timestamp int64 + SHA256 string + EnterpriseCommit string + Branch string } func GrafanaTimestamp(ctx context.Context, dir string) (int64, error) { @@ -42,14 +44,26 @@ func GrafanaRevision(ctx context.Context, grafanaDir string) (Revision, error) { return Revision{}, err } + enterpriseCommit, err := executil.OutputAt(ctx, grafanaDir, "git", "-C", "../grafana-enterprise", "rev-parse", "--short", "HEAD") + if err != nil { + enterpriseCommit, err = executil.OutputAt(ctx, grafanaDir, "git", "-C", "..", "rev-parse", "--short", "HEAD") + if err != nil { + enterpriseCommit, err = executil.OutputAt(ctx, grafanaDir, "git", "-C", "/tmp/grafana-enterprise", "rev-parse", "--short", "HEAD") + if err != nil { + log.Println("Could not get enterprise commit. Error:", err) + } + } + } + branch, err := executil.OutputAt(ctx, grafanaDir, "git", "rev-parse", "--abbrev-ref", "HEAD") if err != nil { return Revision{}, err } return Revision{ - SHA256: sha, - Branch: branch, - Timestamp: stamp, + SHA256: sha, + EnterpriseCommit: enterpriseCommit, + Branch: branch, + Timestamp: stamp, }, nil } diff --git a/pkg/build/config/version.go b/pkg/build/config/version.go index 5eb96ab63f3..05b99cf4678 100644 --- a/pkg/build/config/version.go +++ b/pkg/build/config/version.go @@ -94,7 +94,7 @@ func GetPackageJSONVersion(grafanaDir string) (string, error) { if err != nil { return "", fmt.Errorf("failed to read %q: %w", pkgJSONPath, err) } - pkgObj := map[string]interface{}{} + pkgObj := map[string]any{} if err := json.Unmarshal(pkgJSONB, &pkgObj); err != nil { return "", fmt.Errorf("failed decoding %q: %w", pkgJSONPath, err) } diff --git a/pkg/build/config/versions.go b/pkg/build/config/versions.go index aa618e90358..8decc72593c 100644 --- a/pkg/build/config/versions.go +++ b/pkg/build/config/versions.go @@ -1,7 +1,5 @@ package config -const PublicBucket = "grafana-downloads" - var Versions = VersionMap{ PullRequestMode: { Variants: []Variant{ @@ -9,8 +7,9 @@ var Versions = VersionMap{ VariantLinuxAmd64Musl, VariantDarwinAmd64, VariantWindowsAmd64, - VariantArm64, - VariantArm64Musl, + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // VariantArm64, + // VariantArm64Musl, }, PluginSignature: PluginSignature{ Sign: false, @@ -29,9 +28,10 @@ var Versions = VersionMap{ }, MainMode: { Variants: []Variant{ - VariantArmV6, - VariantArmV7, - VariantArmV7Musl, + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // VariantArmV6, + // VariantArmV7, + // VariantArmV7Musl, VariantArm64, VariantArm64Musl, VariantDarwinAmd64, @@ -48,7 +48,8 @@ var Versions = VersionMap{ Architectures: []Architecture{ ArchAMD64, ArchARM64, - ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable. + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable. }, Distribution: []Distribution{ Alpine, @@ -64,9 +65,10 @@ var Versions = VersionMap{ }, DownstreamMode: { Variants: []Variant{ - VariantArmV6, - VariantArmV7, - VariantArmV7Musl, + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // VariantArmV6, + //VariantArmV7, + // VariantArmV7Musl, VariantArm64, VariantArm64Musl, VariantDarwinAmd64, @@ -83,7 +85,8 @@ var Versions = VersionMap{ Architectures: []Architecture{ ArchAMD64, ArchARM64, - ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable. + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable. }, Distribution: []Distribution{ Alpine, @@ -171,9 +174,10 @@ var Versions = VersionMap{ }, Enterprise2Mode: { Variants: []Variant{ - VariantArmV6, - VariantArmV7, - VariantArmV7Musl, + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // VariantArmV6, + // VariantArmV7, + // VariantArmV7Musl, VariantArm64, VariantArm64Musl, VariantDarwinAmd64, @@ -190,7 +194,8 @@ var Versions = VersionMap{ Architectures: []Architecture{ ArchAMD64, ArchARM64, - ArchARMv7, + // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved + // ArchARMv7, }, Distribution: []Distribution{ Alpine, diff --git a/pkg/build/docker/build.go b/pkg/build/docker/build.go index 64016257b2a..29b2263e62b 100644 --- a/pkg/build/docker/build.go +++ b/pkg/build/docker/build.go @@ -74,7 +74,7 @@ func BuildImage(version string, arch config.Architecture, grafanaDir string, use tagSuffix := "" if useUbuntu { libc = "" - baseImage = fmt.Sprintf("%subuntu:20.04", baseArch) + baseImage = fmt.Sprintf("%subuntu:22.04", baseArch) tagSuffix = "-ubuntu" } diff --git a/pkg/build/droneutil/event_test.go b/pkg/build/droneutil/event_test.go index f0efcfa284e..b69ce50fc54 100644 --- a/pkg/build/droneutil/event_test.go +++ b/pkg/build/droneutil/event_test.go @@ -3,8 +3,9 @@ package droneutil_test import ( "testing" - "github.com/grafana/grafana/pkg/build/droneutil" "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/build/droneutil" ) func TestGetDroneEvent(t *testing.T) { diff --git a/pkg/build/env/fallback_test.go b/pkg/build/env/fallback_test.go index d14e13ee774..8871bb68a72 100644 --- a/pkg/build/env/fallback_test.go +++ b/pkg/build/env/fallback_test.go @@ -15,6 +15,11 @@ const ( flag2 = "flag2" ) +type flagObj struct { + name string + value string +} + func TestRequireListWithEnvFallback(t *testing.T) { var app = cli.NewApp() tests := []struct { @@ -73,7 +78,7 @@ func TestRequireStringWithEnvFallback(t *testing.T) { }{ { testName: "string present in the context", - ctx: cli.NewContext(app, setFlags(t, flag1, flag2, flag.NewFlagSet("test", flag.ContinueOnError)), nil), + ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("test", flag.ContinueOnError), flagObj{name: flag1, value: "a"}), nil), name: flag1, envName: "", expected: "a", @@ -81,7 +86,7 @@ func TestRequireStringWithEnvFallback(t *testing.T) { }, { testName: "string present in env", - ctx: cli.NewContext(app, setFlags(t, "", "", flag.NewFlagSet("test", flag.ContinueOnError)), nil), + ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("test", flag.ContinueOnError)), nil), name: flag1, envName: setEnv(t, flag1, "a"), expected: "a", @@ -89,7 +94,7 @@ func TestRequireStringWithEnvFallback(t *testing.T) { }, { testName: "string absent from both context and env", - ctx: cli.NewContext(app, setFlags(t, "", flag2, flag.NewFlagSet("test", flag.ContinueOnError)), nil), + ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("test", flag.ContinueOnError), flagObj{name: flag2, value: "b"}), nil), name: flag1, envName: "", expected: "", @@ -120,13 +125,12 @@ func applyFlagSet(t *testing.T, aFlag, aValue string) *flag.FlagSet { return set } -func setFlags(t *testing.T, flag1, flag2 string, flagSet *flag.FlagSet) *flag.FlagSet { +func setFlags(t *testing.T, flagSet *flag.FlagSet, flags ...flagObj) *flag.FlagSet { t.Helper() - if flag1 != "" { - flagSet.StringVar(&flag1, "flag1", "a", "") - } - if flag2 != "" { - flagSet.StringVar(&flag2, "flag2", "b", "") + for _, f := range flags { + if f.name != "" { + flagSet.StringVar(&f.name, f.name, f.value, "") + } } return flagSet } @@ -135,9 +139,6 @@ func setEnv(t *testing.T, key, value string) string { t.Helper() os.Clearenv() - err := os.Setenv(key, value) - if err != nil { - require.NoError(t, err) - } + t.Setenv(key, value) return key } diff --git a/pkg/build/env/lookup_test.go b/pkg/build/env/lookup_test.go index cebfb4fac29..ccd44b11e8c 100644 --- a/pkg/build/env/lookup_test.go +++ b/pkg/build/env/lookup_test.go @@ -3,9 +3,9 @@ package env_test import ( "testing" - "github.com/grafana/grafana/pkg/build/env" - "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/build/env" ) func TestLookup(t *testing.T) { diff --git a/pkg/build/frontend/config_test.go b/pkg/build/frontend/config_test.go index f10892e1cef..21be441cc7a 100644 --- a/pkg/build/frontend/config_test.go +++ b/pkg/build/frontend/config_test.go @@ -21,6 +21,11 @@ type packageJson struct { Version string `json:"version"` } +type flagObj struct { + name string + value string +} + var app = cli.NewApp() func TestGetConfig(t *testing.T) { @@ -32,35 +37,35 @@ func TestGetConfig(t *testing.T) { wantErr bool }{ { - ctx: cli.NewContext(app, setFlags(t, jobs, githubToken, "", flag.NewFlagSet("flagSet", flag.ContinueOnError)), nil), + ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil), name: "package.json matches tag", packageJsonVersion: "10.0.0", metadata: config.Metadata{GrafanaVersion: "10.0.0", ReleaseMode: config.ReleaseMode{Mode: config.TagMode}}, wantErr: false, }, { - ctx: cli.NewContext(app, setFlags(t, jobs, githubToken, "", flag.NewFlagSet("flagSet", flag.ContinueOnError)), nil), + ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil), name: "custom tag, package.json doesn't match", packageJsonVersion: "10.0.0", metadata: config.Metadata{GrafanaVersion: "10.0.0-abcd123pre", ReleaseMode: config.ReleaseMode{Mode: config.TagMode}}, wantErr: false, }, { - ctx: cli.NewContext(app, setFlags(t, jobs, githubToken, "", flag.NewFlagSet("flagSet", flag.ContinueOnError)), nil), + ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil), name: "package.json doesn't match tag", packageJsonVersion: "10.1.0", metadata: config.Metadata{GrafanaVersion: "10.0.0", ReleaseMode: config.ReleaseMode{Mode: config.TagMode}}, wantErr: true, }, { - ctx: cli.NewContext(app, setFlags(t, jobs, githubToken, "", flag.NewFlagSet("flagSet", flag.ContinueOnError)), nil), + ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil), name: "test tag event, check should be skipped", packageJsonVersion: "10.1.0", metadata: config.Metadata{GrafanaVersion: "10.1.0-test", ReleaseMode: config.ReleaseMode{Mode: config.TagMode, IsTest: true}}, wantErr: false, }, { - ctx: cli.NewContext(app, setFlags(t, jobs, githubToken, buildID, flag.NewFlagSet("flagSet", flag.ContinueOnError)), nil), + ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}, flagObj{name: buildID, value: "12345"}), nil), name: "non-tag event", packageJsonVersion: "10.1.0-pre", metadata: config.Metadata{GrafanaVersion: "10.1.0-12345pre", ReleaseMode: config.ReleaseMode{Mode: config.PullRequestMode}}, @@ -85,16 +90,12 @@ func TestGetConfig(t *testing.T) { } } -func setFlags(t *testing.T, flag1, flag2, flag3 string, flagSet *flag.FlagSet) *flag.FlagSet { +func setFlags(t *testing.T, flagSet *flag.FlagSet, flags ...flagObj) *flag.FlagSet { t.Helper() - if flag1 != "" { - flagSet.StringVar(&flag1, jobs, "2", "") - } - if flag2 != "" { - flagSet.StringVar(&flag2, githubToken, "token", "") - } - if flag3 != "" { - flagSet.StringVar(&flag3, buildID, "12345", "") + for _, f := range flags { + if f.name != "" { + flagSet.StringVar(&f.name, f.name, f.value, "") + } } return flagSet } diff --git a/pkg/build/fsutil/copy_test.go b/pkg/build/fsutil/copy_test.go index 6c243ee69f7..95cb8e838ae 100644 --- a/pkg/build/fsutil/copy_test.go +++ b/pkg/build/fsutil/copy_test.go @@ -5,9 +5,10 @@ import ( "runtime" "testing" - "github.com/grafana/grafana/pkg/build/fsutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/build/fsutil" ) func TestCopyFile(t *testing.T) { diff --git a/pkg/build/fsutil/exists_test.go b/pkg/build/fsutil/exists_test.go index 1428654b9b8..f46297a0764 100644 --- a/pkg/build/fsutil/exists_test.go +++ b/pkg/build/fsutil/exists_test.go @@ -3,8 +3,9 @@ package fsutil_test import ( "testing" - "github.com/grafana/grafana/pkg/build/fsutil" "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/build/fsutil" ) func TestExists_NonExistent(t *testing.T) { diff --git a/pkg/build/gcloud/storage/gsutil.go b/pkg/build/gcloud/storage/gsutil.go index b963f9bd7e0..711728f4fb0 100644 --- a/pkg/build/gcloud/storage/gsutil.go +++ b/pkg/build/gcloud/storage/gsutil.go @@ -16,10 +16,11 @@ import ( "time" "cloud.google.com/go/storage" - "github.com/grafana/grafana/pkg/build/fsutil" - "github.com/grafana/grafana/pkg/build/gcloud" "google.golang.org/api/iterator" "google.golang.org/api/option" + + "github.com/grafana/grafana/pkg/build/fsutil" + "github.com/grafana/grafana/pkg/build/gcloud" ) var ( @@ -388,11 +389,17 @@ func GetLatestMainBuild(ctx context.Context, bucket *storage.BucketHandle, path return "", ErrorNilBucket } - it := bucket.Objects(ctx, &storage.Query{ + query := &storage.Query{ Prefix: path, - }) + } + err := query.SetAttrSelection([]string{"Name", "Generation"}) + if err != nil { + return "", fmt.Errorf("failed to set attribute selector, err: %q", err) + } + it := bucket.Objects(ctx, query) var files []string + var oldGeneration int64 for { attrs, err := it.Next() if errors.Is(err, iterator.Done) { @@ -401,13 +408,17 @@ func GetLatestMainBuild(ctx context.Context, bucket *storage.BucketHandle, path if err != nil { return "", fmt.Errorf("failed to iterate through bucket, err: %w", err) } - - files = append(files, attrs.Name) + if attrs.Generation >= oldGeneration { + files = append([]string{attrs.Name}, files...) + oldGeneration = attrs.Generation + } else { + files = append(files, attrs.Name) + } } var latestVersion string - for i := len(files) - 1; i >= 0; i-- { - captureVersion := regexp.MustCompile(`(\d+\.\d+\.\d+-\d+pre)`) + for i := 0; i < len(files); i++ { + captureVersion := regexp.MustCompile(`(\d+\.\d+\.\d+-\d+)`) if captureVersion.MatchString(files[i]) { latestVersion = captureVersion.FindString(files[i]) break diff --git a/pkg/build/git.go b/pkg/build/git.go index 6c86d74e004..f955d5b07c1 100644 --- a/pkg/build/git.go +++ b/pkg/build/git.go @@ -15,3 +15,16 @@ func getGitSha() string { } return string(v) } + +func getGitEnterpriseSha() string { + // supporting the old way of dev setup + v, err := runError("git", "-C", "../grafana-enterprise", "rev-parse", "--short", "HEAD") + if err != nil { + // supporting the new way of dev setup + v, err = runError("git", "-C", "..", "rev-parse", "--short", "HEAD") + if err != nil { + return "" + } + } + return string(v) +} diff --git a/pkg/build/git/git.go b/pkg/build/git/git.go index 7539a815de6..9fbcc346e92 100644 --- a/pkg/build/git/git.go +++ b/pkg/build/git/git.go @@ -8,8 +8,9 @@ import ( "regexp" "github.com/google/go-github/v45/github" - "github.com/grafana/grafana/pkg/build/stringutil" "golang.org/x/oauth2" + + "github.com/grafana/grafana/pkg/build/stringutil" ) const ( diff --git a/pkg/build/git/git_checks_test.go b/pkg/build/git/git_checks_test.go index ed3d34a8d28..c9bf0c98d0b 100644 --- a/pkg/build/git/git_checks_test.go +++ b/pkg/build/git/git_checks_test.go @@ -6,8 +6,9 @@ import ( "testing" "github.com/google/go-github/v45/github" - "github.com/grafana/grafana/pkg/build/git" "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/build/git" ) type TestChecksService struct { diff --git a/pkg/build/git/git_issues_test.go b/pkg/build/git/git_issues_test.go index 4eaa3bb6169..dab9fddde77 100644 --- a/pkg/build/git/git_issues_test.go +++ b/pkg/build/git/git_issues_test.go @@ -6,8 +6,9 @@ import ( "testing" "github.com/google/go-github/v45/github" - "github.com/grafana/grafana/pkg/build/git" "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/build/git" ) type TestLabelsService struct { diff --git a/pkg/build/git/git_test.go b/pkg/build/git/git_test.go index c0cbd4c8cd8..3b47ce08dd0 100644 --- a/pkg/build/git/git_test.go +++ b/pkg/build/git/git_test.go @@ -3,8 +3,9 @@ package git_test import ( "testing" - "github.com/grafana/grafana/pkg/build/git" "github.com/stretchr/testify/assert" + + "github.com/grafana/grafana/pkg/build/git" ) func TestPRCheckRegexp(t *testing.T) { diff --git a/pkg/build/grafana/build.go b/pkg/build/grafana/build.go index 4d661e8d851..586b2d22839 100644 --- a/pkg/build/grafana/build.go +++ b/pkg/build/grafana/build.go @@ -23,13 +23,19 @@ const ( ) func GrafanaLDFlags(version string, r config.Revision) []string { - return []string{ + cmd := []string{ "-w", fmt.Sprintf("-X main.version=%s", version), fmt.Sprintf("-X main.commit=%s", r.SHA256), fmt.Sprintf("-X main.buildstamp=%d", r.Timestamp), fmt.Sprintf("-X main.buildBranch=%s", r.Branch), } + + if r.EnterpriseCommit != "" { + cmd = append(cmd, fmt.Sprintf("-X main.enterpriseCommit=%s", r.EnterpriseCommit)) + } + + return cmd } // BinaryFolder returns the path to where the Grafana binary is build given the provided arguments. @@ -99,7 +105,7 @@ func BuildGrafanaBinary(ctx context.Context, name, version string, args BuildArg descriptor := GrafanaDescriptor(opts) - log.Printf("Building %q for %s\nwith env: %v", binary, descriptor, opts.Env()) + log.Printf("Building %q for %s", binary, descriptor) opts.LdFlags = append(args.LdFlags, GrafanaLDFlags(version, revision)...) diff --git a/pkg/build/lerna/lerna.go b/pkg/build/lerna/lerna.go index de04f5f0e88..3bd67a87b44 100644 --- a/pkg/build/lerna/lerna.go +++ b/pkg/build/lerna/lerna.go @@ -47,7 +47,7 @@ func GetLernaVersion(grafanaDir string) (string, error) { if err != nil { return "", fmt.Errorf("failed to read %q: %w", lernaJSONPath, err) } - pkgObj := map[string]interface{}{} + pkgObj := map[string]any{} if err := json.Unmarshal(lernaJSONB, &pkgObj); err != nil { return "", fmt.Errorf("failed decoding %q: %w", lernaJSONPath, err) } diff --git a/pkg/build/metrics/publish.go b/pkg/build/metrics/publish.go index 934f8168802..4e686eb6dea 100644 --- a/pkg/build/metrics/publish.go +++ b/pkg/build/metrics/publish.go @@ -5,10 +5,9 @@ import ( "encoding/json" "fmt" "log" + "net/http" "strconv" "time" - - "net/http" ) type payload struct { diff --git a/pkg/build/npm/npm.go b/pkg/build/npm/npm.go index c2b93aef46b..5b1a895ec49 100644 --- a/pkg/build/npm/npm.go +++ b/pkg/build/npm/npm.go @@ -21,11 +21,11 @@ const NpmArtifactDir = "./npm-artifacts" var packages = []string{ "@grafana/ui", "@grafana/data", - "@grafana/toolkit", "@grafana/runtime", "@grafana/e2e", "@grafana/e2e-selectors", "@grafana/schema", + "@grafana/flamegraph", } // PublishNpmPackages will publish local NPM packages to NPM registry. @@ -117,11 +117,11 @@ func FetchNpmPackages(ctx context.Context, tag, bucketName string) error { // Latest and next is 9.1.6. // 9.2.0-beta1 is released, the latest should stay on 9.1.6, next should point to 9.2.0-beta1 // No move of dist-tags -// 9.1.7 is relased, the latest should point to 9.1.7, next should stay to 9.2.0-beta1 +// 9.1.7 is released, the latest should point to 9.1.7, next should stay to 9.2.0-beta1 // No move of dist-tags // Next week 9.2.0-beta2 is released, the latest should point to 9.1.7, next should point to 9.2.0-beta2 // No move of dist-tags -// In two weeks 9.2.0 stable is relased, the latest and next should point to 9.2.0. +// In two weeks 9.2.0 stable is released, the latest and next should point to 9.2.0. // The next dist-tag is moved to point to 9.2.0. // // 3. Releasing an older stable than the current stable diff --git a/pkg/build/packaging/grafana.go b/pkg/build/packaging/grafana.go index 4d435ac30bf..58f4dbf7209 100644 --- a/pkg/build/packaging/grafana.go +++ b/pkg/build/packaging/grafana.go @@ -381,6 +381,9 @@ func executeFPM(options linuxPackageOptions, packageRoot, srcDir string) error { "--vendor", vendor, "-a", string(options.packageArch), } + if options.prermSrc != "" { + args = append(args, "--before-remove", options.prermSrc) + } if options.edition == config.EditionEnterprise || options.edition == config.EditionEnterprise2 || options.goArch == config.ArchARMv6 { args = append(args, "--conflicts", "grafana") } @@ -530,7 +533,7 @@ func copyPlugins(ctx context.Context, v config.Variant, grafanaDir, tmpDir strin if err != nil { return fmt.Errorf("failed to read %q: %w", filepath.Join(srcDir, "plugin.json"), err) } - var plugJSON map[string]interface{} + var plugJSON map[string]any if err := json.Unmarshal(jsonB, &plugJSON); err != nil { return err } @@ -729,6 +732,7 @@ func realPackageVariant(ctx context.Context, v config.Variant, edition config.Ed initdScriptFilePath: "/etc/init.d/grafana-server", systemdServiceFilePath: "/usr/lib/systemd/system/grafana-server.service", postinstSrc: filepath.Join(grafanaDir, "packaging", "deb", "control", "postinst"), + prermSrc: filepath.Join(grafanaDir, "packaging", "deb", "control", "prerm"), initdScriptSrc: filepath.Join(grafanaDir, "packaging", "deb", "init.d", "grafana-server"), defaultFileSrc: filepath.Join(grafanaDir, "packaging", "deb", "default", "grafana-server"), systemdFileSrc: filepath.Join(grafanaDir, "packaging", "deb", "systemd", "grafana-server.service"), @@ -767,8 +771,7 @@ func realPackageVariant(ctx context.Context, v config.Variant, edition config.Ed defaultFileSrc: filepath.Join(grafanaDir, "packaging", "rpm", "sysconfig", "grafana-server"), systemdFileSrc: filepath.Join(grafanaDir, "packaging", "rpm", "systemd", "grafana-server.service"), wrapperFilePath: filepath.Join(grafanaDir, "packaging", "wrappers"), - // chkconfig is depended on since our systemd service wraps a SysV init script, and that requires chkconfig - depends: []string{"/sbin/service", "chkconfig", "fontconfig", "freetype"}, + depends: []string{"/sbin/service", "fontconfig", "freetype"}, }); err != nil { return err } @@ -845,6 +848,7 @@ type linuxPackageOptions struct { initdScriptFilePath string systemdServiceFilePath string postinstSrc string + prermSrc string initdScriptSrc string defaultFileSrc string systemdFileSrc string diff --git a/pkg/build/packaging/grafana_test.go b/pkg/build/packaging/grafana_test.go index 8f143893085..7f45a632718 100644 --- a/pkg/build/packaging/grafana_test.go +++ b/pkg/build/packaging/grafana_test.go @@ -3,9 +3,10 @@ package packaging_test import ( "testing" + "github.com/stretchr/testify/assert" + "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/packaging" - "github.com/stretchr/testify/assert" ) func TestPackageRegexp(t *testing.T) { diff --git a/pkg/build/version.go b/pkg/build/version.go index d01e16ea368..bf66e13a865 100644 --- a/pkg/build/version.go +++ b/pkg/build/version.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "os" + "path/filepath" "strings" "time" ) @@ -14,7 +15,7 @@ type PackageJSON struct { // Opens the package.json file in the provided directory and returns a struct that represents its contents func OpenPackageJSON(dir string) (PackageJSON, error) { - reader, err := os.Open("package.json") + reader, err := os.Open(filepath.Clean(dir + "/package.json")) if err != nil { return PackageJSON{}, err } diff --git a/scripts/drone/TAGS b/scripts/drone/TAGS deleted file mode 100644 index 1c9f9da89a5..00000000000 --- a/scripts/drone/TAGS +++ /dev/null @@ -1,628 +0,0 @@ - -events/release.star,6652 -ver_mode =ver_mode64,1602 -release_trigger =release_trigger65,1623 -def store_npm_packages_step():store_npm_packages_step74,1752 -def retrieve_npm_packages_step():retrieve_npm_packages_step90,2193 -def release_npm_packages_step():release_npm_packages_step107,2663 -def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):oss_pipelines123,3076 - environment =environment135,3492 - edition =edition136,3529 - services =services137,3549 - volumes =volumes138,3609 - package_steps =package_steps139,3659 - publish_steps =publish_steps140,3682 - should_publish =should_publish141,3705 - should_upload =should_upload142,3748 - init_steps =init_steps143,3818 - build_steps =build_steps152,4033 - integration_test_steps =integration_test_steps159,4342 - build_storybook =build_storybook182,5254 - publish_step =publish_step190,5674 - store_npm_step =store_npm_step191,5758 - windows_package_steps =windows_package_steps196,5957 - windows_pipeline =windows_pipeline198,6044 - name =name199,6077 - edition =edition200,6127 - trigger =trigger201,6154 - steps =steps202,6181 - platform =platform203,6256 - depends_on =depends_on204,6286 - environment =environment207,6393 - pipelines =pipelines209,6434 - name =name211,6470 - edition =edition212,6550 - trigger =trigger213,6581 - services =services214,6612 - steps =steps215,6639 - environment =environment216,6717 - volumes =volumes217,6756 - name =name225,6970 - edition =edition226,7038 - trigger =trigger227,7073 - services =services228,7108 - steps =steps229,7145 - environment =environment230,7329 - volumes =volumes231,7372 - deps =deps234,7433 -def enterprise_pipelines(ver_mode = ver_mode, trigger = release_trigger):enterprise_pipelines247,7856 - environment =environment259,8284 - edition =edition260,8328 - services =services261,8355 - volumes =volumes262,8415 - package_steps =package_steps263,8465 - publish_steps =publish_steps264,8488 - should_publish =should_publish265,8511 - should_upload =should_upload266,8554 - include_enterprise =include_enterprise267,8624 - edition2 =edition2268,8673 - init_steps =init_steps269,8702 - build_steps =build_steps277,8909 - integration_test_steps =integration_test_steps284,9218 - build_storybook =build_storybook312,10299 - publish_step =publish_step324,10892 - store_npm_step =store_npm_step325,10976 - windows_package_steps =windows_package_steps330,11175 - step =step333,11284 - deps_on_clone_enterprise_step =deps_on_clone_enterprise_step337,11418 - windows_pipeline =windows_pipeline347,11746 - name =name348,11779 - edition =edition349,11836 - trigger =trigger350,11863 - steps =steps351,11890 - platform =platform352,11965 - depends_on =depends_on353,11995 - environment =environment356,12109 - pipelines =pipelines358,12150 - name =name360,12186 - edition =edition361,12273 - trigger =trigger362,12304 - services =services363,12335 - steps =steps364,12362 - environment =environment365,12440 - volumes =volumes366,12479 - name =name374,12711 - edition =edition375,12786 - trigger =trigger376,12821 - services =services377,12856 - steps =steps378,12893 - environment =environment379,13213 - volumes =volumes380,13256 - deps =deps383,13317 -def enterprise2_pipelines(prefix = "", ver_mode = ver_mode, trigger = release_trigger):enterprise2_pipelines397,13769 - environment =environment412,14364 - edition =edition415,14424 - volumes =volumes416,14451 - package_steps =package_steps417,14501 - publish_steps =publish_steps418,14524 - should_publish =should_publish419,14547 - should_upload =should_upload420,14590 - include_enterprise =include_enterprise421,14660 - edition2 =edition2422,14709 - init_steps =init_steps423,14738 - build_steps =build_steps431,14945 - fetch_images =fetch_images442,15355 - upload_cdn =upload_cdn444,15497 - step =step458,16187 - deps_on_clone_enterprise_step =deps_on_clone_enterprise_step462,16321 - pipelines =pipelines472,16608 - name =name474,16644 - edition =edition475,16742 - trigger =trigger476,16773 - services =services477,16804 - steps =steps478,16831 - volumes =volumes479,16909 - environment =environment480,16940 -def publish_artifacts_step(mode):publish_artifacts_step486,17019 - security =security487,17053 - security =security489,17098 -def publish_artifacts_pipelines(mode):publish_artifacts_pipelines501,17538 - trigger =trigger502,17577 - steps =steps506,17655 - name =name512,17768 - trigger =trigger513,17820 - steps =steps514,17847 - edition =edition515,17870 - environment =environment516,17895 -def publish_packages_pipeline():publish_packages_pipeline519,17945 - trigger =trigger526,18162 - oss_steps =oss_steps530,18244 - enterprise_steps =enterprise_steps538,18560 - deps =deps545,18903 - name =name552,19062 - trigger =trigger553,19101 - steps =steps554,19128 - edition =edition555,19155 - depends_on =depends_on556,19180 - environment =environment557,19207 - name =name559,19266 - trigger =trigger560,19312 - steps =steps561,19339 - edition =edition562,19373 - depends_on =depends_on563,19398 - environment =environment564,19425 -def publish_npm_pipelines(mode):publish_npm_pipelines567,19482 - trigger =trigger568,19515 - steps =steps572,19593 - name =name580,19772 - trigger =trigger581,19827 - steps =steps582,19854 - edition =edition583,19877 - environment =environment584,19902 -def artifacts_page_pipeline():artifacts_page_pipeline587,19952 - trigger =trigger588,19983 - name =name593,20087 - trigger =trigger594,20128 - steps =steps595,20155 - edition =edition596,20220 - environment =environment597,20245 -def get_e2e_suffix():get_e2e_suffix600,20295 - -events/cron.star,1016 -aquasec_trivy_image =aquasec_trivy_image8,209 -def cronjobs(edition):cronjobs10,255 - grafana_com_nightly_pipeline =grafana_com_nightly_pipeline11,278 - cronName =cronName12,332 - name =name13,374 - steps =steps14,412 -def cron_job_pipeline(cronName, name, steps):cron_job_pipeline24,773 -def scan_docker_image_pipeline(edition, tag):scan_docker_image_pipeline43,1175 - edition =edition55,1530 - edition =edition57,1579 - docker_image =docker_image59,1608 - cronName =cronName62,1695 - name =name63,1725 - steps =steps64,1775 -def scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image):scan_docker_image_unkown_low_medium_vulnerabilities_step71,2047 -def scan_docker_image_high_critical_vulnerabilities_step(docker_image):scan_docker_image_high_critical_vulnerabilities_step80,2353 -def slack_job_failed_step(channel, image):slack_job_failed_step89,2646 -def post_to_grafana_com_step():post_to_grafana_com_step103,3069 - -events/main.star,633 -ver_mode =ver_mode49,966 -trigger =trigger50,984 -def main_pipelines(edition):main_pipelines62,1168 - drone_change_trigger =drone_change_trigger63,1197 - pipelines =pipelines79,1513 - name =name89,1951 - slack_channel =slack_channel90,1994 - trigger =trigger91,2045 - template =template92,2089 - secret =secret93,2135 - name =name97,2276 - slack_channel =slack_channel98,2310 - trigger =trigger99,2366 - depends_on =depends_on100,2425 - template =template101,2563 - secret =secret102,2604 - -events/pr.star,252 -ver_mode =ver_mode48,997 -trigger =trigger49,1013 -def pr_pipelines(edition):pr_pipelines62,1198 -def get_pr_trigger(include_paths = None, exclude_paths = None):get_pr_trigger76,2396 - paths_ex =paths_ex91,3080 - paths_in =paths_in92,3115 - -services/services.star,225 -def integration_test_services_volumes():integration_test_services_volumes5,79 -def integration_test_services(edition):integration_test_services14,292 - services =services15,332 -def ldap_service():ldap_service59,1616 - -utils/utils.star,561 -failure_template =failure_template11,191 -drone_change_template =drone_change_template12,509 - services =services19,932 - platform =platform20,955 - depends_on =depends_on21,983 - environment =environment22,1008 - volumes =volumes23,1036 - platform_conf =platform_conf50,2166 - platform_conf =platform_conf62,2534 - pipeline =pipeline70,2713 -def notify_pipeline(name, slack_channel, trigger, depends_on = [], template = None, secret = None):notify_pipeline105,3545 - trigger =trigger106,3645 - -pipelines/trigger_downstream.star,440 -trigger =trigger14,249 -def enterprise_downstream_pipeline(edition, ver_mode):enterprise_downstream_pipeline26,433 - environment =environment27,488 - steps =steps28,527 - deps =deps29,587 - name =name31,672 - edition =edition32,714 - trigger =trigger33,741 - services =services34,768 - steps =steps35,791 - depends_on =depends_on36,814 - environment =environment37,841 - -pipelines/verify_starlark.star,323 -def verify_starlark(trigger, ver_mode):verify_starlark17,305 - environment =environment18,345 - steps =steps19,382 - name =name26,546 - edition =edition27,600 - trigger =trigger28,625 - services =services29,652 - steps =steps30,675 - environment =environment31,698 - -pipelines/build.star,508 -def build_e2e(trigger, ver_mode, edition):build_e2e39,936 - environment =environment50,1096 - variants =variants51,1135 - init_steps =init_steps52,1219 - build_steps =build_steps61,1491 - publish_suffix =publish_suffix107,4049 - publish_suffix =publish_suffix109,4100 - name =name112,4158 - edition =edition113,4224 - environment =environment114,4249 - services =services115,4284 - steps =steps116,4307 - trigger =trigger117,4349 - -pipelines/shellcheck.star,386 -trigger =trigger15,235 -def shellcheck_step():shellcheck_step31,483 -def shellcheck_pipeline():shellcheck_pipeline43,725 - environment =environment44,752 - steps =steps45,789 - name =name50,886 - edition =edition51,918 - trigger =trigger52,943 - services =services53,970 - steps =steps54,993 - environment =environment55,1016 - -pipelines/verify_drone.star,317 -def verify_drone(trigger, ver_mode):verify_drone17,293 - environment =environment18,330 - steps =steps19,367 - name =name26,528 - edition =edition27,579 - trigger =trigger28,604 - services =services29,631 - steps =steps30,654 - environment =environment31,677 - -pipelines/test_backend.star,474 -def test_backend(trigger, ver_mode, edition = "oss"):test_backend23,463 - environment =environment35,882 - init_steps =init_steps36,921 - test_steps =test_steps46,1291 - pipeline_name =pipeline_name51,1387 - pipeline_name =pipeline_name53,1492 - name =name55,1584 - edition =edition56,1614 - trigger =trigger57,1641 - services =services58,1668 - steps =steps59,1691 - environment =environment60,1732 - -pipelines/lint_frontend.star,415 -def lint_frontend_pipeline(trigger, ver_mode):lint_frontend_pipeline16,260 - environment =environment26,546 - yarn_step =yarn_step27,583 - init_steps =init_steps29,660 - test_steps =test_steps33,736 - name =name37,812 - edition =edition38,864 - trigger =trigger39,889 - services =services40,916 - steps =steps41,939 - environment =environment42,980 - -pipelines/docs.star,494 -docs_paths =docs_paths19,383 -def docs_pipelines(edition, ver_mode, trigger):docs_pipelines28,511 - environment =environment29,559 - steps =steps30,598 - name =name40,815 - edition =edition41,858 - trigger =trigger42,885 - services =services43,912 - steps =steps44,935 - environment =environment45,958 -def lint_docs():lint_docs48,1000 -def trigger_docs_main():trigger_docs_main63,1328 -def trigger_docs_pr():trigger_docs_pr72,1478 - -pipelines/test_frontend.star,476 -def test_frontend(trigger, ver_mode, edition = "oss"):test_frontend20,374 - environment =environment32,794 - init_steps =init_steps33,833 - test_steps =test_steps41,1102 - pipeline_name =pipeline_name45,1205 - pipeline_name =pipeline_name47,1311 - name =name49,1404 - edition =edition50,1434 - trigger =trigger51,1461 - services =services52,1488 - steps =steps53,1511 - environment =environment54,1552 - -pipelines/integration_tests.star,483 -def integration_tests(trigger, ver_mode, edition):integration_tests26,542 - environment =environment37,900 - services =services38,939 - volumes =volumes39,989 - init_steps =init_steps40,1039 - test_steps =test_steps48,1282 - name =name54,1412 - edition =edition55,1468 - trigger =trigger56,1493 - services =services57,1520 - steps =steps58,1549 - environment =environment59,1590 - volumes =volumes60,1625 - -pipelines/windows.star,954 -def windows(trigger, edition, ver_mode):windows17,339 - environment =environment29,798 - init_cmds =init_cmds30,837 - steps =steps38,1205 - bucket =bucket49,1497 - ver_part =ver_part51,1590 - dir =dir52,1628 - dir =dir54,1670 - bucket =bucket55,1695 - build_no =build_no56,1736 - ver_part =ver_part57,1780 - installer_commands =installer_commands58,1842 - committish =committish100,3763 - committish =committish102,3846 - committish =committish104,3906 - download_grabpl_step_cmds =download_grabpl_step_cmds107,4057 - clone_cmds =clone_cmds113,4363 - name =name146,5711 - edition =edition147,5742 - trigger =trigger148,5769 - steps =steps149,5830 - depends_on =depends_on150,5889 - platform =platform151,6007 - environment =environment152,6037 - -pipelines/lint_backend.star,418 -def lint_backend_pipeline(trigger, ver_mode):lint_backend_pipeline18,306 - environment =environment28,590 - wire_step =wire_step29,627 - init_steps =init_steps31,704 - test_steps =test_steps36,809 - name =name43,959 - edition =edition44,1010 - trigger =trigger45,1035 - services =services46,1062 - steps =steps47,1085 - environment =environment48,1126 - -pipelines/publish_images.star,998 -def publish_image_steps(edition, mode, docker_repo):publish_image_steps17,303 - additional_docker_repo =additional_docker_repo31,922 - additional_docker_repo =additional_docker_repo33,979 - steps =steps34,1034 -def publish_image_pipelines_public():publish_image_pipelines_public45,1369 - mode =mode51,1521 - trigger =trigger52,1541 - name =name57,1641 - trigger =trigger58,1694 - steps =steps59,1721 - edition =edition60,1813 - environment =environment61,1835 - name =name63,1894 - trigger =trigger64,1954 - steps =steps65,1981 - edition =edition66,2091 - environment =environment67,2113 -def publish_image_pipelines_security():publish_image_pipelines_security70,2170 - mode =mode71,2210 - trigger =trigger72,2232 - name =name77,2332 - trigger =trigger78,2392 - steps =steps79,2419 - edition =edition80,2529 - environment =environment81,2551 - -steps/lib.star,8579 -grabpl_version =grabpl_version7,181 -build_image =build_image8,208 -publish_image =publish_image9,254 -deploy_docker_image =deploy_docker_image10,304 -alpine_image =alpine_image11,380 -curl_image =curl_image12,411 -windows_image =windows_image13,452 -wix_image =wix_image14,501 -go_image =go_image15,536 -disable_tests =disable_tests17,564 -trigger_oss =trigger_oss18,586 -def slack_step(channel, template, secret):slack_step24,653 -def yarn_install_step(edition = "oss"):yarn_install_step35,918 - deps =deps36,958 - deps =deps38,1004 -def wire_install_step():wire_install_step48,1222 -def identify_runner_step(platform = "linux"):identify_runner_step60,1454 -def clone_enterprise_step(ver_mode):clone_enterprise_step78,1916 - committish =committish87,2193 - committish =committish89,2268 - committish =committish91,2317 -def init_enterprise_step(ver_mode):init_enterprise_step105,2747 - source_commit =source_commit115,3098 - source_commit =source_commit117,3151 - environment =environment118,3191 - token =token121,3280 - environment =environment123,3369 - token =token126,3458 - environment =environment128,3518 - token =token129,3543 -def download_grabpl_step(platform = "linux"):download_grabpl_step148,4147 -def lint_drone_step():lint_drone_step173,4973 -def lint_starlark_step():lint_starlark_step185,5216 -def enterprise_downstream_step(edition, ver_mode):enterprise_downstream_step206,6000 - repo =repo219,6482 - step =step225,6623 -def lint_backend_step():lint_backend_step247,7248 -def benchmark_ldap_step():benchmark_ldap_step265,7713 -def build_storybook_step(edition, ver_mode):build_storybook_step278,8087 -def store_storybook_step(edition, ver_mode, trigger = None):store_storybook_step300,8743 - commands =commands314,9202 - commands =commands323,9521 - step =step325,9593 - when_cond =when_cond338,10125 - step =step346,10330 -def e2e_tests_artifacts(edition):e2e_tests_artifacts349,10391 -def upload_cdn_step(edition, ver_mode, trigger = None):upload_cdn_step386,12378 - deps =deps397,12763 - step =step407,12970 - step =step420,13423 -def build_backend_step(edition, ver_mode, variants = None):build_backend_step423,13482 - variants_str =variants_str437,14070 - variants_str =variants_str439,14109 - cmds =cmds443,14256 - build_no =build_no449,14418 - cmds =cmds450,14461 -def build_frontend_step(edition, ver_mode):build_frontend_step468,14906 - build_no =build_no478,15246 - cmds =cmds482,15356 - cmds =cmds487,15505 -def build_frontend_package_step(edition, ver_mode):build_frontend_package_step505,15960 - build_no =build_no515,16312 - cmds =cmds519,16422 - cmds =cmds524,16580 -def build_plugins_step(edition, ver_mode):build_plugins_step542,17053 - env =env544,17121 - env =env548,17220 -def test_backend_step():test_backend_step563,17607 -def test_backend_integration_step():test_backend_integration_step575,17880 -def betterer_frontend_step(edition = "oss"):betterer_frontend_step587,18187 - deps =deps596,18427 -def test_frontend_step(edition = "oss"):test_frontend_step609,18728 - deps =deps618,18962 -def lint_frontend_step():lint_frontend_step634,19343 -def test_a11y_frontend_step(ver_mode, edition, port = 3001):test_a11y_frontend_step652,19793 - commands =commands664,20279 - failure =failure667,20345 - failure =failure672,20483 -def frontend_metrics_step(edition, trigger = None):frontend_metrics_step693,21146 - step =step706,21507 - step =step721,22007 -def codespell_step():codespell_step724,22066 -def package_step(edition, ver_mode, variants = None):package_step736,22468 - deps =deps750,23006 - variants_str =variants_str757,23167 - variants_str =variants_str759,23206 - sign_args =sign_args762,23332 - env =env763,23362 - test_args =test_args769,23628 - sign_args =sign_args771,23661 - env =env772,23684 - test_args =test_args773,23703 - cmds =cmds777,23829 - build_no =build_no784,24036 - cmds =cmds785,24079 -def grafana_server_step(edition, port = 3001):grafana_server_step798,24459 - package_file_pfx =package_file_pfx808,24729 - package_file_pfx =package_file_pfx810,24788 - package_file_pfx =package_file_pfx812,24889 - environment =environment814,24938 -def e2e_tests_step(suite, edition, port = 3001, tries = None):e2e_tests_step837,25554 - cmd =cmd838,25617 -def cloud_plugins_e2e_tests_step(suite, edition, cloud, trigger = None):cloud_plugins_e2e_tests_step856,26186 - environment =environment869,26649 - when =when870,26670 - when =when872,26700 - environment =environment874,26748 - when =when882,27129 - branch =branch888,27345 - step =step889,27401 - step =step901,27822 -def build_docs_website_step():build_docs_website_step904,27874 -def copy_packages_for_docker_step(edition = None):copy_packages_for_docker_step916,28272 -def build_docker_images_step(edition, archs = None, ubuntu = False, publish = False):build_docker_images_step929,28622 - cmd =cmd943,29193 - ubuntu_sfx =ubuntu_sfx947,29307 - ubuntu_sfx =ubuntu_sfx949,29342 - environment =environment955,29468 -def fetch_images_step(edition):fetch_images_step979,30079 -def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None):publish_images_step997,30745 - name =name1013,31562 - docker_repo =docker_repo1014,31585 - mode =mode1016,31663 - mode =mode1018,31709 - environment =environment1020,31728 - cmd =cmd1026,31912 - deps =deps1029,32041 - deps =deps1032,32147 - name =name1035,32250 - docker_repo =docker_repo1036,32273 - cmd =cmd1038,32459 - step =step1040,32565 - step =step1052,32929 -def postgres_integration_tests_step():postgres_integration_tests_step1056,32989 - cmds =cmds1057,33028 -def mysql_integration_tests_step():mysql_integration_tests_step1079,33850 - cmds =cmds1080,33886 -def redis_integration_tests_step():redis_integration_tests_step1100,34629 -def memcached_integration_tests_step():memcached_integration_tests_step1114,35026 -def release_canary_npm_packages_step(edition, trigger = None):release_canary_npm_packages_step1128,35435 - step =step1141,35805 - step =step1153,36143 -def enterprise2_suffix(edition):enterprise2_suffix1156,36202 -def upload_packages_step(edition, ver_mode, trigger = None):upload_packages_step1161,36320 - deps =deps1176,36816 - step =step1184,37036 - step =step1195,37471 -def publish_grafanacom_step(edition, ver_mode):publish_grafanacom_step1198,37530 - cmd =cmd1211,38044 - build_no =build_no1215,38188 - cmd =cmd1216,38231 -def publish_linux_packages_step(edition, package_manager = "deb"):publish_linux_packages_step1239,38866 -def get_windows_steps(edition, ver_mode):get_windows_steps1261,39989 - init_cmds =init_cmds1270,40281 - steps =steps1278,40649 - bucket =bucket1289,40941 - ver_part =ver_part1291,41034 - dir =dir1292,41072 - dir =dir1294,41114 - bucket =bucket1295,41139 - build_no =build_no1296,41180 - ver_part =ver_part1297,41224 - installer_commands =installer_commands1298,41286 - committish =committish1340,43207 - committish =committish1342,43290 - committish =committish1344,43350 - download_grabpl_step_cmds =download_grabpl_step_cmds1347,43501 - clone_cmds =clone_cmds1353,43807 -def verify_gen_cue_step(edition):verify_gen_cue_step1387,45152 - deps =deps1388,45186 -def verify_gen_jsonnet_step(edition):verify_gen_jsonnet_step1402,45694 - deps =deps1403,45732 -def trigger_test_release():trigger_test_release1417,46236 -def artifacts_page_step():artifacts_page_step1451,47731 -def end_to_end_tests_deps():end_to_end_tests_deps1466,48058 -def compile_build_cmd(edition = "oss"):compile_build_cmd1476,48321 - dependencies =dependencies1477,48361 - dependencies =dependencies1479,48432 -def get_trigger_storybook(ver_mode):get_trigger_storybook1492,48780 - trigger_storybook =trigger_storybook1500,49031 - trigger_storybook =trigger_storybook1502,49088 - trigger_storybook =trigger_storybook1506,49168 - -vault.star,444 -pull_secret =pull_secret4,87 -github_token =github_token5,120 -drone_token =drone_token6,150 -prerelease_bucket =prerelease_bucket7,178 -gcp_upload_artifacts_key =gcp_upload_artifacts_key8,218 -azure_sp_app_id =azure_sp_app_id9,272 -azure_sp_app_pw =azure_sp_app_pw10,308 -azure_tenant =azure_tenant11,344 -def from_secret(secret):from_secret13,375 -def vault_secret(name, path, key):vault_secret18,451 -def secrets():secrets28,633 - -version.star,116 -ver_mode =ver_mode12,197 -trigger =trigger13,225 -def version_branch_pipelines():version_branch_pipelines15,268 diff --git a/scripts/drone/events/cron.star b/scripts/drone/events/cron.star index 6715c175b17..6f9e524139b 100644 --- a/scripts/drone/events/cron.star +++ b/scripts/drone/events/cron.star @@ -2,15 +2,11 @@ This module provides functions for cronjob pipelines and steps used within. """ -load("scripts/drone/vault.star", "from_secret") -load( - "scripts/drone/steps/lib.star", - "compile_build_cmd", -) load( "scripts/drone/utils/images.star", "images", ) +load("scripts/drone/vault.star", "from_secret") aquasec_trivy_image = "aquasec/trivy:0.21.0" @@ -21,7 +17,6 @@ def cronjobs(): scan_docker_image_pipeline("latest-ubuntu"), scan_docker_image_pipeline("main-ubuntu"), scan_build_test_publish_docker_image_pipeline(), - grafana_com_nightly_pipeline(), ] def authenticate_gcr_step(): @@ -59,6 +54,10 @@ def cron_job_pipeline(cronName, name, steps): "path": "/var/run/docker.sock", }, }, + { + "name": "config", + "temp": {}, + }, ], } @@ -117,13 +116,13 @@ def scan_docker_image_unknown_low_medium_vulnerabilities_step(docker_image): for key in images: cmds = cmds + ["trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + images[key]] else: - cmds = ["trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image] + cmds = ["trivy image --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image] return { "name": "scan-unknown-low-medium-vulnerabilities", "image": aquasec_trivy_image, "commands": cmds, "depends_on": ["authenticate-gcr"], - "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}, {"name": "config", "path": "/root/.docker/"}], } def scan_docker_image_high_critical_vulnerabilities_step(docker_image): @@ -141,19 +140,22 @@ def scan_docker_image_high_critical_vulnerabilities_step(docker_image): for key in images: cmds = cmds + ["trivy --exit-code 1 --severity HIGH,CRITICAL " + images[key]] else: - cmds = ["trivy --exit-code 1 --severity HIGH,CRITICAL " + docker_image] + cmds = ["trivy image --exit-code 1 --severity HIGH,CRITICAL " + docker_image] return { "name": "scan-high-critical-vulnerabilities", "image": aquasec_trivy_image, "commands": cmds, "depends_on": ["authenticate-gcr"], - "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + "environment": { + "GOOGLE_APPLICATION_CREDENTIALS": from_secret("gcr_credentials_json"), + }, + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}, {"name": "config", "path": "/root/.docker/"}], } def slack_job_failed_step(channel, image): return { "name": "slack-notify-failure", - "image": images["plugins_slack_image"], + "image": images["plugins_slack"], "settings": { "webhook": from_secret("slack_webhook_backend"), "channel": channel, @@ -167,7 +169,7 @@ def slack_job_failed_step(channel, image): def post_to_grafana_com_step(): return { "name": "post-to-grafana-com", - "image": images["publish_image"], + "image": images["publish"], "environment": { "GRAFANA_COM_API_KEY": from_secret("grafana_api_key"), "GCP_KEY": from_secret("gcp_key"), @@ -175,13 +177,3 @@ def post_to_grafana_com_step(): "depends_on": ["compile-build-cmd"], "commands": ["./bin/build publish grafana-com --edition oss"], } - -def grafana_com_nightly_pipeline(): - return cron_job_pipeline( - cronName = "grafana-com-nightly", - name = "grafana-com-nightly", - steps = [ - compile_build_cmd(), - post_to_grafana_com_step(), - ], - ) diff --git a/scripts/drone/events/main.star b/scripts/drone/events/main.star index 7c592c53ae8..ee47392ee5c 100644 --- a/scripts/drone/events/main.star +++ b/scripts/drone/events/main.star @@ -3,40 +3,18 @@ This module returns all the pipelines used in the event of pushes to the main br """ load( - "scripts/drone/utils/utils.star", - "drone_change_template", - "failure_template", - "notify_pipeline", + "scripts/drone/pipelines/build.star", + "build_e2e", ) load( "scripts/drone/pipelines/docs.star", "docs_pipelines", "trigger_docs_main", ) -load( - "scripts/drone/pipelines/test_frontend.star", - "test_frontend", -) -load( - "scripts/drone/pipelines/test_backend.star", - "test_backend", -) load( "scripts/drone/pipelines/integration_tests.star", "integration_tests", ) -load( - "scripts/drone/pipelines/build.star", - "build_e2e", -) -load( - "scripts/drone/pipelines/windows.star", - "windows", -) -load( - "scripts/drone/pipelines/trigger_downstream.star", - "enterprise_downstream_pipeline", -) load( "scripts/drone/pipelines/lint_backend.star", "lint_backend_pipeline", @@ -45,6 +23,27 @@ load( "scripts/drone/pipelines/lint_frontend.star", "lint_frontend_pipeline", ) +load( + "scripts/drone/pipelines/test_backend.star", + "test_backend", +) +load( + "scripts/drone/pipelines/test_frontend.star", + "test_frontend", +) +load( + "scripts/drone/pipelines/trigger_downstream.star", + "enterprise_downstream_pipeline", +) +load( + "scripts/drone/pipelines/windows.star", + "windows", +) +load( + "scripts/drone/utils/utils.star", + "failure_template", + "notify_pipeline", +) ver_mode = "main" trigger = { @@ -59,27 +58,12 @@ trigger = { "latest.json", ], }, + "repo": [ + "grafana/grafana", + ], } def main_pipelines(): - drone_change_trigger = { - "event": [ - "push", - ], - "branch": "main", - "repo": [ - "grafana/grafana", - ], - "paths": { - "include": [ - ".drone.yml", - ], - "exclude": [ - "exclude", - ], - }, - } - pipelines = [ docs_pipelines(ver_mode, trigger_docs_main()), test_frontend(trigger, ver_mode), @@ -89,13 +73,6 @@ def main_pipelines(): build_e2e(trigger, ver_mode), integration_tests(trigger, prefix = ver_mode, ver_mode = ver_mode), windows(trigger, ver_mode = ver_mode), - notify_pipeline( - name = "notify-drone-changes", - slack_channel = "slack-webhooks-test", - trigger = drone_change_trigger, - template = drone_change_template, - secret = "drone-changes-webhook", - ), enterprise_downstream_pipeline(), notify_pipeline( name = "main-notify", diff --git a/scripts/drone/events/pr.star b/scripts/drone/events/pr.star index 476ef660e76..cbd425efcf6 100644 --- a/scripts/drone/events/pr.star +++ b/scripts/drone/events/pr.star @@ -4,37 +4,21 @@ It also includes a function generating a PR trigger from a list of included and """ load( - "scripts/drone/pipelines/test_frontend.star", - "test_frontend", -) -load( - "scripts/drone/pipelines/test_backend.star", - "test_backend", -) -load( - "scripts/drone/pipelines/integration_tests.star", - "integration_tests", + "scripts/drone/pipelines/benchmarks.star", + "integration_benchmarks", ) load( "scripts/drone/pipelines/build.star", "build_e2e", ) -load( - "scripts/drone/pipelines/verify_drone.star", - "verify_drone", -) -load( - "scripts/drone/pipelines/verify_starlark.star", - "verify_starlark", -) load( "scripts/drone/pipelines/docs.star", "docs_pipelines", "trigger_docs_pr", ) load( - "scripts/drone/pipelines/shellcheck.star", - "shellcheck_pipeline", + "scripts/drone/pipelines/integration_tests.star", + "integration_tests", ) load( "scripts/drone/pipelines/lint_backend.star", @@ -44,6 +28,26 @@ load( "scripts/drone/pipelines/lint_frontend.star", "lint_frontend_pipeline", ) +load( + "scripts/drone/pipelines/shellcheck.star", + "shellcheck_pipeline", +) +load( + "scripts/drone/pipelines/test_backend.star", + "test_backend", +) +load( + "scripts/drone/pipelines/test_frontend.star", + "test_frontend", +) +load( + "scripts/drone/pipelines/verify_drone.star", + "verify_drone", +) +load( + "scripts/drone/pipelines/verify_starlark.star", + "verify_starlark", +) ver_mode = "pr" trigger = { @@ -130,10 +134,12 @@ def pr_pipelines(): ], ), prefix = ver_mode, - ver_mode = ver_mode, ), docs_pipelines(ver_mode, trigger_docs_pr()), shellcheck_pipeline(), + integration_benchmarks( + prefix = ver_mode, + ), ] def get_pr_trigger(include_paths = None, exclude_paths = None): diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index 494a453ca8a..5e71f1eed04 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -2,53 +2,36 @@ This module returns all the pipelines used in the event of a release along with supporting functions. """ -load( - "scripts/drone/steps/lib.star", - "build_backend_step", - "build_docker_images_step", - "build_frontend_package_step", - "build_frontend_step", - "build_plugins_step", - "build_storybook_step", - "compile_build_cmd", - "copy_packages_for_docker_step", - "download_grabpl_step", - "e2e_tests_artifacts", - "e2e_tests_step", - "get_windows_steps", - "grafana_server_step", - "identify_runner_step", - "mysql_integration_tests_step", - "package_step", - "postgres_integration_tests_step", - "publish_grafanacom_step", - "publish_linux_packages_step", - "store_storybook_step", - "trigger_oss", - "upload_cdn_step", - "upload_packages_step", - "verify_gen_cue_step", - "verify_gen_jsonnet_step", - "wire_install_step", - "yarn_install_step", -) load( "scripts/drone/services/services.star", "integration_test_services", "integration_test_services_volumes", ) +load( + "scripts/drone/steps/lib.star", + "compile_build_cmd", + "download_grabpl_step", + "identify_runner_step", + "memcached_integration_tests_steps", + "mysql_integration_tests_steps", + "postgres_integration_tests_steps", + "publish_grafanacom_step", + "publish_linux_packages_step", + "redis_integration_tests_steps", + "remote_alertmanager_integration_tests_steps", + "verify_gen_cue_step", + "verify_gen_jsonnet_step", + "wire_install_step", + "yarn_install_step", +) +load( + "scripts/drone/utils/images.star", + "images", +) load( "scripts/drone/utils/utils.star", "pipeline", ) -load( - "scripts/drone/pipelines/test_frontend.star", - "test_frontend", -) -load( - "scripts/drone/pipelines/test_backend.star", - "test_backend", -) load( "scripts/drone/vault.star", "from_secret", @@ -57,51 +40,13 @@ load( "prerelease_bucket", "rgm_gcp_key_base64", ) -load( - "scripts/drone/utils/images.star", - "images", -) -load( - "scripts/drone/pipelines/whats_new_checker.star", - "whats_new_checker_pipeline", -) ver_mode = "release" -release_trigger = { - "event": { - "exclude": [ - "promote", - ], - }, - "ref": { - "include": [ - "refs/tags/v*", - ], - "exclude": [ - "refs/tags/*-cloud*", - ], - }, -} - -def store_npm_packages_step(): - return { - "name": "store-npm-packages", - "image": images["build_image"], - "depends_on": [ - "compile-build-cmd", - "build-frontend-packages", - ], - "environment": { - "GCP_KEY": from_secret(gcp_grafanauploads_base64), - "PRERELEASE_BUCKET": from_secret(prerelease_bucket), - }, - "commands": ["./bin/build artifacts npm store --tag ${DRONE_TAG}"], - } def retrieve_npm_packages_step(): return { "name": "retrieve-npm-packages", - "image": images["publish_image"], + "image": images["publish"], "depends_on": [ "compile-build-cmd", "yarn-install", @@ -117,7 +62,7 @@ def retrieve_npm_packages_step(): def release_npm_packages_step(): return { "name": "release-npm-packages", - "image": images["build_image"], + "image": images["go"], "depends_on": [ "compile-build-cmd", "retrieve-npm-packages", @@ -129,142 +74,10 @@ def release_npm_packages_step(): "commands": ["./bin/build artifacts npm release --tag ${DRONE_TAG}"], } -def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger): - """Generates all pipelines used for Grafana OSS. - - Args: - ver_mode: controls which steps are included in the pipeline. - Defaults to 'release'. - trigger: controls which events can trigger the pipeline execution. - Defaults to tag events for tags with a 'v' prefix. - - Returns: - List of Drone pipelines. - """ - environment = {"EDITION": "oss"} - - services = integration_test_services(edition = "oss") - volumes = integration_test_services_volumes() - - init_steps = [ - identify_runner_step(), - download_grabpl_step(), - verify_gen_cue_step(), - wire_install_step(), - yarn_install_step(), - compile_build_cmd(), - ] - - build_steps = [ - build_backend_step(ver_mode = ver_mode), - build_frontend_step(ver_mode = ver_mode), - build_frontend_package_step(ver_mode = ver_mode), - build_plugins_step(ver_mode = ver_mode), - package_step(ver_mode = ver_mode), - copy_packages_for_docker_step(), - build_docker_images_step(publish = True), - build_docker_images_step( - publish = True, - ubuntu = True, - ), - grafana_server_step(), - e2e_tests_step("dashboards-suite", tries = 3), - e2e_tests_step("smoke-tests-suite", tries = 3), - e2e_tests_step("panels-suite", tries = 3), - e2e_tests_step("various-suite", tries = 3), - e2e_tests_artifacts(), - build_storybook_step(ver_mode = ver_mode), - ] - - publish_steps = [] - - if ver_mode in ( - "release", - "release-branch", - ): - publish_steps.extend( - [ - upload_cdn_step(ver_mode = ver_mode, trigger = trigger_oss), - upload_packages_step( - ver_mode = ver_mode, - trigger = trigger_oss, - ), - ], - ) - - if ver_mode in ("release",): - publish_steps.extend( - [ - store_storybook_step(ver_mode = ver_mode), - store_npm_packages_step(), - ], - ) - - integration_test_steps = [ - postgres_integration_tests_step(), - mysql_integration_tests_step(), - ] - - pipelines = [] - - # We don't need to run integration tests at release time since they have - # been run multiple times before: - if ver_mode in ("release"): - pipelines.append(whats_new_checker_pipeline(release_trigger)) - integration_test_steps = [] - volumes = [] - - windows_pipeline_dependencies = [ - "{}-build-e2e-publish".format(ver_mode), - "{}-test-frontend".format(ver_mode), - ] - pipelines.extend([ - pipeline( - name = "{}-build-e2e-publish".format(ver_mode), - trigger = trigger, - services = [], - steps = init_steps + build_steps + publish_steps, - environment = environment, - volumes = volumes, - ), - test_frontend(trigger, ver_mode), - test_backend(trigger, ver_mode), - ]) - - if ver_mode not in ("release"): - pipelines.append(pipeline( - name = "{}-integration-tests".format(ver_mode), - trigger = trigger, - services = services, - steps = [ - download_grabpl_step(), - identify_runner_step(), - verify_gen_cue_step(), - verify_gen_jsonnet_step(), - wire_install_step(), - ] + - integration_test_steps, - environment = environment, - volumes = volumes, - )) - - windows_pipeline = pipeline( - name = "{}-windows".format(ver_mode), - trigger = trigger, - steps = get_windows_steps(ver_mode = ver_mode), - platform = "windows", - depends_on = windows_pipeline_dependencies, - environment = environment, - ) - - pipelines.append(windows_pipeline) - - return pipelines - def publish_artifacts_step(): return { "name": "publish-artifacts", - "image": images["publish_image"], + "image": images["publish"], "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -278,7 +91,7 @@ def publish_artifacts_step(): def publish_static_assets_step(): return { "name": "publish-static-assets", - "image": images["publish_image"], + "image": images["publish"], "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -293,7 +106,7 @@ def publish_static_assets_step(): def publish_storybook_step(): return { "name": "publish-storybook", - "image": images["publish_image"], + "image": images["publish"], "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -405,15 +218,17 @@ def integration_test_pipelines(): } pipelines = [] volumes = integration_test_services_volumes() - oss_integration_test_steps = [ - postgres_integration_tests_step(), - mysql_integration_tests_step(), - ] + integration_test_steps = postgres_integration_tests_steps() + \ + mysql_integration_tests_steps("mysql57", "5.7") + \ + mysql_integration_tests_steps("mysql80", "8.0") + \ + redis_integration_tests_steps() + \ + memcached_integration_tests_steps() + \ + remote_alertmanager_integration_tests_steps() pipelines.append(pipeline( name = "integration-tests", trigger = trigger, - services = integration_test_services(edition = "oss"), + services = integration_test_services(), steps = [ download_grabpl_step(), identify_runner_step(), @@ -421,7 +236,7 @@ def integration_test_pipelines(): verify_gen_jsonnet_step(), wire_install_step(), ] + - oss_integration_test_steps, + integration_test_steps, environment = {"EDITION": "oss"}, volumes = volumes, )) @@ -433,7 +248,7 @@ def verify_release_pipeline( bucket = from_secret(prerelease_bucket), gcp_key = from_secret(rgm_gcp_key_base64), version = "${DRONE_TAG}", - trigger = release_trigger, + trigger = {}, depends_on = [ "release-build-e2e-publish", "release-windows", @@ -447,7 +262,7 @@ def verify_release_pipeline( step = { "name": "gsutil-stat", "depends_on": ["clone"], - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "environment": { "BUCKET": bucket, "GCP_KEY": gcp_key, diff --git a/scripts/drone/pipelines/benchmarks.star b/scripts/drone/pipelines/benchmarks.star new file mode 100644 index 00000000000..ae5a9611675 --- /dev/null +++ b/scripts/drone/pipelines/benchmarks.star @@ -0,0 +1,80 @@ +""" +This module returns the pipeline used for integration benchmarks. +""" + +load( + "scripts/drone/services/services.star", + "integration_test_services", + "integration_test_services_volumes", +) +load( + "scripts/drone/steps/lib.star", + "compile_build_cmd", + "enterprise_setup_step", + "integration_benchmarks_step", + "verify_gen_cue_step", + "verify_gen_jsonnet_step", + "wire_install_step", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", +) + +def integration_benchmarks(prefix): + """Generate a pipeline for integration tests. + + Args: + prefix: used in the naming of the pipeline. + Returns: + Drone pipeline. + """ + environment = {"EDITION": "oss"} + + services = integration_test_services() + volumes = integration_test_services_volumes() + + # In pull requests, attempt to clone grafana enterprise. + init_steps = [enterprise_setup_step(isPromote = True)] + + verify_step = verify_gen_cue_step() + verify_jsonnet_step = verify_gen_jsonnet_step() + + # Ensure that verif_gen_cue happens after we clone enterprise + # At the time of writing this, very_gen_cue is depended on by the wire step which is what everything else depends on. + verify_step["depends_on"].append("clone-enterprise") + verify_jsonnet_step["depends_on"].append("clone-enterprise") + + init_steps += [ + compile_build_cmd(), + verify_step, + verify_jsonnet_step, + wire_install_step(), + ] + + benchmark_steps = integration_benchmarks_step("sqlite") + \ + integration_benchmarks_step("postgres", { + "PGPASSWORD": "grafanatest", + "GRAFANA_TEST_DB": "postgres", + "POSTGRES_HOST": "postgres", + }) + \ + integration_benchmarks_step("mysql-5.7", { + "GRAFANA_TEST_DB": "mysql", + "MYSQL_HOST": "mysql57", + }) + \ + integration_benchmarks_step("mysql-8.0", { + "GRAFANA_TEST_DB": "mysql", + "MYSQL_HOST": "mysql80", + }) + + return pipeline( + name = "{}-integration-benchmarks".format(prefix), + trigger = { + "event": ["promote"], + "target": ["gobenchmarks"], + }, + environment = environment, + services = services, + volumes = volumes, + steps = init_steps + benchmark_steps, + ) diff --git a/scripts/drone/pipelines/build.star b/scripts/drone/pipelines/build.star index cbf6532f08a..88b5ed8e52a 100644 --- a/scripts/drone/pipelines/build.star +++ b/scripts/drone/pipelines/build.star @@ -2,15 +2,10 @@ load( "scripts/drone/steps/lib.star", - "build_backend_step", - "build_docker_images_step", "build_frontend_package_step", - "build_frontend_step", - "build_plugins_step", "build_storybook_step", "cloud_plugins_e2e_tests_step", "compile_build_cmd", - "copy_packages_for_docker_step", "download_grabpl_step", "e2e_tests_artifacts", "e2e_tests_step", @@ -18,12 +13,12 @@ load( "frontend_metrics_step", "grafana_server_step", "identify_runner_step", - "package_step", "publish_images_step", "release_canary_npm_packages_step", "store_storybook_step", "test_a11y_frontend_step", "trigger_oss", + "update_package_json_version", "upload_cdn_step", "upload_packages_step", "verify_gen_cue_step", @@ -31,6 +26,15 @@ load( "wire_install_step", "yarn_install_step", ) +load( + "scripts/drone/steps/rgm.star", + "rgm_build_docker_step", + "rgm_package_step", +) +load( + "scripts/drone/utils/images.star", + "images", +) load( "scripts/drone/utils/utils.star", "pipeline", @@ -38,11 +42,11 @@ load( # @unused def build_e2e(trigger, ver_mode): - """Perform e2e building, testing, and publishing." + """Perform e2e building, testing, and publishing. Args: trigger: controls which events can trigger the pipeline execution. - ver_mode: used in the naming of the pipeline. + ver_mode: used in the naming of the pipeline. Either 'pr' or 'main'. Returns: Drone pipeline. @@ -64,17 +68,19 @@ def build_e2e(trigger, ver_mode): if ver_mode == "pr": build_steps.extend( [ + build_frontend_package_step(), enterprise_downstream_step(ver_mode = ver_mode), ], ) + else: + build_steps.extend([ + update_package_json_version(), + build_frontend_package_step(depends_on = ["update-package-json-version"]), + ]) build_steps.extend( [ - build_backend_step(ver_mode = ver_mode), - build_frontend_step(ver_mode = ver_mode), - build_frontend_package_step(ver_mode = ver_mode), - build_plugins_step(ver_mode = ver_mode), - package_step(ver_mode = ver_mode), + rgm_package_step(distros = "linux/amd64,linux/arm64", file = "packages.txt"), grafana_server_step(), e2e_tests_step("dashboards-suite"), e2e_tests_step("smoke-tests-suite"), @@ -87,7 +93,6 @@ def build_e2e(trigger, ver_mode): ), e2e_tests_artifacts(), build_storybook_step(ver_mode = ver_mode), - copy_packages_for_docker_step(), test_a11y_frontend_step(ver_mode = ver_mode), ], ) @@ -97,12 +102,12 @@ def build_e2e(trigger, ver_mode): [ store_storybook_step(trigger = trigger_oss, ver_mode = ver_mode), frontend_metrics_step(trigger = trigger_oss), - build_docker_images_step( - publish = False, - ), - build_docker_images_step( - publish = False, - ubuntu = True, + rgm_build_docker_step( + "packages.txt", + images["ubuntu"], + images["alpine"], + tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}", + ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}", ), publish_images_step( docker_repo = "grafana", @@ -128,16 +133,12 @@ def build_e2e(trigger, ver_mode): elif ver_mode == "pr": build_steps.extend( [ - build_docker_images_step( - archs = [ - "amd64", - ], - ), - build_docker_images_step( - archs = [ - "amd64", - ], - ubuntu = True, + rgm_build_docker_step( + "packages.txt", + images["ubuntu"], + images["alpine"], + tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}", + ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}", ), publish_images_step( docker_repo = "grafana", diff --git a/scripts/drone/pipelines/ci_images.star b/scripts/drone/pipelines/ci_images.star index 65df476754b..e8180d6d0ef 100644 --- a/scripts/drone/pipelines/ci_images.star +++ b/scripts/drone/pipelines/ci_images.star @@ -3,21 +3,21 @@ This module contains steps and pipelines relating to creating CI Docker images. """ load( - "scripts/drone/utils/utils.star", - "pipeline", + "scripts/drone/utils/images.star", + "images", ) load( - "scripts/drone/vault.star", - "from_secret", - "gcp_download_build_container_assets_key", + "scripts/drone/utils/utils.star", + "pipeline", ) load( "scripts/drone/utils/windows_images.star", "windows_images", ) load( - "scripts/drone/utils/images.star", - "images", + "scripts/drone/vault.star", + "from_secret", + "gcp_download_build_container_assets_key", ) def publish_ci_windows_test_image_pipeline(): @@ -32,7 +32,7 @@ def publish_ci_windows_test_image_pipeline(): steps = [ { "name": "clone", - "image": windows_images["wix_image"], + "image": windows_images["wix"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, @@ -43,7 +43,7 @@ def publish_ci_windows_test_image_pipeline(): }, { "name": "build-and-publish", - "image": windows_images["windows_server_core_image"], + "image": windows_images["windows_server_core"], "environment": { "DOCKER_USERNAME": from_secret("docker_username"), "DOCKER_PASSWORD": from_secret("docker_password"), @@ -81,14 +81,14 @@ def publish_ci_build_container_image_pipeline(): steps = [ { "name": "validate-version", - "image": images["alpine_image"], + "image": images["alpine"], "commands": [ "if [ -z \"${BUILD_CONTAINER_VERSION}\" ]; then echo Missing BUILD_CONTAINER_VERSION; false; fi", ], }, { "name": "download-macos-sdk", - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "environment": { "GCP_KEY": from_secret(gcp_download_build_container_assets_key), }, @@ -100,7 +100,7 @@ def publish_ci_build_container_image_pipeline(): }, { "name": "build-and-publish", # Consider splitting the build and the upload task. - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], "environment": { "DOCKER_USERNAME": from_secret("docker_username"), diff --git a/scripts/drone/pipelines/docs.star b/scripts/drone/pipelines/docs.star index 5f2cbdb8ef3..536749b9e04 100644 --- a/scripts/drone/pipelines/docs.star +++ b/scripts/drone/pipelines/docs.star @@ -7,6 +7,7 @@ load( "build_docs_website_step", "codespell_step", "identify_runner_step", + "verify_gen_cue_step", "yarn_install_step", ) load( @@ -35,6 +36,7 @@ def docs_pipelines(ver_mode, trigger): codespell_step(), lint_docs(), build_docs_website_step(), + verify_gen_cue_step(), ] return pipeline( @@ -48,7 +50,7 @@ def docs_pipelines(ver_mode, trigger): def lint_docs(): return { "name": "lint-docs", - "image": images["build_image"], + "image": images["node"], "depends_on": [ "yarn-install", ], @@ -66,6 +68,9 @@ def trigger_docs_main(): "event": [ "push", ], + "repo": [ + "grafana/grafana", + ], "paths": docs_paths, } @@ -74,5 +79,8 @@ def trigger_docs_pr(): "event": [ "pull_request", ], + "repo": [ + "grafana/grafana", + ], "paths": docs_paths, } diff --git a/scripts/drone/pipelines/integration_tests.star b/scripts/drone/pipelines/integration_tests.star index b63b634755b..e51b55bf48f 100644 --- a/scripts/drone/pipelines/integration_tests.star +++ b/scripts/drone/pipelines/integration_tests.star @@ -2,29 +2,32 @@ This module returns the pipeline used for integration tests. """ -load( - "scripts/drone/steps/lib.star", - "compile_build_cmd", - "download_grabpl_step", - "enterprise_setup_step", - "identify_runner_step", - "mysql_integration_tests_step", - "postgres_integration_tests_step", - "verify_gen_cue_step", - "verify_gen_jsonnet_step", - "wire_install_step", -) load( "scripts/drone/services/services.star", "integration_test_services", "integration_test_services_volumes", ) +load( + "scripts/drone/steps/lib.star", + "compile_build_cmd", + "download_grabpl_step", + "enterprise_setup_step", + "identify_runner_step", + "memcached_integration_tests_steps", + "mysql_integration_tests_steps", + "postgres_integration_tests_steps", + "redis_integration_tests_steps", + "remote_alertmanager_integration_tests_steps", + "verify_gen_cue_step", + "verify_gen_jsonnet_step", + "wire_install_step", +) load( "scripts/drone/utils/utils.star", "pipeline", ) -def integration_tests(trigger, prefix, ver_mode): +def integration_tests(trigger, prefix, ver_mode = "pr"): """Generate a pipeline for integration tests. Args: @@ -37,11 +40,14 @@ def integration_tests(trigger, prefix, ver_mode): """ environment = {"EDITION": "oss"} - services = integration_test_services(edition = "oss") + services = integration_test_services() volumes = integration_test_services_volumes() init_steps = [] + verify_step = verify_gen_cue_step() + verify_jsonnet_step = verify_gen_jsonnet_step() + if ver_mode == "pr": # In pull requests, attempt to clone grafana enterprise. init_steps.append(enterprise_setup_step()) @@ -50,15 +56,18 @@ def integration_tests(trigger, prefix, ver_mode): download_grabpl_step(), compile_build_cmd(), identify_runner_step(), - verify_gen_cue_step(), - verify_gen_jsonnet_step(), + verify_step, + verify_jsonnet_step, wire_install_step(), ] - test_steps = [ - postgres_integration_tests_step(), - mysql_integration_tests_step(), - ] + # test_steps = [a, b] + [c, d] + [e, f]... + test_steps = postgres_integration_tests_steps() + \ + mysql_integration_tests_steps("mysql57", "5.7") + \ + mysql_integration_tests_steps("mysql80", "8.0") + \ + redis_integration_tests_steps() + \ + memcached_integration_tests_steps() + \ + remote_alertmanager_integration_tests_steps() return pipeline( name = "{}-integration-tests".format(prefix), diff --git a/scripts/drone/pipelines/lint_backend.star b/scripts/drone/pipelines/lint_backend.star index f8253d0c544..e0c25495b05 100644 --- a/scripts/drone/pipelines/lint_backend.star +++ b/scripts/drone/pipelines/lint_backend.star @@ -44,6 +44,10 @@ def lint_backend_pipeline(trigger, ver_mode): test_steps = [ lint_backend_step(), + # modowners doesn't exist for versions below 10.1.x. + # validate_modfile_step(), + # OpenAPI spec validation is disabled for versions below 10.2.x. + # validate_openapi_spec_step(), ] if ver_mode == "main": diff --git a/scripts/drone/pipelines/lint_frontend.star b/scripts/drone/pipelines/lint_frontend.star index 8093fa4d27c..c0e035babcf 100644 --- a/scripts/drone/pipelines/lint_frontend.star +++ b/scripts/drone/pipelines/lint_frontend.star @@ -27,6 +27,9 @@ def lint_frontend_pipeline(trigger, ver_mode): environment = {"EDITION": "oss"} init_steps = [] + lint_step = lint_frontend_step() + # i18n step is disabled for versions below 10.1.x. + # i18n_step = verify_i18n_step() if ver_mode == "pr": # In pull requests, attempt to clone grafana enterprise. @@ -36,9 +39,10 @@ def lint_frontend_pipeline(trigger, ver_mode): identify_runner_step(), yarn_install_step(), ] - test_steps = [ - lint_frontend_step(), + lint_step, + # i18n step is disabled for versions below 10.1.x. + # i18n_step, ] return pipeline( diff --git a/scripts/drone/pipelines/shellcheck.star b/scripts/drone/pipelines/shellcheck.star index 5b8d481094c..7f56c553976 100644 --- a/scripts/drone/pipelines/shellcheck.star +++ b/scripts/drone/pipelines/shellcheck.star @@ -3,14 +3,14 @@ This module returns a Drone step and pipeline for linting with shellcheck. """ load("scripts/drone/steps/lib.star", "compile_build_cmd") -load( - "scripts/drone/utils/utils.star", - "pipeline", -) load( "scripts/drone/utils/images.star", "images", ) +load( + "scripts/drone/utils/utils.star", + "pipeline", +) trigger = { "event": [ @@ -29,12 +29,10 @@ trigger = { def shellcheck_step(): return { "name": "shellcheck", - "image": images["build_image"], - "depends_on": [ - "compile-build-cmd", - ], + "image": images["ubuntu"], "commands": [ - "./bin/build shellcheck", + "apt-get update -yq && apt-get install shellcheck", + "shellcheck -e SC1071 -e SC2162 scripts/**/*.sh", ], } diff --git a/scripts/drone/pipelines/test_backend.star b/scripts/drone/pipelines/test_backend.star index 627b398a3bf..c5bb47da11e 100644 --- a/scripts/drone/pipelines/test_backend.star +++ b/scripts/drone/pipelines/test_backend.star @@ -2,13 +2,8 @@ This module returns the pipeline used for testing backend code. """ -load( - "scripts/drone/utils/utils.star", - "pipeline", -) load( "scripts/drone/steps/lib.star", - "compile_build_cmd", "enterprise_setup_step", "identify_runner_step", "test_backend_integration_step", @@ -17,6 +12,10 @@ load( "verify_gen_jsonnet_step", "wire_install_step", ) +load( + "scripts/drone/utils/utils.star", + "pipeline", +) def test_backend(trigger, ver_mode): """Generates the pipeline used for testing OSS backend code. @@ -32,15 +31,17 @@ def test_backend(trigger, ver_mode): steps = [] + verify_step = verify_gen_cue_step() + verify_jsonnet_step = verify_gen_jsonnet_step() + if ver_mode == "pr": # In pull requests, attempt to clone grafana enterprise. steps.append(enterprise_setup_step()) steps += [ identify_runner_step(), - compile_build_cmd(), - verify_gen_cue_step(), - verify_gen_jsonnet_step(), + verify_step, + verify_jsonnet_step, wire_install_step(), test_backend_step(), test_backend_integration_step(), diff --git a/scripts/drone/pipelines/test_frontend.star b/scripts/drone/pipelines/test_frontend.star index 17e0b708fae..9a01f101c58 100644 --- a/scripts/drone/pipelines/test_frontend.star +++ b/scripts/drone/pipelines/test_frontend.star @@ -2,19 +2,18 @@ This module returns the pipeline used for testing backend code. """ -load( - "scripts/drone/utils/utils.star", - "pipeline", -) load( "scripts/drone/steps/lib.star", "betterer_frontend_step", - "download_grabpl_step", "enterprise_setup_step", "identify_runner_step", "test_frontend_step", "yarn_install_step", ) +load( + "scripts/drone/utils/utils.star", + "pipeline", +) def test_frontend(trigger, ver_mode): """Generates the pipeline used for testing frontend code. @@ -30,7 +29,6 @@ def test_frontend(trigger, ver_mode): steps = [ identify_runner_step(), - download_grabpl_step(), yarn_install_step(), betterer_frontend_step(), ] diff --git a/scripts/drone/pipelines/whats_new_checker.star b/scripts/drone/pipelines/whats_new_checker.star index e15a8d7a291..2c56be573ce 100644 --- a/scripts/drone/pipelines/whats_new_checker.star +++ b/scripts/drone/pipelines/whats_new_checker.star @@ -2,14 +2,14 @@ This module contains logic for checking if the package.json whats new url matches with the in-flight tag. """ -load( - "scripts/drone/utils/images.star", - "images", -) load( "scripts/drone/steps/lib.star", "compile_build_cmd", ) +load( + "scripts/drone/utils/images.star", + "images", +) load( "scripts/drone/utils/utils.star", "pipeline", @@ -18,7 +18,7 @@ load( def whats_new_checker_step(): return { "name": "whats-new-checker", - "image": images["go_image"], + "image": images["go"], "depends_on": [ "compile-build-cmd", ], diff --git a/scripts/drone/pipelines/windows.star b/scripts/drone/pipelines/windows.star index fd5b99c9bf0..59c12a19536 100644 --- a/scripts/drone/pipelines/windows.star +++ b/scripts/drone/pipelines/windows.star @@ -3,15 +3,15 @@ This module returns the pipeline used for building Grafana on Windows. """ load( - "scripts/drone/utils/utils.star", - "pipeline", + "scripts/drone/steps/lib_windows.star", + "clone_step_windows", + "get_windows_steps", + "test_backend_step_windows", + "wire_install_step_windows", ) load( - "scripts/drone/steps/lib.star", - "get_windows_steps", - "windows_clone_step", - "windows_test_backend_step", - "windows_wire_install_step", + "scripts/drone/utils/utils.star", + "pipeline", ) load( "scripts/drone/utils/windows_images.star", @@ -30,19 +30,19 @@ def windows_test_backend(trigger, edition, ver_mode): """ environment = {"EDITION": edition} steps = [ - windows_clone_step(), + clone_step_windows(), ] steps.extend([{ "name": "windows-init", - "image": windows_images["windows_go_image"], + "image": windows_images["go"], "depends_on": ["clone"], "commands": [], }]) steps.extend([ - windows_wire_install_step(edition), - windows_test_backend_step(), + wire_install_step_windows(edition), + test_backend_step_windows(), ]) pl = pipeline( name = "{}-test-backend-windows".format(ver_mode), diff --git a/scripts/drone/rgm.star b/scripts/drone/rgm.star index 0721f76b32b..cd4df9b9de8 100644 --- a/scripts/drone/rgm.star +++ b/scripts/drone/rgm.star @@ -4,50 +4,52 @@ rgm uses 'github.com/grafana/grafana-build' to build Grafana on the following ev * A tag that begins with a 'v' """ -load( - "scripts/drone/steps/lib.star", - "get_windows_steps", -) -load( - "scripts/drone/utils/utils.star", - "ignore_failure", - "pipeline", -) load( "scripts/drone/events/release.star", "verify_release_pipeline", ) -load( - "scripts/drone/pipelines/test_frontend.star", - "test_frontend", -) load( "scripts/drone/pipelines/test_backend.star", "test_backend", ) +load( + "scripts/drone/pipelines/test_frontend.star", + "test_frontend", +) load( "scripts/drone/pipelines/whats_new_checker.star", "whats_new_checker_pipeline", ) +load( + "scripts/drone/steps/lib_windows.star", + "get_windows_steps", +) +load( + "scripts/drone/utils/images.star", + "images", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", + "with_deps", +) +load( + "scripts/drone/variables.star", + "golang_version", +) load( "scripts/drone/vault.star", "from_secret", + "npm_token", + "rgm_cdn_destination", "rgm_dagger_token", "rgm_destination", + "rgm_downloads_destination", "rgm_gcp_key_base64", "rgm_github_token", + "rgm_storybook_destination", ) -rgm_env_secrets = { - "GCP_KEY_BASE64": from_secret(rgm_gcp_key_base64), - "DESTINATION": from_secret(rgm_destination), - "GITHUB_TOKEN": from_secret(rgm_github_token), - "_EXPERIMENTAL_DAGGER_CLOUD_TOKEN": from_secret(rgm_dagger_token), - "GPG_PRIVATE_KEY": from_secret("packages_gpg_private_key"), - "GPG_PUBLIC_KEY": from_secret("packages_gpg_public_key"), - "GPG_PASSPHRASE": from_secret("packages_gpg_passphrase"), -} - docs_paths = { "exclude": [ "*.md", @@ -73,75 +75,233 @@ tag_trigger = { }, } -def rgm_build(script = "drone_publish_main.sh", canFail = True): - rgm_build_step = { - "name": "rgm-build", +main_trigger = { + "event": [ + "push", + ], + "branch": "main", + "paths": docs_paths, + "repo": [ + "grafana/grafana", + ], +} + +nightly_trigger = { + "event": { + "include": [ + "cron", + ], + }, + "cron": { + "include": [ + "nightly-release", + ], + }, +} + +version_branch_trigger = {"ref": ["refs/heads/v[0-9]*"]} + +def rgm_env_secrets(env): + """Adds the rgm secret ENV variables to the given env arg + + Args: + env: A map of environment varables. This function will adds the necessary secrets to it (and potentially overwrite them). + Returns: + Drone step. + """ + env["DESTINATION"] = from_secret(rgm_destination) + env["STORYBOOK_DESTINATION"] = from_secret(rgm_storybook_destination) + env["CDN_DESTINATION"] = from_secret(rgm_cdn_destination) + env["DOWNLOADS_DESTINATION"] = from_secret(rgm_downloads_destination) + + env["GCP_KEY_BASE64"] = from_secret(rgm_gcp_key_base64) + env["GITHUB_TOKEN"] = from_secret(rgm_github_token) + env["_EXPERIMENTAL_DAGGER_CLOUD_TOKEN"] = from_secret(rgm_dagger_token) + env["GPG_PRIVATE_KEY"] = from_secret("packages_gpg_private_key") + env["GPG_PUBLIC_KEY"] = from_secret("packages_gpg_public_key") + env["GPG_PASSPHRASE"] = from_secret("packages_gpg_passphrase") + env["DOCKER_USERNAME"] = from_secret("docker_username") + env["DOCKER_PASSWORD"] = from_secret("docker_password") + env["NPM_TOKEN"] = from_secret(npm_token) + env["GCOM_API_KEY"] = from_secret("grafana_api_key") + return env + +def rgm_run(name, script): + """Returns a pipeline that does a full build & package of Grafana. + + Args: + name: The name of the pipeline step. + script: The script in the container to run. + Returns: + Drone step. + """ + env = { + "GO_VERSION": golang_version, + } + rgm_run_step = { + "name": name, "image": "grafana/grafana-build:main", + "pull": "always", "commands": [ "export GRAFANA_DIR=$$(pwd)", "cd /src && ./scripts/{}".format(script), ], - "environment": rgm_env_secrets, + "environment": rgm_env_secrets(env), # The docker socket is a requirement for running dagger programs # In the future we should find a way to use dagger without mounting the docker socket. "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], } - if canFail: - rgm_build_step["failure"] = "ignore" return [ - rgm_build_step, + rgm_run_step, ] -def rgm_main(): - trigger = { - "event": [ - "push", - ], - "branch": "main", - "paths": docs_paths, - "repo": [ - "grafana/grafana", - ], +def rgm_copy(src, dst): + """Copies file from/to GCS. + + Args: + src: source of the files. + dst: destination of the files. + + Returns: + Drone steps. + """ + commands = [ + "printenv GCP_KEY_BASE64 | base64 -d > /tmp/key.json", + "gcloud auth activate-service-account --key-file=/tmp/key.json", + "gcloud storage cp -r {} {}".format(src, dst), + ] + + if not dst.startswith("gs://"): + commands.insert(0, "mkdir -p {}".format(dst)) + + rgm_copy_step = { + "name": "rgm-copy", + "image": "google/cloud-sdk:alpine", + "commands": commands, + "environment": rgm_env_secrets({}), } + return [ + rgm_copy_step, + ] + +def rgm_publish_packages(bucket = "grafana-packages"): + """Publish deb and rpm packages. + + Args: + bucket: target bucket to publish the packages. + + Returns: + Drone steps. + """ + steps = [] + for package_manager in ["deb", "rpm"]: + steps.append({ + "name": "publish-{}".format(package_manager), + # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image + "image": images["package_publish"], + "privileged": True, + "settings": { + "access_key_id": from_secret("packages_access_key_id"), + "secret_access_key": from_secret("packages_secret_access_key"), + "service_account_json": from_secret("packages_service_account"), + "target_bucket": bucket, + "gpg_passphrase": from_secret("packages_gpg_passphrase"), + "gpg_public_key": from_secret("packages_gpg_public_key"), + "gpg_private_key": from_secret("packages_gpg_private_key"), + "package_path": "file:///drone/src/dist/*.{}".format(package_manager), + }, + }) + + return steps + +def rgm_main(): + # Runs a package / build process (with some distros) when commits are merged to main return pipeline( name = "rgm-main-prerelease", - trigger = trigger, - steps = rgm_build(canFail = True), + trigger = main_trigger, + steps = rgm_run("rgm-build", "drone_publish_main.sh"), depends_on = ["main-test-backend", "main-test-frontend"], ) def rgm_tag(): + # Runs a package / build process (with all distros) when a tag is made return pipeline( name = "rgm-tag-prerelease", trigger = tag_trigger, - steps = rgm_build(script = "drone_publish_tag_grafana.sh", canFail = False), + steps = rgm_run("rgm-build", "drone_publish_tag_grafana.sh"), depends_on = ["release-test-backend", "release-test-frontend"], ) -def rgm_windows(): +def rgm_tag_windows(): return pipeline( name = "rgm-tag-prerelease-windows", trigger = tag_trigger, - steps = ignore_failure( - get_windows_steps( - ver_mode = "release", - bucket = "grafana-prerelease", - ), + steps = get_windows_steps( + ver_mode = "release", + bucket = "grafana-prerelease", ), depends_on = ["rgm-tag-prerelease"], platform = "windows", ) -def rgm(): +def rgm_version_branch(): + # Runs a package / build proces (with all distros) when a commit lands on a version branch + return pipeline( + name = "rgm-version-branch-prerelease", + trigger = version_branch_trigger, + steps = rgm_run("rgm-build", "drone_publish_tag_grafana.sh"), + depends_on = ["release-test-backend", "release-test-frontend"], + ) + +def rgm_nightly_build(): + src = "$${DRONE_WORKSPACE}/dist/*" + dst = "$${DESTINATION}/$${DRONE_BUILD_EVENT}" + + copy_steps = with_deps(rgm_copy(src, dst), ["rgm-build"]) + + return pipeline( + name = "rgm-nightly-build", + trigger = nightly_trigger, + steps = rgm_run("rgm-build", "drone_build_nightly_grafana.sh") + copy_steps, + depends_on = ["nightly-test-backend", "nightly-test-frontend"], + ) + +def rgm_nightly_publish(): + """Nightly publish pipeline. + + Returns: + Drone pipeline. + """ + src = "$${DESTINATION}/$${DRONE_BUILD_EVENT}/*_$${DRONE_BUILD_NUMBER}_*" + dst = "$${DRONE_WORKSPACE}/dist" + + publish_steps = with_deps(rgm_run("rgm-publish", "drone_publish_nightly_grafana.sh"), ["rgm-copy"]) + package_steps = with_deps(rgm_publish_packages(), ["rgm-publish"]) + + return pipeline( + name = "rgm-nightly-publish", + trigger = nightly_trigger, + steps = rgm_copy(src, dst) + publish_steps + package_steps, + depends_on = ["rgm-nightly-build"], + ) + +def rgm_nightly_pipeline(): + return [ + test_frontend(nightly_trigger, "nightly"), + test_backend(nightly_trigger, "nightly"), + rgm_nightly_build(), + rgm_nightly_publish(), + ] + +def rgm_tag_pipeline(): return [ whats_new_checker_pipeline(tag_trigger), test_frontend(tag_trigger, "release"), test_backend(tag_trigger, "release"), - rgm_main(), rgm_tag(), - rgm_windows(), + rgm_tag_windows(), verify_release_pipeline( trigger = tag_trigger, name = "rgm-tag-verify-prerelease-assets", @@ -152,3 +312,29 @@ def rgm(): ], ), ] + +def rgm_version_branch_pipeline(): + return [ + rgm_version_branch(), + verify_release_pipeline( + trigger = version_branch_trigger, + name = "rgm-prerelease-verify-prerelease-assets", + bucket = "grafana-prerelease", + depends_on = [ + "rgm-version-branch-prerelease", + ], + ), + ] + +def rgm_main_pipeline(): + return [ + rgm_main(), + ] + +def rgm(): + return ( + rgm_main_pipeline() + + rgm_tag_pipeline() + + rgm_version_branch_pipeline() + + rgm_nightly_pipeline() + ) diff --git a/scripts/drone/services/services.star b/scripts/drone/services/services.star index 827eb161200..911aacfb000 100644 --- a/scripts/drone/services/services.star +++ b/scripts/drone/services/services.star @@ -10,14 +10,15 @@ load( def integration_test_services_volumes(): return [ {"name": "postgres", "temp": {"medium": "memory"}}, - {"name": "mysql", "temp": {"medium": "memory"}}, + {"name": "mysql57", "temp": {"medium": "memory"}}, + {"name": "mysql80", "temp": {"medium": "memory"}}, ] -def integration_test_services(edition): +def integration_test_services(): services = [ { "name": "postgres", - "image": images["postgres_alpine_image"], + "image": images["postgres_alpine"], "environment": { "POSTGRES_USER": "grafanatest", "POSTGRES_PASSWORD": "grafanatest", @@ -29,40 +30,53 @@ def integration_test_services(edition): ], }, { - "name": "mysql", - "image": images["mysql5_image"], + "name": "mysql57", + "image": images["mysql5"], "environment": { "MYSQL_ROOT_PASSWORD": "rootpass", "MYSQL_DATABASE": "grafana_tests", "MYSQL_USER": "grafana", "MYSQL_PASSWORD": "password", }, - "volumes": [{"name": "mysql", "path": "/var/lib/mysql"}], + "volumes": [{"name": "mysql57", "path": "/var/lib/mysql"}], + "commands": ["docker-entrypoint.sh mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci"], + }, + { + "name": "mysql80", + "image": images["mysql8"], + "environment": { + "MYSQL_ROOT_PASSWORD": "rootpass", + "MYSQL_DATABASE": "grafana_tests", + "MYSQL_USER": "grafana", + "MYSQL_PASSWORD": "password", + }, + "volumes": [{"name": "mysql80", "path": "/var/lib/mysql"}], + "commands": ["docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password"], + }, + { + "name": "mimir_backend", + "image": images["mimir"], + "environment": {}, + "commands": ["/bin/mimir -target=backend"], + }, + { + "name": "redis", + "image": images["redis_alpine"], + "environment": {}, + }, + { + "name": "memcached", + "image": images["memcached_alpine"], + "environment": {}, }, ] - if edition in ("enterprise", "enterprise2"): - services.extend( - [ - { - "name": "redis", - "image": "redis:6.2.1-alpine", - "environment": {}, - }, - { - "name": "memcached", - "image": "memcached:1.6.9-alpine", - "environment": {}, - }, - ], - ) - return services def ldap_service(): return { "name": "ldap", - "image": images["openldap_image"], + "image": images["openldap"], "environment": { "LDAP_ADMIN_PASSWORD": "grafana", "LDAP_DOMAIN": "grafana.org", diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index e5f22d7602c..c723d84dd61 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -2,6 +2,18 @@ This module is a library of Drone steps and other pipeline components. """ +load( + "scripts/drone/steps/rgm.star", + "rgm_build_backend_step", +) +load( + "scripts/drone/utils/images.star", + "images", +) +load( + "scripts/drone/variables.star", + "grabpl_version", +) load( "scripts/drone/vault.star", "from_secret", @@ -11,16 +23,6 @@ load( "npm_token", "prerelease_bucket", ) -load( - "scripts/drone/utils/images.star", - "images", -) -load( - "scripts/drone/utils/windows_images.star", - "windows_images", -) - -grabpl_version = "v3.0.41" trigger_oss = { "repo": [ @@ -28,21 +30,10 @@ trigger_oss = { ], } -def slack_step(channel, template, secret): - return { - "name": "slack", - "image": images["plugins_slack_image"], - "settings": { - "webhook": from_secret(secret), - "channel": channel, - "template": template, - }, - } - def yarn_install_step(): return { "name": "yarn-install", - "image": images["build_image"], + "image": images["node"], "commands": [ "yarn install --immutable", ], @@ -52,8 +43,9 @@ def yarn_install_step(): def wire_install_step(): return { "name": "wire-install", - "image": images["build_image"], + "image": images["go"], "commands": [ + "apk add --update make", "make gen-go", ], "depends_on": [ @@ -61,37 +53,15 @@ def wire_install_step(): ], } -def windows_wire_install_step(edition): +def identify_runner_step(): return { - "name": "wire-install", - "image": windows_images["windows_go_image"], + "name": "identify-runner", + "image": images["alpine"], "commands": [ - "go install github.com/google/wire/cmd/wire@v0.5.0", - "wire gen -tags {} ./pkg/server".format(edition), - ], - "depends_on": [ - "windows-init", + "echo $DRONE_RUNNER_NAME", ], } -def identify_runner_step(platform = "linux"): - if platform == "linux": - return { - "name": "identify-runner", - "image": images["alpine_image"], - "commands": [ - "echo $DRONE_RUNNER_NAME", - ], - } - else: - return { - "name": "identify-runner", - "image": windows_images["1809_image"], - "commands": [ - "echo $env:DRONE_RUNNER_NAME", - ], - } - def enterprise_setup_step(source = "${DRONE_SOURCE_BRANCH}", canFail = True, isPromote = False): """Setup the enterprise source into the ./grafana-enterprise directory. @@ -135,11 +105,12 @@ def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFai step = { "name": "clone-enterprise", - "image": images["build_image"], + "image": images["git"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, "commands": [ + "apk add --update curl jq bash", ] + check + [ 'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" ' + location, "cd {}".format(location), @@ -152,22 +123,10 @@ def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFai return step -def download_grabpl_step(platform = "linux"): - if platform == "windows": - return { - "name": "grabpl", - "image": windows_images["wix_image"], - "commands": [ - '$$ProgressPreference = "SilentlyContinue"', - "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( - grabpl_version, - ), - ], - } - +def download_grabpl_step(): return { "name": "grabpl", - "image": images["curl_image"], + "image": images["curl"], "commands": [ "mkdir -p bin", "curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/grabpl".format( @@ -180,7 +139,7 @@ def download_grabpl_step(platform = "linux"): def lint_drone_step(): return { "name": "lint-drone", - "image": images["curl_image"], + "image": images["curl"], "commands": [ "./bin/build verify-drone", ], @@ -192,9 +151,10 @@ def lint_drone_step(): def lint_starlark_step(): return { "name": "lint-starlark", - "image": images["build_image"], + "image": images["go"], "commands": [ - "./bin/build verify-starlark .", + "go install github.com/bazelbuild/buildtools/buildifier@latest", + "buildifier --lint=warn -mode=check -r .", ], "depends_on": [ "compile-build-cmd", @@ -219,7 +179,7 @@ def enterprise_downstream_step(ver_mode): step = { "name": "trigger-enterprise-downstream", - "image": images["drone_downstream_image"], + "image": images["drone_downstream"], "settings": { "server": "https://drone.grafana.net", "token": from_secret("drone_token"), @@ -242,8 +202,7 @@ def enterprise_downstream_step(ver_mode): def lint_backend_step(): return { "name": "lint-backend", - # TODO: build_image or go_image? - "image": images["go_image"], + "image": images["go"], "environment": { # We need CGO because of go-sqlite3 "CGO_ENABLED": "1", @@ -252,32 +211,47 @@ def lint_backend_step(): "wire-install", ], "commands": [ - "apt-get update && apt-get install make", + "apk add --update make build-base", # Don't use Make since it will re-download the linters "make lint-go", ], } -def benchmark_ldap_step(): +def validate_modfile_step(): return { - "name": "benchmark-ldap", - "image": images["build_image"], - "environment": { - "LDAP_HOSTNAME": "ldap", - }, + "name": "validate-modfile", + "image": images["go"], "commands": [ - "dockerize -wait tcp://ldap:389 -timeout 120s", - 'go test -benchmem -run=^$ ./pkg/extensions/ldapsync -bench "^(Benchmark50Users)$"', + "go run scripts/modowners/modowners.go check go.mod", + ], + } + +def validate_openapi_spec_step(): + return { + "name": "validate-openapi-spec", + "image": images["go"], + "commands": [ + "apk add --update make", + "make swagger-validate", + ], + } + +def dockerize_step(name, hostname, port): + return { + "name": name, + "image": images["dockerize"], + "commands": [ + "dockerize -wait tcp://{}:{} -timeout 120s".format(hostname, port), ], } def build_storybook_step(ver_mode): return { "name": "build-storybook", - "image": images["build_image"], + "image": images["node"], "depends_on": [ # Best to ensure that this step doesn't mess with what's getting built and packaged - "build-frontend", + "rgm-package", "build-frontend-packages", ], "environment": { @@ -318,7 +292,7 @@ def store_storybook_step(ver_mode, trigger = None): step = { "name": "store-storybook", - "image": images["publish_image"], + "image": images["publish"], "depends_on": [ "build-storybook", ] + @@ -348,7 +322,7 @@ def store_storybook_step(ver_mode, trigger = None): def e2e_tests_artifacts(): return { "name": "e2e-tests-artifacts-upload", - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "depends_on": [ "end-to-end-tests-dashboards-suite", "end-to-end-tests-panels-suite", @@ -396,7 +370,7 @@ def upload_cdn_step(ver_mode, trigger = None): step = { "name": "upload-cdn-assets", - "image": images["publish_image"], + "image": images["publish"], "depends_on": [ "grafana-server", ], @@ -412,75 +386,27 @@ def upload_cdn_step(ver_mode, trigger = None): step = dict(step, when = trigger) return step -def build_backend_step(ver_mode, variants = None): +def build_backend_step(distros = "linux/amd64,linux/arm64"): """Build the backend code using the Grafana build tool. Args: - ver_mode: if ver_mode != 'release', pass the DRONE_BUILD_NUMBER environment - variable as the value for the --build-id option. - TODO: is this option actually used by the build-backend subcommand? - variants: a list of variants be passed to the build-backend subcommand - using the --variants option. - Defaults to None. + distros: a list of distributes to be built. For a full list, see `go tool dist list`. Returns: Drone step. """ - variants_str = "" - if variants: - variants_str = " --variants {}".format(",".join(variants)) - # TODO: Convert number of jobs to percentage - if ver_mode == "release": - cmds = [ - "./bin/build build-backend --jobs 8 --edition oss ${DRONE_TAG}", - ] - else: - build_no = "${DRONE_BUILD_NUMBER}" - cmds = [ - "./bin/build build-backend --jobs 8 --edition oss --build-id {}{}".format( - build_no, - variants_str, - ), - ] + return rgm_build_backend_step(distros) - return { - "name": "build-backend", - "image": images["build_image"], - "depends_on": [ - "wire-install", - "compile-build-cmd", - ], - "commands": cmds, - } - -def build_frontend_step(ver_mode): - """Build the frontend code using the Grafana build tool. - - Args: - ver_mode: if ver_mode != 'release', use the DRONE_BUILD_NUMBER environment - variable as a build identifier. +def build_frontend_step(): + """Build the frontend code to ensure it's compilable Returns: Drone step. """ - build_no = "${DRONE_BUILD_NUMBER}" - - # TODO: Use percentage for num jobs - if ver_mode == "release": - cmds = [ - "./bin/build build-frontend --jobs 8 " + - "--edition oss ${DRONE_TAG}", - ] - else: - cmds = [ - "./bin/build build-frontend --jobs 8 --edition oss" + - "--build-id {}".format(build_no), - ] - return { "name": "build-frontend", - "image": images["build_image"], + "image": images["node"], "environment": { "NODE_OPTIONS": "--max_old_space_size=8192", }, @@ -488,43 +414,60 @@ def build_frontend_step(ver_mode): "compile-build-cmd", "yarn-install", ], - "commands": cmds, + "commands": [ + "yarn build", + ], } -def build_frontend_package_step(ver_mode): +def update_package_json_version(): + """Updates the packages/ to use a version that has the build ID in it: 10.0.0pre -> 10.0.0-5432pre + + Returns: + Drone step that updates the 'version' key in package.json + """ + + return { + "name": "update-package-json-version", + "image": images["node"], + "depends_on": [ + "yarn-install", + ], + "commands": [ + "apk add --update jq", + "new_version=$(cat package.json | jq .version | sed s/pre/${DRONE_BUILD_NUMBER}/g)", + "echo \"New version: $new_version\"", + "yarn run lerna version $new_version --exact --no-git-tag-version --no-push --force-publish -y", + "yarn install --mode=update-lockfile", + ], + } + +def build_frontend_package_step(depends_on = []): """Build the frontend packages using the Grafana build tool. Args: - ver_mode: if ver_mode != 'release', use the DRONE_BUILD_NUMBER environment - variable as a build identifier. + depends_on: a list of step names (strings) that must complete before this step runs. Returns: Drone step. """ - build_no = "${DRONE_BUILD_NUMBER}" - # TODO: Use percentage for num jobs - if ver_mode == "release": - cmds = [ - "./bin/build build-frontend-packages --jobs 8 " + - "--edition oss ${DRONE_TAG}", - ] - else: - cmds = [ - "./bin/build build-frontend-packages --jobs 8 --edition oss" + - "--build-id {}".format(build_no), - ] + cmds = [ + "apk add --update jq bash", # bash is needed for the validate-npm-packages.sh script since it has a 'bash' + # shebang. + "yarn packages:build", + "yarn packages:pack", + "./scripts/validate-npm-packages.sh", + ] return { "name": "build-frontend-packages", - "image": images["build_image"], + "image": images["node"], "environment": { "NODE_OPTIONS": "--max_old_space_size=8192", }, "depends_on": [ - "compile-build-cmd", "yarn-install", - ], + ] + depends_on, "commands": cmds, } @@ -537,43 +480,42 @@ def build_plugins_step(ver_mode): env = None return { "name": "build-plugins", - "image": images["build_image"], + "image": images["node"], "environment": env, "depends_on": [ - "compile-build-cmd", "yarn-install", ], "commands": [ - # TODO: Use percentage for num jobs - "./bin/build build-plugins --jobs 8 --edition oss", + "apk add --update findutils", # Replaces the busybox 'find' with the GNU one. + "yarn plugins:build", ], } -def test_backend_step(image = images["build_image"]): +def test_backend_step(): return { "name": "test-backend", - "image": image, + "image": images["go"], "depends_on": [ "wire-install", ], "commands": [ + # shared-mime-info and shared-mime-info-lang is used for exactly 1 test for the + # mime.TypeByExtension function. + "apk add --update build-base shared-mime-info shared-mime-info-lang", "go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/...", ], } -def windows_test_backend_step(): - step = test_backend_step(image = windows_images["windows_go_image"]) - return step - def test_backend_integration_step(): return { "name": "test-backend-integration", - "image": images["build_image"], + "image": images["go"], "depends_on": [ "wire-install", ], "commands": [ - "go test -run Integration -covermode=atomic -timeout=5m ./pkg/...", + "apk add --update build-base", + "go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\\(.*\\)/' | sort -u)", ], } @@ -586,11 +528,12 @@ def betterer_frontend_step(): return { "name": "betterer-frontend", - "image": images["build_image"], + "image": images["node"], "depends_on": [ "yarn-install", ], "commands": [ + "apk add --update git bash", "yarn betterer ci", ], } @@ -604,7 +547,7 @@ def test_frontend_step(): return { "name": "test-frontend", - "image": images["build_image"], + "image": images["node"], "environment": { "TEST_MAX_WORKERS": "50%", }, @@ -619,7 +562,7 @@ def test_frontend_step(): def lint_frontend_step(): return { "name": "lint-frontend", - "image": images["build_image"], + "image": images["node"], "environment": { "TEST_MAX_WORKERS": "50%", }, @@ -629,11 +572,35 @@ def lint_frontend_step(): "commands": [ "yarn run prettier:check", "yarn run lint", - "yarn run i18n:compile", # TODO: right place for this? "yarn run typecheck", ], } +def verify_i18n_step(): + extract_error_message = "\nExtraction failed. Make sure that you have no dynamic translation phrases, such as 't(\\`preferences.theme.\\$${themeID}\\`, themeName)' and that no translation key is used twice. Search the output for '[warning]' to find the offending file." + uncommited_error_message = "\nTranslation extraction has not been committed. Please run 'yarn i18n:extract', commit the changes and push again." + return { + "name": "verify-i18n", + "image": images["node"], + "depends_on": [ + "yarn-install", + ], + "commands": [ + "apk add --update git", + "yarn run i18n:extract || (echo \"{}\" && false)".format(extract_error_message), + # Verify that translation extraction has been committed + ''' + file_diff=$(git diff --dirstat public/locales) + if [ -n "$file_diff" ]; then + echo $file_diff + echo "{}" + exit 1 + fi + '''.format(uncommited_error_message), + "yarn run i18n:compile", + ], + } + def test_a11y_frontend_step(ver_mode, port = 3001): """Runs automated accessiblity tests against the frontend. @@ -667,7 +634,7 @@ def test_a11y_frontend_step(ver_mode, port = 3001): return { "name": "test-a11y-frontend", # TODO which image should be used? - "image": images["docker_puppeteer_image"], + "image": images["docker_puppeteer"], "depends_on": [ "grafana-server", ], @@ -692,7 +659,7 @@ def frontend_metrics_step(trigger = None): """ step = { "name": "publish-frontend-metrics", - "image": images["build_image"], + "image": images["node"], "depends_on": [ "test-a11y-frontend", ], @@ -701,7 +668,8 @@ def frontend_metrics_step(trigger = None): }, "failure": "ignore", "commands": [ - "./scripts/ci-frontend-metrics.sh | ./bin/build publish-metrics $${GRAFANA_MISC_STATS_API_KEY}", + "apk add --update bash grep git", + "./scripts/ci-frontend-metrics.sh ./grafana/public/build | ./bin/build publish-metrics $$GRAFANA_MISC_STATS_API_KEY", ], } if trigger: @@ -711,95 +679,38 @@ def frontend_metrics_step(trigger = None): def codespell_step(): return { "name": "codespell", - "image": images["build_image"], + "image": images["python"], "commands": [ - # Important: all words have to be in lowercase, and separated by "\n". - 'echo -e "unknwon\nreferer\nerrorstring\neror\niam\nwan" > words_to_ignore.txt', - "codespell -I words_to_ignore.txt docs/", - "rm words_to_ignore.txt", + "pip3 install codespell", + "codespell -I .codespellignore docs/", ], } -def package_step(ver_mode): - """Packages Grafana with the Grafana build tool. - - Args: - ver_mode: controls whether the packages are signed for a release. - If ver_mode != 'release', use the DRONE_BUILD_NUMBER environment - variable as a build identifier. - - Returns: - Drone step. - """ - deps = [ - "build-plugins", - "build-backend", - "build-frontend", - "build-frontend-packages", - ] - - if ver_mode in ("main", "release", "release-branch"): - sign_args = " --sign" - env = { - "GRAFANA_API_KEY": from_secret("grafana_api_key"), - "GPG_PRIV_KEY": from_secret("packages_gpg_private_key"), - "GPG_PUB_KEY": from_secret("packages_gpg_public_key"), - "GPG_KEY_PASSWORD": from_secret("packages_gpg_passphrase"), - } - test_args = "" - else: - sign_args = "" - env = None - - # TODO: env vars no longer needed by build if not signing - test_args = ". scripts/build/gpg-test-vars.sh && " - - # TODO: Use percentage for jobs - if ver_mode == "release": - cmds = [ - "{}./bin/build package --jobs 8 --edition oss ".format(test_args) + - "{} $${{DRONE_TAG}}".format(sign_args), - ] - else: - build_no = "${DRONE_BUILD_NUMBER}" - cmds = [ - "{}./bin/build package --jobs 8 --edition oss ".format(test_args) + - "--build-id {}{}".format(build_no, sign_args), - ] - - return { - "name": "package", - "image": images["build_image"], - "depends_on": deps, - "environment": env, - "commands": cmds, - } - -def grafana_server_step(port = 3001): +def grafana_server_step(): """Runs the grafana-server binary as a service. - Args: - port: port to listen on. - Defaults to 3001. - Returns: Drone step. """ - environment = {"PORT": port, "ARCH": "linux-amd64"} + environment = { + "GF_SERVER_HTTP_PORT": "3001", + "GF_SERVER_ROUTER_LOGGING": "1", + "GF_APP_MODE": "development", + } return { "name": "grafana-server", - "image": images["build_image"], + "image": images["alpine"], "detach": True, "depends_on": [ - "build-plugins", - "build-backend", - "build-frontend", - "build-frontend-packages", + "rgm-package", ], "environment": environment, "commands": [ - "./scripts/grafana-server/start-server", + "apk add --update tar bash", + "mkdir grafana", + "tar --strip-components=1 -xvf ./dist/*amd64.tar.gz -C grafana", + "cp -r devenv scripts tools grafana && cd grafana && ./scripts/grafana-server/start-server", ], } @@ -809,7 +720,7 @@ def e2e_tests_step(suite, port = 3001, tries = None): cmd += " --tries {}".format(tries) return { "name": "end-to-end-tests-{}".format(suite), - "image": images["cypress_image"], + "image": images["cypress"], "depends_on": [ "grafana-server", ], @@ -817,7 +728,6 @@ def e2e_tests_step(suite, port = 3001, tries = None): "HOST": "grafana-server", }, "commands": [ - "apt-get install -y netcat", cmd, ], } @@ -853,7 +763,7 @@ def cloud_plugins_e2e_tests_step(suite, cloud, trigger = None): paths = { "include": [ "pkg/tsdb/azuremonitor/**", - "public/app/plugins/datasource/grafana-azure-monitor-datasource/**", + "public/app/plugins/datasource/azuremonitor/**", "e2e/cloud-plugins-suite/azure-monitor.spec.ts", ], }, @@ -861,7 +771,7 @@ def cloud_plugins_e2e_tests_step(suite, cloud, trigger = None): branch = "${DRONE_SOURCE_BRANCH}".replace("/", "-") step = { "name": "end-to-end-tests-{}-{}".format(suite, cloud), - "image": images["cloud_datasources_e2e_image"], + "image": "us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:3.0.0", "depends_on": [ "grafana-server", ], @@ -875,73 +785,19 @@ def build_docs_website_step(): return { "name": "build-docs-website", # Use latest revision here, since we want to catch if it breaks - "image": images["docs_image"], + "image": images["docs"], "commands": [ - "mkdir -p /hugo/content/docs/grafana", + "mkdir -p /hugo/content/docs/grafana/latest", + "echo -e '---\\nredirectURL: /docs/grafana/latest/\\ntype: redirect\\nversioned: true\\n---\\n' > /hugo/content/docs/grafana/_index.md", "cp -r docs/sources/* /hugo/content/docs/grafana/latest/", "cd /hugo && make prod", ], } -def copy_packages_for_docker_step(): - return { - "name": "copy-packages-for-docker", - "image": images["build_image"], - "depends_on": [ - "package", - ], - "commands": [ - "ls dist/*.tar.gz*", - "cp dist/*.tar.gz* packaging/docker/", - ], - } - -def build_docker_images_step(archs = None, ubuntu = False, publish = False): - """Build Docker images using the Grafana build tool. - - Args: - archs: a list of architectures to build the image for. - Defaults to None. - ubuntu: controls whether the final image is built from an Ubuntu base image. - Defaults to False. - publish: controls whether the built image is saved to a pre-release repository. - Defaults to False. - - Returns: - Drone step. - """ - cmd = "./bin/build build-docker --edition oss" - if publish: - cmd += " --shouldSave" - - ubuntu_sfx = "" - if ubuntu: - ubuntu_sfx = "-ubuntu" - cmd += " --ubuntu" - - if archs: - cmd += " -archs {}".format(",".join(archs)) - - environment = { - "GCP_KEY": from_secret(gcp_grafanauploads), - } - - return { - "name": "build-docker-images" + ubuntu_sfx, - "image": images["cloudsdk_image"], - "depends_on": [ - "copy-packages-for-docker", - "compile-build-cmd", - ], - "commands": [cmd], - "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], - "environment": environment, - } - def fetch_images_step(): return { "name": "fetch-images", - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "environment": { "GCP_KEY": from_secret(gcp_grafanauploads), "DOCKER_USER": from_secret("docker_username"), @@ -982,7 +838,7 @@ def publish_images_step(ver_mode, docker_repo, trigger = None): docker_repo, ) - deps = ["build-docker-images", "build-docker-images-ubuntu"] + deps = ["rgm-build-docker"] if ver_mode == "release": deps = ["fetch-images"] cmd += " --version-tag ${DRONE_TAG}" @@ -998,7 +854,7 @@ def publish_images_step(ver_mode, docker_repo, trigger = None): step = { "name": "publish-images-{}".format(name), - "image": images["cloudsdk_image"], + "image": images["cloudsdk"], "environment": environment, "commands": [cmd], "depends_on": deps, @@ -1011,80 +867,126 @@ def publish_images_step(ver_mode, docker_repo, trigger = None): return step -def postgres_integration_tests_step(): +def integration_tests_steps(name, cmds, hostname = None, port = None, environment = None): + """Integration test steps + + Args: + name: the name of the step. + cmds: the commands to run to perform the integration tests. + hostname: the hostname where the remote server is available. + port: the port where the remote server is available. + environment: Any extra environment variables needed to run the integration tests. + + Returns: + A list of drone steps. If a hostname / port were provided, then a step to wait for the remove server to be + available is also returned. + """ + dockerize_name = "wait-for-{}".format(name) + + depends = [ + "wire-install", + ] + + step = { + "name": "{}-integration-tests".format(name), + "image": images["go"], + "depends_on": depends, + "commands": [ + "apk add --update build-base", + ] + cmds, + } + + if environment: + step["environment"] = environment + + if hostname == None: + return [step] + + depends = depends.append(dockerize_name) + + return [ + dockerize_step(dockerize_name, hostname, port), + step, + ] + +def integration_benchmarks_step(name, environment = None): cmds = [ - "apt-get update", - "apt-get install -yq postgresql-client", - "dockerize -wait tcp://postgres:5432 -timeout 120s", + "if [ -z ${GO_PACKAGES} ]; then echo 'missing GO_PACKAGES'; false; fi", + "go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES}", + ] + + return integration_tests_steps("{}-benchmark".format(name), cmds, environment = environment) + +def postgres_integration_tests_steps(): + cmds = [ + "apk add --update postgresql-client", "psql -p 5432 -h postgres -U grafanatest -d grafanatest -f " + "devenv/docker/blocks/postgres_tests/setup.sql", - # Make sure that we don't use cached results for another database "go clean -testcache", - "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=5m {}'", + "go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\\(.*\\)/' | sort -u)", ] - return { - "name": "postgres-integration-tests", - "image": images["build_image"], - "depends_on": ["wire-install"], - "environment": { - "PGPASSWORD": "grafanatest", - "GRAFANA_TEST_DB": "postgres", - "POSTGRES_HOST": "postgres", - }, - "commands": cmds, + + environment = { + "PGPASSWORD": "grafanatest", + "GRAFANA_TEST_DB": "postgres", + "POSTGRES_HOST": "postgres", } -def mysql_integration_tests_step(): + return integration_tests_steps("postgres", cmds, "postgres", "5432", environment) + +def mysql_integration_tests_steps(hostname, version): cmds = [ - "apt-get update", - "apt-get install -yq default-mysql-client", - "dockerize -wait tcp://mysql:3306 -timeout 120s", - "cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root -prootpass", - # Make sure that we don't use cached results for another database + "apk add --update mysql-client", + "cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h {} -P 3306 -u root -prootpass".format(hostname), "go clean -testcache", - "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=5m {}'", + "go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\\(.*\\)/' | sort -u)", ] - return { - "name": "mysql-integration-tests", - "image": images["build_image"], - "depends_on": ["wire-install"], - "environment": { - "GRAFANA_TEST_DB": "mysql", - "MYSQL_HOST": "mysql", - }, - "commands": cmds, + + environment = { + "GRAFANA_TEST_DB": "mysql", + "MYSQL_HOST": hostname, } -def redis_integration_tests_step(): - return { - "name": "redis-integration-tests", - "image": images["build_image"], - "depends_on": ["wire-install"], - "environment": { - "REDIS_URL": "redis://redis:6379/0", - }, - "commands": [ - "dockerize -wait tcp://redis:6379/0 -timeout 120s", - "go clean -testcache", - "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=5m {}'", - ], + return integration_tests_steps("mysql-{}".format(version), cmds, hostname, "3306", environment) + +def redis_integration_tests_steps(): + cmds = [ + "go clean -testcache", + "go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...", + ] + + environment = { + "REDIS_URL": "redis://redis:6379/0", } -def memcached_integration_tests_step(): - return { - "name": "memcached-integration-tests", - "image": images["build_image"], - "depends_on": ["wire-install"], - "environment": { - "MEMCACHED_HOSTS": "memcached:11211", - }, - "commands": [ - "dockerize -wait tcp://memcached:11211 -timeout 120s", - "go clean -testcache", - "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=5m {}'", - ], + return integration_tests_steps("redis", cmds, "redis", "6379", environment = environment) + +def remote_alertmanager_integration_tests_steps(): + cmds = [ + "go clean -testcache", + "go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/...", + ] + + environment = { + "AM_TENANT_ID": "test", + "AM_PASSWORD": "test", + "AM_URL": "http://mimir_backend:8080", } + return integration_tests_steps("remote-alertmanager", cmds, "mimir_backend", "8080", environment = environment) + +def memcached_integration_tests_steps(): + cmds = [ + "go clean -testcache", + "go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...", + ] + + environment = { + "MEMCACHED_HOSTS": "memcached:11211", + } + + return integration_tests_steps("memcached", cmds, "memcached", "11211", environment) + def release_canary_npm_packages_step(trigger = None): """Releases canary NPM packages. @@ -1097,17 +999,30 @@ def release_canary_npm_packages_step(trigger = None): """ step = { "name": "release-canary-npm-packages", - "image": images["build_image"], - "depends_on": end_to_end_tests_deps(), + "image": images["node"], + "depends_on": end_to_end_tests_deps() + ["build-frontend-packages"], "environment": { "NPM_TOKEN": from_secret(npm_token), }, "commands": [ - "./scripts/circle-release-canary-packages.sh", + "apk add --update bash", + "./scripts/publish-npm-packages.sh --dist-tag 'canary' --registry 'https://registry.npmjs.org'", ], } + if trigger: - step = dict(step, when = trigger) + step = dict( + step, + when = dict( + trigger, + paths = { + "include": [ + "packages/**", + ], + }, + ), + ) + return step def upload_packages_step(ver_mode, trigger = None): @@ -1124,7 +1039,7 @@ def upload_packages_step(ver_mode, trigger = None): """ step = { "name": "upload-packages", - "image": images["publish_image"], + "image": images["publish"], "depends_on": end_to_end_tests_deps(), "environment": { "GCP_KEY": from_secret(gcp_grafanauploads_base64), @@ -1145,7 +1060,7 @@ def publish_grafanacom_step(ver_mode): ver_mode: if ver_mode == 'main', pass the DRONE_BUILD_NUMBER environment variable as the value for the --build-id option. TODO: is this actually used by the grafanacom subcommand? I think it might - just use the environment varaiable directly. + just use the environment variable directly. Returns: Drone step. @@ -1162,7 +1077,7 @@ def publish_grafanacom_step(ver_mode): return { "name": "publish-grafanacom", - "image": images["publish_image"], + "image": images["publish"], "depends_on": [ "publish-linux-packages-deb", "publish-linux-packages-rpm", @@ -1180,7 +1095,7 @@ def publish_linux_packages_step(package_manager = "deb"): return { "name": "publish-linux-packages-{}".format(package_manager), # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image - "image": images["package_publish_image"], + "image": images["package_publish"], "depends_on": ["compile-build-cmd"], "privileged": True, "settings": { @@ -1198,122 +1113,15 @@ def publish_linux_packages_step(package_manager = "deb"): }, } -def windows_clone_step(): - return { - "name": "clone", - "image": windows_images["wix_image"], - "environment": { - "GITHUB_TOKEN": from_secret("github_token"), - }, - "commands": [ - 'git clone "https://$$env:GITHUB_TOKEN@github.com/$$env:DRONE_REPO.git" .', - "git checkout -f $$env:DRONE_COMMIT", - ], - } - -def get_windows_steps(ver_mode, bucket = "%PRERELEASE_BUCKET%"): - """Generate the list of Windows steps. - - Args: - ver_mode: used to differentiate steps for different version modes. - bucket: used to override prerelease bucket. - - Returns: - List of Drone steps. - """ - steps = [ - identify_runner_step("windows"), - ] - - init_cmds = [ - '$$ProgressPreference = "SilentlyContinue"', - "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( - grabpl_version, - ), - ] - - steps.extend( - [ - { - "name": "windows-init", - "image": windows_images["wix_image"], - "commands": init_cmds, - }, - ], - ) - - if ver_mode in ( - "release", - "release-branch", - "main", - ): - gcp_bucket = "{}/artifacts/downloads".format(bucket) - if ver_mode == "release": - ver_part = "${DRONE_TAG}" - dir = "release" - else: - dir = "main" - gcp_bucket = "grafana-downloads" - build_no = "DRONE_BUILD_NUMBER" - ver_part = "--build-id $$env:{}".format(build_no) - installer_commands = [ - "$$gcpKey = $$env:GCP_KEY", - "[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json", - # gcloud fails to read the file unless converted with dos2unix - "dos2unix gcpkey.json", - "gcloud auth activate-service-account --key-file=gcpkey.json", - "rm gcpkey.json", - "cp C:\\App\\nssm-2.24.zip .", - ] - - if ver_mode == "release": - version = "${DRONE_TAG:1}" - installer_commands.extend( - [ - ".\\grabpl.exe windows-installer --target {} --edition oss {}".format("gs://{}/{}/oss/{}/grafana-{}.windows-amd64.zip".format(gcp_bucket, ver_part, ver_mode, version), ver_part), - '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', - 'gsutil cp "$$fname" gs://{}/{}/oss/{}/'.format(gcp_bucket, ver_part, dir), - 'gsutil cp "$$fname.sha256" gs://{}/{}/oss/{}/'.format(gcp_bucket, ver_part, dir), - ], - ) - if ver_mode in ("main"): - installer_commands.extend( - [ - ".\\grabpl.exe windows-installer --edition oss --build-id $$env:DRONE_BUILD_NUMBER", - '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', - 'gsutil cp "$$fname" gs://{}/oss/{}/'.format(gcp_bucket, dir), - 'gsutil cp "$$fname.sha256" gs://{}/oss/{}/'.format( - gcp_bucket, - dir, - ), - ], - ) - steps.append( - { - "name": "build-windows-installer", - "image": windows_images["wix_image"], - "depends_on": [ - "windows-init", - ], - "environment": { - "GCP_KEY": from_secret(gcp_grafanauploads_base64), - "PRERELEASE_BUCKET": from_secret(prerelease_bucket), - "GITHUB_TOKEN": from_secret("github_token"), - }, - "commands": installer_commands, - }, - ) - - return steps - def verify_gen_cue_step(): return { "name": "verify-gen-cue", - "image": images["build_image"], + "image": images["go"], "depends_on": [], "commands": [ "# It is required that code generated from Thema/CUE be committed and in sync with its inputs.", "# The following command will fail if running code generators produces any diff in output.", + "apk add --update make", "CODEGEN_VERIFY=1 make gen-cue", ], } @@ -1321,11 +1129,12 @@ def verify_gen_cue_step(): def verify_gen_jsonnet_step(): return { "name": "verify-gen-jsonnet", - "image": images["build_image"], + "image": images["go"], "depends_on": [], "commands": [ "# It is required that generated jsonnet is committed and in sync with its inputs.", "# The following command will fail if running code generators produces any diff in output.", + "apk add --update make", "CODEGEN_VERIFY=1 make gen-jsonnet", ], } @@ -1343,7 +1152,7 @@ def compile_build_cmd(): return { "name": "compile-build-cmd", - "image": images["go_image"], + "image": images["go"], "commands": [ "go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd", ], @@ -1374,3 +1183,14 @@ def get_trigger_storybook(ver_mode): }, } return trigger_storybook + +def slack_step(channel, template, secret): + return { + "name": "slack", + "image": images["plugins_slack"], + "settings": { + "webhook": from_secret(secret), + "channel": channel, + "template": template, + }, + } diff --git a/scripts/drone/steps/lib_windows.star b/scripts/drone/steps/lib_windows.star new file mode 100644 index 00000000000..f9b11bcf85e --- /dev/null +++ b/scripts/drone/steps/lib_windows.star @@ -0,0 +1,187 @@ +""" +This module is a library of Drone steps that exclusively run on windows machines. +""" + +load( + "scripts/drone/utils/windows_images.star", + "windows_images", +) +load( + "scripts/drone/variables.star", + "grabpl_version", +) +load( + "scripts/drone/vault.star", + "from_secret", + "gcp_grafanauploads_base64", + "prerelease_bucket", +) + +def identify_runner_step_windows(): + return { + "name": "identify-runner", + "image": windows_images["1809"], + "commands": [ + "echo $env:DRONE_RUNNER_NAME", + ], + } + +def get_windows_steps(ver_mode, bucket = "%PRERELEASE_BUCKET%"): + """Generate the list of Windows steps. + + Args: + ver_mode: used to differentiate steps for different version modes. + bucket: used to override prerelease bucket. + + Returns: + List of Drone steps. + """ + steps = [ + identify_runner_step_windows(), + ] + + init_cmds = [ + '$$ProgressPreference = "SilentlyContinue"', + "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( + grabpl_version, + ), + ] + + steps.extend( + [ + { + "name": "windows-init", + "image": windows_images["wix"], + "commands": init_cmds, + }, + ], + ) + + if ver_mode in ( + "release", + "release-branch", + ): + gcp_bucket = "{}/artifacts/downloads".format(bucket) + if ver_mode == "release": + ver_part = "${DRONE_TAG}" + dir = "release" + else: + dir = "main" + gcp_bucket = "grafana-downloads" + build_no = "DRONE_BUILD_NUMBER" + ver_part = "--build-id $$env:{}".format(build_no) + installer_commands = [ + "$$gcpKey = $$env:GCP_KEY", + "[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json", + # gcloud fails to read the file unless converted with dos2unix + "dos2unix gcpkey.json", + "gcloud auth activate-service-account --key-file=gcpkey.json", + "rm gcpkey.json", + "cp C:\\App\\nssm-2.24.zip .", + ] + + if ver_mode in ("release",): + version = "${DRONE_TAG:1}" + installer_commands.extend( + [ + ".\\grabpl.exe windows-installer --target {} --edition oss {}".format( + "gs://{}/{}/oss/{}/grafana-{}.windows-amd64.zip".format(gcp_bucket, ver_part, ver_mode, version), + ver_part, + ), + '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', + ], + ) + if ver_mode == "main": + installer_commands.extend( + [ + "gsutil cp $$fname gs://{}/oss/{}/".format(gcp_bucket, dir), + 'gsutil cp "$$fname.sha256" gs://{}/oss/{}/'.format( + gcp_bucket, + dir, + ), + ], + ) + else: + installer_commands.extend( + [ + "gsutil cp $$fname gs://{}/{}/oss/{}/".format( + gcp_bucket, + ver_part, + dir, + ), + 'gsutil cp "$$fname.sha256" gs://{}/{}/oss/{}/'.format( + gcp_bucket, + ver_part, + dir, + ), + ], + ) + steps.append( + { + "name": "build-windows-installer", + "image": windows_images["wix"], + "depends_on": [ + "windows-init", + ], + "environment": { + "GCP_KEY": from_secret(gcp_grafanauploads_base64), + "PRERELEASE_BUCKET": from_secret(prerelease_bucket), + "GITHUB_TOKEN": from_secret("github_token"), + }, + "commands": installer_commands, + }, + ) + + return steps + +def download_grabpl_step_windows(): + return { + "name": "grabpl", + "image": windows_images["wix"], + "commands": [ + '$$ProgressPreference = "SilentlyContinue"', + "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( + grabpl_version, + ), + ], + } + +def test_backend_step_windows(): + # TODO: This is mostly a duplicate of "test_backend_step" in lib.star; but this file can't import that one, + # otherwise it creates an import cycle. + return { + "name": "test-backend", + "image": windows_images["go"], + "depends_on": [ + "wire-install", + ], + "commands": [ + "go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/...", + ], + } + +def clone_step_windows(): + return { + "name": "clone", + "image": windows_images["wix"], + "environment": { + "GITHUB_TOKEN": from_secret("github_token"), + }, + "commands": [ + 'git clone "https://$$env:GITHUB_TOKEN@github.com/$$env:DRONE_REPO.git" .', + "git checkout -f $$env:DRONE_COMMIT", + ], + } + +def wire_install_step_windows(edition): + return { + "name": "wire-install", + "image": windows_images["go"], + "commands": [ + "go install github.com/google/wire/cmd/wire@v0.5.0", + "wire gen -tags {} ./pkg/server".format(edition), + ], + "depends_on": [ + "windows-init", + ], + } diff --git a/scripts/drone/steps/rgm.star b/scripts/drone/steps/rgm.star new file mode 100644 index 00000000000..495b443e4a0 --- /dev/null +++ b/scripts/drone/steps/rgm.star @@ -0,0 +1,61 @@ +""" +Individual steps that use 'grafana-build' to replace existing individual steps. +These aren't used in releases. +""" + +load( + "scripts/drone/variables.star", + "golang_version", +) + +# rgm_package_step will create a tar.gz for use in e2e tests or other PR testing related activities.. +def rgm_package_step(distros = "linux/amd64,linux/arm64", file = "packages.txt"): + return { + "name": "rgm-package", + "image": "grafana/grafana-build:main", + "pull": "always", + "depends_on": ["yarn-install"], + "commands": [ + "/src/grafana-build package --distro={} ".format(distros) + + "--go-version={} ".format(golang_version) + + "--yarn-cache=$$YARN_CACHE_FOLDER " + + "--build-id=$$DRONE_BUILD_NUMBER " + + "--grafana-dir=$$PWD > {}".format(file), + ], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + } + +# rgm_build_backend will create compile the grafana backend for various platforms. It's preferred to use +# 'rgm_package_step' if you creating a "usable" artifact. This should really only be used to verify that the code is +# compilable. +def rgm_build_backend_step(distros = "linux/amd64,linux/arm64"): + return { + "name": "rgm-package", + "image": "grafana/grafana-build:main", + "pull": "always", + "commands": [ + "/src/grafana-build build " + + "--go-version={} ".format(golang_version) + + "--distro={} --grafana-dir=$$PWD".format(distros), + ], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + } + +def rgm_build_docker_step(packages, ubuntu, alpine, depends_on = ["rgm-package"], file = "docker.txt", tag_format = "{{ .version }}-{{ .arch }}", ubuntu_tag_format = "{{ .version }}-ubuntu-{{ .arch }}"): + return { + "name": "rgm-build-docker", + "image": "grafana/grafana-build:main", + "pull": "always", + "commands": [ + "docker run --privileged --rm tonistiigi/binfmt --install all", + "/src/grafana-build docker " + + "$(cat {} | grep tar.gz | grep -v docker | grep -v sha256 | awk '{{print \"--package=\" $0}}') ".format(packages) + + "--ubuntu-base={} ".format(ubuntu) + + "--alpine-base={} ".format(alpine) + + "--tag-format='{}' ".format(tag_format) + + "--ubuntu-tag-format='{}' > {}".format(ubuntu_tag_format, file), + "find ./dist -name '*docker*.tar.gz' -type f | xargs -n1 docker load -i", + ], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + "depends_on": depends_on, + } diff --git a/scripts/drone/utils/images.star b/scripts/drone/utils/images.star index dee4eef652c..7f5b40c0e1e 100644 --- a/scripts/drone/utils/images.star +++ b/scripts/drone/utils/images.star @@ -2,24 +2,35 @@ This module contains all the docker images that are used to build test and publish Grafana. """ +load( + "scripts/drone/variables.star", + "golang_version", + "nodejs_version", +) + images = { - "cloudsdk_image": "google/cloud-sdk:431.0.0", - "build_image": "grafana/build-container:1.7.4", - "publish_image": "grafana/grafana-ci-deploy:1.3.3", - "alpine_image": "alpine:3.18.3", - "curl_image": "byrnedo/alpine-curl:0.1.8", - "go_image": "golang:1.20.10", - "plugins_slack_image": "plugins/slack", - "postgres_alpine_image": "postgres:12.3-alpine", - "mysql5_image": "mysql:5.7.39", - "mysql8_image": "mysql:8.0.32", - "redis_alpine_image": "redis:6.2.11-alpine", - "memcached_alpine_image": "memcached:1.6.9-alpine", - "package_publish_image": "us.gcr.io/kubernetes-dev/package-publish:latest", - "openldap_image": "osixia/openldap:1.4.0", - "drone_downstream_image": "grafana/drone-downstream", - "docker_puppeteer_image": "grafana/docker-puppeteer:1.1.0", - "docs_image": "grafana/docs-base:dbd975af06", - "cypress_image": "cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97", - "cloud_datasources_e2e_image": "us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest", + "git": "alpine/git:2.40.1", + "go": "golang:{}-alpine".format(golang_version), + "node": "node:{}-alpine".format(nodejs_version), + "cloudsdk": "google/cloud-sdk:431.0.0", + "publish": "grafana/grafana-ci-deploy:1.3.3", + "alpine": "alpine:3.18.3", + "ubuntu": "ubuntu:22.04", + "curl": "byrnedo/alpine-curl:0.1.8", + "plugins_slack": "plugins/slack", + "python": "python:3.8", + "postgres_alpine": "postgres:12.3-alpine", + "mimir": "grafana/mimir:latest", + "mysql5": "mysql:5.7.39", + "mysql8": "mysql:8.0.32", + "redis_alpine": "redis:6.2.11-alpine", + "memcached_alpine": "memcached:1.6.9-alpine", + "package_publish": "us.gcr.io/kubernetes-dev/package-publish:latest", + "openldap": "osixia/openldap:1.4.0", + "drone_downstream": "grafana/drone-downstream", + "docker_puppeteer": "grafana/docker-puppeteer:1.1.0", + "docs": "grafana/docs-base:dbd975af06", + "cypress": "cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97", + "dockerize": "jwilder/dockerize:0.6.1", + "shellcheck": "koalaman/shellcheck:stable", } diff --git a/scripts/drone/utils/utils.star b/scripts/drone/utils/utils.star index 63961a1d3ca..aea832018cf 100644 --- a/scripts/drone/utils/utils.star +++ b/scripts/drone/utils/utils.star @@ -9,7 +9,6 @@ load( load("scripts/drone/vault.star", "pull_secret") failure_template = "Build {{build.number}} failed for commit: : {{build.link}}\nBranch: \nAuthor: {{build.author}}" -drone_change_template = "`.drone.yml` and `starlark` files have been changed on the OSS repo, by: {{build.author}}. \nBranch: \nCommit hash: " def pipeline( name, @@ -31,7 +30,7 @@ def pipeline( name: controls the pipeline name. trigger: a Drone trigger for the pipeline. steps: the Drone steps for the pipeline. - services: auxilliary services used during the pipeline. + services: auxiliary services used during the pipeline. Defaults to []. platform: abstracts platform specific configuration primarily for different Drone behavior on Windows. Defaults to 'linux'. diff --git a/scripts/drone/utils/windows_images.star b/scripts/drone/utils/windows_images.star index 2a605727ba8..a25c637a11e 100644 --- a/scripts/drone/utils/windows_images.star +++ b/scripts/drone/utils/windows_images.star @@ -4,9 +4,14 @@ All the windows images needed to be in a different file than the other images, s by trivy. Related issue: https://github.com/aquasecurity/trivy/issues/1392 """ +load( + "scripts/drone/variables.star", + "golang_version", +) + windows_images = { - "1809_image": "mcr.microsoft.com/windows:1809", - "wix_image": "grafana/ci-wix:0.1.1", - "windows_server_core_image": "docker:windowsservercore-1809", - "windows_go_image": "grafana/grafana-ci-windows-test:0.1.0", + "1809": "mcr.microsoft.com/windows:1809", + "wix": "grafana/ci-wix:0.1.1", + "windows_server_core": "docker:windowsservercore-1809", + "go": "golang:{}-windowsservercore-1809".format(golang_version), } diff --git a/scripts/drone/variables.star b/scripts/drone/variables.star new file mode 100644 index 00000000000..f4d1b97b322 --- /dev/null +++ b/scripts/drone/variables.star @@ -0,0 +1,9 @@ +""" +global variables +""" + +grabpl_version = "v3.0.42" +golang_version = "1.20.10" + +# nodejs_version should match what's in ".nvmrc", but without the v prefix. +nodejs_version = "18.12.0" diff --git a/scripts/drone/vault.star b/scripts/drone/vault.star index b8d996d633f..6a27cf51785 100644 --- a/scripts/drone/vault.star +++ b/scripts/drone/vault.star @@ -14,6 +14,9 @@ azure_tenant = "azure_tenant" rgm_gcp_key_base64 = "gcp_key_base64" rgm_destination = "destination" +rgm_storybook_destination = "rgm_storybook_destination" +rgm_cdn_destination = "rgm_cdn_destination" +rgm_downloads_destination = "rgm_downloads_destination" rgm_github_token = "github_token" rgm_dagger_token = "dagger_token" @@ -122,6 +125,21 @@ def secrets(): "infra/data/ci/grafana-release-eng/rgm", "destination_prod", ), + vault_secret( + rgm_storybook_destination, + "infra/data/ci/grafana-release-eng/rgm", + "storybook_destination", + ), + vault_secret( + rgm_cdn_destination, + "infra/data/ci/grafana-release-eng/rgm", + "cdn_destination", + ), + vault_secret( + rgm_downloads_destination, + "infra/data/ci/grafana-release-eng/rgm", + "downloads_destination", + ), vault_secret( rgm_dagger_token, "infra/data/ci/grafana-release-eng/rgm", @@ -143,21 +161,6 @@ def secrets(): "infra/data/ci/grafana-release-eng/grafana-delivery-bot", "app-private-key", ), - vault_secret( - rgm_gcp_key_base64, - "infra/data/ci/grafana-release-eng/rgm", - "gcp_service_account_base64", - ), - vault_secret( - rgm_destination, - "infra/data/ci/grafana-release-eng/rgm", - "destination", - ), - vault_secret( - rgm_github_token, - "infra/data/ci/github/grafanabot", - "pat", - ), vault_secret( "gcr_credentials", "secret/data/common/gcr", diff --git a/scripts/drone/version.star b/scripts/drone/version.star deleted file mode 100644 index 140cd4d35d8..00000000000 --- a/scripts/drone/version.star +++ /dev/null @@ -1,16 +0,0 @@ -""" -This module returns the pipeline used for version branches. -""" - -load( - "scripts/drone/events/release.star", - "oss_pipelines", -) - -ver_mode = "release-branch" -trigger = {"ref": ["refs/heads/v[0-9]*"]} - -def version_branch_pipelines(): - return ( - oss_pipelines(ver_mode = ver_mode, trigger = trigger) - ) diff --git a/scripts/validate-npm-packages.sh b/scripts/validate-npm-packages.sh new file mode 100755 index 00000000000..4e2a27184fa --- /dev/null +++ b/scripts/validate-npm-packages.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# This script is used to validate the npm packages that are published to npmjs.org are in the correct format. +# It won't catch things like malformed JS or Types but it will assert that the package has +# the correct files and package.json properties. +ARTIFACTS_DIR="./npm-artifacts" + +for file in "$ARTIFACTS_DIR"/*.tgz; do + echo "🔍 Checking NPM package: $file" + # get filename then strip everything after package name. + dir_name=$(basename "$file" .tgz | sed -E 's/@([a-zA-Z0-9-]+)-[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9-]+)?/\1/') + mkdir -p "./npm-artifacts/$dir_name" + tar -xzf "$file" -C "./npm-artifacts/$dir_name" --strip-components=1 + + # Make sure the tar wasn't empty + if [ ! -d "./npm-artifacts/$dir_name" ]; then + echo -e "❌ Failed: Empty package $dir_name.\n" + exit 1 + fi + + # Navigate inside the new extracted directory + pushd "./npm-artifacts/$dir_name" || exit + + # Check for required files + check_files=("package.json" "README.md" "CHANGELOG.md" "LICENSE_APACHE2") + for check_file in "${check_files[@]}"; do + if [ ! -f "$check_file" ]; then + echo -e "❌ Failed: Missing required file $check_file in package $dir_name.\n" + exit 1 + fi + done + + # @grafana/toolkit structure is different to the other packages + if [[ "$dir_name" == "grafana-toolkit" ]]; then + if [ ! -d bin ] || [ ! -f bin/grafana-toolkit.js ]; then + echo -e "❌ Failed: Missing 'bin' directory or required files in package $dir_name.\n" + exit 1 + fi + + echo -e "✅ Passed: package checks for $file.\n" + popd || exit + continue + fi + + # Assert commonjs builds + if [ ! -d dist ] || [ ! -f dist/index.js ] || [ ! -f dist/index.d.ts ]; then + echo -e "❌ Failed: Missing 'dist' directory or required commonjs files in package $dir_name.\n" + exit 1 + fi + + if [ "$(jq -r '.main' package.json)" != "dist/index.js" ] || \ + [ "$(jq -r '.types' package.json)" != "dist/index.d.ts" ]; then + echo -e "❌ Failed: Incorrect package.json properties in package $dir_name.\n" + exit 1 + fi + + # Assert esm builds + esm_packages=("grafana-data" "grafana-ui" "grafana-runtime" "grafana-e2e-selectors" "grafana-schema") + for esm_package in "${esm_packages[@]}"; do + if [[ "$dir_name" == "$esm_package" ]]; then + if [ ! -d dist/esm ] || [ ! -f dist/esm/index.js ]; then + echo -e "❌ Failed: Missing 'dist/esm' directory or required esm files in package $dir_name.\n" + exit 1 + fi + + if [ "$(jq -r '.module' package.json)" != "dist/esm/index.js" ]; then + echo -e "❌ Failed: Incorrect package.json properties in package $dir_name.\n" + exit 1 + fi + fi + done + + echo -e "✅ Passed: package checks for $file.\n" + popd || exit + +done + +echo "🚀 All NPM package checks passed! 🚀" +rm -rf "${ARTIFACTS_DIR:?}/"*/ +exit 0