diff --git a/.drone.star b/.drone.star index 9d2a65ebc63..8c2a3c03767 100644 --- a/.drone.star +++ b/.drone.star @@ -3,20 +3,55 @@ # 2. Login to drone and export the env variables (token and server) shown here: https://drone.grafana.net/account # 3. Run `make drone` # More information about this process here: https://github.com/grafana/deployment_tools/blob/master/docs/infrastructure/drone/signing.md +""" +This module returns a Drone configuration including pipelines and secrets. +""" -load('scripts/drone/events/pr.star', 'pr_pipelines') -load('scripts/drone/events/main.star', 'main_pipelines') -load('scripts/drone/pipelines/docs.star', 'docs_pipelines') -load('scripts/drone/events/release.star', 'release_pipelines', 'publish_artifacts_pipelines', 'publish_npm_pipelines', 'publish_packages_pipeline', 'artifacts_page_pipeline') -load('scripts/drone/pipelines/publish_images.star', 'publish_image_pipelines_public', 'publish_image_pipelines_security') -load('scripts/drone/version.star', 'version_branch_pipelines') -load('scripts/drone/events/cron.star', 'cronjobs') -load('scripts/drone/vault.star', 'secrets') +load("scripts/drone/events/pr.star", "pr_pipelines") +load("scripts/drone/events/main.star", "main_pipelines") +load( + "scripts/drone/events/release.star", + "artifacts_page_pipeline", + "enterprise2_pipelines", + "enterprise_pipelines", + "oss_pipelines", + "publish_artifacts_pipelines", + "publish_npm_pipelines", + "publish_packages_pipeline", +) +load( + "scripts/drone/pipelines/publish_images.star", + "publish_image_pipelines_public", + "publish_image_pipelines_security", +) +load("scripts/drone/pipelines/github.star", "publish_github_pipeline") +load("scripts/drone/pipelines/aws_marketplace.star", "publish_aws_marketplace_pipeline") +load("scripts/drone/version.star", "version_branch_pipelines") +load("scripts/drone/events/cron.star", "cronjobs") +load("scripts/drone/vault.star", "secrets") -def main(ctx): - edition = 'oss' - return pr_pipelines(edition=edition) + main_pipelines(edition=edition) + release_pipelines() + \ - publish_image_pipelines_public() + publish_image_pipelines_security() + \ - publish_artifacts_pipelines('security') + publish_artifacts_pipelines('public') + \ - publish_npm_pipelines('public') + publish_packages_pipeline() + artifacts_page_pipeline() + \ - version_branch_pipelines() + cronjobs(edition=edition) + secrets() +def main(_ctx): + return ( + pr_pipelines() + + main_pipelines() + + oss_pipelines() + + enterprise_pipelines() + + enterprise2_pipelines() + + enterprise2_pipelines( + prefix = "custom-", + trigger = {"event": ["custom"]}, + ) + + publish_image_pipelines_public() + + publish_image_pipelines_security() + + publish_github_pipeline("public") + + publish_github_pipeline("security") + + publish_aws_marketplace_pipeline("public") + + publish_artifacts_pipelines("security") + + publish_artifacts_pipelines("public") + + publish_npm_pipelines() + + publish_packages_pipeline() + + artifacts_page_pipeline() + + version_branch_pipelines() + + cronjobs() + + secrets() + ) diff --git a/.drone.yml b/.drone.yml index 85641d836fd..edcbec2c520 100644 --- a/.drone.yml +++ b/.drone.yml @@ -2,6 +2,8 @@ clone: retries: 3 depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -15,11 +17,11 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -56,6 +58,63 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: oss +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: pr-verify-starlark +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.17.1 + name: identify-runner +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] + environment: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd +- commands: + - ./bin/build verify-starlark . + depends_on: + - compile-build-cmd + image: grafana/build-container:v1.7.1 + name: lint-starlark +trigger: + event: + - pull_request + paths: + exclude: + - docs/** + - '*.md' + include: + - scripts/drone/** + - .drone.star +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -69,33 +128,53 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install - commands: - yarn betterer ci depends_on: - yarn-install - failure: ignore - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: betterer-frontend +- commands: + - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" + | jq .head.repo.fork) + - if [ "$is_fork" != false ]; then return 1; fi + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + ../grafana-enterprise + - cd ../grafana-enterprise + - if git checkout ${DRONE_SOURCE_BRANCH}; then echo "checked out ${DRONE_SOURCE_BRANCH}"; + elif git checkout ${DRONE_TARGET_BRANCH}; then echo "git checkout ${DRONE_TARGET_BRANCH}"; + else git checkout main; fi + - cd ../ + - ln -s src grafana + - cd ./grafana-enterprise + - ./build.sh + environment: + GITHUB_TOKEN: + from_secret: github_token + failure: ignore + image: grafana/build-container:v1.7.1 + name: clone-enterprise - commands: - yarn run ci:test-frontend depends_on: - yarn-install + - clone-enterprise environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-frontend trigger: event: @@ -118,6 +197,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -129,14 +210,34 @@ platform: os: linux services: [] steps: +- commands: + - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" + | jq .head.repo.fork) + - if [ "$is_fork" != false ]; then return 1; fi + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + ../grafana-enterprise + - cd ../grafana-enterprise + - if git checkout ${DRONE_SOURCE_BRANCH}; then echo "checked out ${DRONE_SOURCE_BRANCH}"; + elif git checkout ${DRONE_TARGET_BRANCH}; then echo "git checkout ${DRONE_TARGET_BRANCH}"; + else git checkout main; fi + - cd ../ + - ln -s src grafana + - cd ./grafana-enterprise + - ./build.sh + environment: + GITHUB_TOKEN: + from_secret: github_token + failure: ignore + image: grafana/build-container:v1.7.1 + name: clone-enterprise - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install - commands: - yarn run prettier:check @@ -145,9 +246,10 @@ steps: - yarn run typecheck depends_on: - yarn-install + - clone-enterprise environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: lint-frontend trigger: event: @@ -183,9 +285,29 @@ platform: os: linux services: [] steps: +- commands: + - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" + | jq .head.repo.fork) + - if [ "$is_fork" != false ]; then return 1; fi + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + ../grafana-enterprise + - cd ../grafana-enterprise + - if git checkout ${DRONE_SOURCE_BRANCH}; then echo "checked out ${DRONE_SOURCE_BRANCH}"; + elif git checkout ${DRONE_TARGET_BRANCH}; then echo "git checkout ${DRONE_TARGET_BRANCH}"; + else git checkout main; fi + - cd ../ + - ln -s src grafana + - cd ./grafana-enterprise + - ./build.sh + environment: + GITHUB_TOKEN: + from_secret: github_token + failure: ignore + image: grafana/build-container:v1.7.1 + name: clone-enterprise - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd @@ -200,26 +322,27 @@ steps: - '# The following command will fail if running code generators produces any diff in output.' - CODEGEN_VERIFY=1 make gen-cue - depends_on: [] - image: grafana/build-container:1.6.6 + depends_on: + - clone-enterprise + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - - go test -short -covermode=atomic -timeout=30m ./pkg/... + - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-backend - commands: - - go test -run Integration -covermode=atomic -timeout=30m ./pkg/... + - go test -run Integration -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-backend-integration trigger: event: @@ -246,6 +369,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -259,7 +384,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd @@ -268,10 +393,31 @@ steps: CGO_ENABLED: 0 image: golang:1.19.4 name: compile-build-cmd +- commands: + - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" + | jq .head.repo.fork) + - if [ "$is_fork" != false ]; then return 1; fi + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + ../grafana-enterprise + - cd ../grafana-enterprise + - if git checkout ${DRONE_SOURCE_BRANCH}; then echo "checked out ${DRONE_SOURCE_BRANCH}"; + elif git checkout ${DRONE_TARGET_BRANCH}; then echo "git checkout ${DRONE_TARGET_BRANCH}"; + else git checkout main; fi + - cd ../ + - ln -s src grafana + - cd ./grafana-enterprise + - ./build.sh + environment: + GITHUB_TOKEN: + from_secret: github_token + failure: ignore + image: grafana/build-container:v1.7.1 + name: clone-enterprise - commands: - make gen-go - depends_on: [] - image: grafana/build-container:1.6.6 + depends_on: + - clone-enterprise + image: grafana/build-container:v1.7.1 name: wire-install - commands: - apt-get update && apt-get install make @@ -297,6 +443,7 @@ trigger: - go.mod - public/app/plugins/**/plugin.json - devenv/** + - .bingo/** type: docker volumes: - host: @@ -306,6 +453,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -319,11 +468,11 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -341,19 +490,51 @@ steps: in output.' - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install +- commands: + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + --depth=1 + - cd grafana-enterprise + - git fetch origin "refs/tags/*:refs/tags/*" --quiet + - if git show-ref --tags $${TEST_TAG} --quiet; then git tag -d $${TEST_TAG} && git + push --delete origin $${TEST_TAG}; fi + - git tag $${TEST_TAG} && git push origin $${TEST_TAG} + - cd - + - git fetch https://$${GITHUB_TOKEN}@github.com/grafana/grafana.git "refs/tags/*:refs/tags/*" + --quiet && git fetch --quiet + - if git show-ref --tags $${TEST_TAG} --quiet; then git tag -d $${TEST_TAG} && git + push --delete https://$${GITHUB_TOKEN}@github.com/grafana/grafana.git $${TEST_TAG}; + fi + - git tag $${TEST_TAG} && git push https://$${GITHUB_TOKEN}@github.com/grafana/grafana.git + $${TEST_TAG} + environment: + DOWNSTREAM_REPO: + from_secret: downstream + GITHUB_TOKEN: + from_secret: github_token_pr + TEST_TAG: v0.0.0-test + failure: ignore + image: grafana/build-container:v1.7.1 + name: trigger-test-release + when: + paths: + include: + - .drone.yml + - pkg/build/** + repo: + - grafana/grafana - failure: ignore image: grafana/drone-downstream name: trigger-enterprise-downstream @@ -372,7 +553,7 @@ steps: depends_on: - wire-install - compile-build-cmd - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-backend - commands: - ./bin/build build-frontend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} @@ -381,7 +562,7 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-frontend - commands: - ./bin/build build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} @@ -390,7 +571,7 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-frontend-packages - commands: - ./bin/build build-plugins --jobs 8 --edition oss @@ -398,18 +579,18 @@ steps: - compile-build-cmd - yarn-install environment: null - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-plugins - commands: - . scripts/build/gpg-test-vars.sh && ./bin/build package --jobs 8 --edition oss - --build-id ${DRONE_BUILD_NUMBER} --variants linux-amd64,linux-amd64-musl,darwin-amd64,windows-amd64 + --build-id ${DRONE_BUILD_NUMBER} depends_on: - build-plugins - build-backend - build-frontend - build-frontend-packages environment: null - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: package - commands: - ./scripts/grafana-server/start-server @@ -422,7 +603,7 @@ steps: environment: ARCH: linux-amd64 PORT: 3001 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: grafana-server - commands: - apt-get install -y netcat @@ -460,11 +641,35 @@ steps: HOST: grafana-server image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-various-suite +- commands: + - cd / + - ./cpp-e2e/scripts/ci-run.sh azure ${DRONE_SOURCE_BRANCH} + depends_on: + - grafana-server + environment: + AZURE_SP_APP_ID: + from_secret: azure_sp_app_id + AZURE_SP_PASSWORD: + from_secret: azure_sp_app_pw + AZURE_TENANT: + from_secret: azure_tenant + CYPRESS_CI: "true" + GITHUB_TOKEN: + from_secret: github_token_pr + HOST: grafana-server + image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + name: end-to-end-tests-cloud-plugins-suite-azure + when: + paths: + include: + - pkg/tsdb/azuremonitor/** + - public/app/plugins/datasource/grafana-azure-monitor-datasource/** + - e2e/cloud-plugins-suite/azure-monitor.spec.ts + repo: + - grafana/grafana - commands: - apt-get update - apt-get install -yq zip - - ls -lah ./e2e - - find ./e2e -type f -name "*.mp4" - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ @@ -486,7 +691,8 @@ steps: from_secret: gcp_upload_artifacts_key GITHUB_TOKEN: from_secret: github_token - image: google/cloud-sdk:367.0.0 + failure: ignore + image: google/cloud-sdk:406.0.0 name: e2e-tests-artifacts-upload when: status: @@ -494,13 +700,13 @@ steps: - failure - commands: - yarn storybook:build - - ./bin/grabpl verify-storybook + - ./bin/build verify-storybook depends_on: - build-frontend - build-frontend-packages environment: NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-storybook when: paths: @@ -511,7 +717,7 @@ steps: - cp dist/*.tar.gz* packaging/docker/ depends_on: - package - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: copy-packages-for-docker - commands: - yarn wait-on http://$HOST:$PORT @@ -556,6 +762,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -587,9 +795,29 @@ services: - name: mysql path: /var/lib/mysql steps: +- commands: + - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" + | jq .head.repo.fork) + - if [ "$is_fork" != false ]; then return 1; fi + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + ../grafana-enterprise + - cd ../grafana-enterprise + - if git checkout ${DRONE_SOURCE_BRANCH}; then echo "checked out ${DRONE_SOURCE_BRANCH}"; + elif git checkout ${DRONE_TARGET_BRANCH}; then echo "git checkout ${DRONE_TARGET_BRANCH}"; + else git checkout main; fi + - cd ../ + - ln -s src grafana + - cd ./grafana-enterprise + - ./build.sh + environment: + GITHUB_TOKEN: + from_secret: github_token + failure: ignore + image: grafana/build-container:v1.7.1 + name: clone-enterprise - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -602,7 +830,7 @@ steps: name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -610,14 +838,15 @@ steps: - '# The following command will fail if running code generators produces any diff in output.' - CODEGEN_VERIFY=1 make gen-cue - depends_on: [] - image: grafana/build-container:1.6.6 + depends_on: + - clone-enterprise + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - apt-get update @@ -626,14 +855,14 @@ steps: - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=30m {}' + -timeout=5m {}' depends_on: - wire-install environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: postgres-integration-tests - commands: - apt-get update @@ -643,13 +872,13 @@ steps: -prootpass - go clean -testcache - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=30m {}' + -timeout=5m {}' depends_on: - wire-install environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: mysql-integration-tests trigger: event: @@ -681,6 +910,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -694,18 +925,18 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install - commands: - |- @@ -717,7 +948,7 @@ steps: wan" > words_to_ignore.txt - codespell -I words_to_ignore.txt docs/ - rm words_to_ignore.txt - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: codespell - commands: - yarn run prettier:checkDocs @@ -725,7 +956,7 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: lint-docs - commands: - mkdir -p /hugo/content/docs/grafana @@ -751,6 +982,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -773,7 +1006,7 @@ steps: - ./bin/build shellcheck depends_on: - compile-build-cmd - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: shellcheck trigger: event: @@ -794,6 +1027,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -807,18 +1042,18 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install - commands: - |- @@ -830,7 +1065,7 @@ steps: wan" > words_to_ignore.txt - codespell -I words_to_ignore.txt docs/ - rm words_to_ignore.txt - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: codespell - commands: - yarn run prettier:checkDocs @@ -838,7 +1073,7 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: lint-docs - commands: - mkdir -p /hugo/content/docs/grafana @@ -865,6 +1100,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -878,25 +1115,24 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install - commands: - yarn betterer ci depends_on: - yarn-install - failure: ignore - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: betterer-frontend - commands: - yarn run ci:test-frontend @@ -904,7 +1140,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-frontend trigger: branch: main @@ -924,6 +1160,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -937,12 +1175,12 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install - commands: - yarn run prettier:check @@ -953,7 +1191,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: lint-frontend trigger: branch: main @@ -988,7 +1226,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd @@ -1004,25 +1242,25 @@ steps: in output.' - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - - go test -short -covermode=atomic -timeout=30m ./pkg/... + - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-backend - commands: - - go test -run Integration -covermode=atomic -timeout=30m ./pkg/... + - go test -run Integration -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-backend-integration trigger: branch: main @@ -1042,6 +1280,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -1055,7 +1295,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd @@ -1067,7 +1307,7 @@ steps: - commands: - make gen-go depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - apt-get update && apt-get install make @@ -1102,6 +1342,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -1115,11 +1357,11 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1137,53 +1379,25 @@ steps: in output.' - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - --depth=1 - - cd grafana-enterprise - - git fetch origin "refs/tags/*:refs/tags/*" - - git tag -d $${TEST_TAG} && git push --delete origin $${TEST_TAG} && git tag $${TEST_TAG} - && git push origin $${TEST_TAG} - - cd - - - git fetch origin "refs/tags/*:refs/tags/*" - - git remote add downstream https://$${GITHUB_TOKEN}@github.com/grafana/$${DOWNSTREAM_REPO}.git - - git tag -d $${TEST_TAG} && git push --delete downstream --quiet $${TEST_TAG} && - git tag $${TEST_TAG} && git push downstream $${TEST_TAG} --quiet - environment: - DOWNSTREAM_REPO: - from_secret: downstream - GITHUB_TOKEN: - from_secret: github_token - TEST_TAG: v0.0.0-test - failure: ignore - image: grafana/build-container:1.6.6 - name: trigger-test-release - when: - paths: - include: - - .drone.yml - - pkg/build/** - repo: - - grafana/grafana - commands: - ./bin/build build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} depends_on: - wire-install - compile-build-cmd - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-backend - commands: - ./bin/build build-frontend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} @@ -1192,7 +1406,7 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-frontend - commands: - ./bin/build build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} @@ -1201,7 +1415,7 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-frontend-packages - commands: - ./bin/build build-plugins --jobs 8 --edition oss @@ -1211,7 +1425,7 @@ steps: environment: GRAFANA_API_KEY: from_secret: grafana_api_key - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-plugins - commands: - ./bin/build package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --sign @@ -1229,7 +1443,7 @@ steps: from_secret: packages_gpg_public_key GRAFANA_API_KEY: from_secret: grafana_api_key - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: package - commands: - ./scripts/grafana-server/start-server @@ -1242,7 +1456,7 @@ steps: environment: ARCH: linux-amd64 PORT: 3001 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: grafana-server - commands: - apt-get install -y netcat @@ -1280,11 +1494,35 @@ steps: HOST: grafana-server image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 name: end-to-end-tests-various-suite +- commands: + - cd / + - ./cpp-e2e/scripts/ci-run.sh azure ${DRONE_SOURCE_BRANCH} + depends_on: + - grafana-server + environment: + AZURE_SP_APP_ID: + from_secret: azure_sp_app_id + AZURE_SP_PASSWORD: + from_secret: azure_sp_app_pw + AZURE_TENANT: + from_secret: azure_tenant + CYPRESS_CI: "true" + GITHUB_TOKEN: + from_secret: github_token_pr + HOST: grafana-server + image: us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + name: end-to-end-tests-cloud-plugins-suite-azure + when: + paths: + include: + - pkg/tsdb/azuremonitor/** + - public/app/plugins/datasource/grafana-azure-monitor-datasource/** + - e2e/cloud-plugins-suite/azure-monitor.spec.ts + repo: + - grafana/grafana - commands: - apt-get update - apt-get install -yq zip - - ls -lah ./e2e - - find ./e2e -type f -name "*.mp4" - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ @@ -1306,7 +1544,8 @@ steps: from_secret: gcp_upload_artifacts_key GITHUB_TOKEN: from_secret: github_token - image: google/cloud-sdk:367.0.0 + failure: ignore + image: google/cloud-sdk:406.0.0 name: e2e-tests-artifacts-upload when: status: @@ -1314,13 +1553,13 @@ steps: - failure - commands: - yarn storybook:build - - ./bin/grabpl verify-storybook + - ./bin/build verify-storybook depends_on: - build-frontend - build-frontend-packages environment: NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-storybook when: paths: @@ -1331,7 +1570,7 @@ steps: - cp dist/*.tar.gz* packaging/docker/ depends_on: - package - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: copy-packages-for-docker - commands: - yarn wait-on http://$HOST:$PORT @@ -1375,7 +1614,7 @@ steps: GRAFANA_MISC_STATS_API_KEY: from_secret: grafana_misc_stats_api_key failure: ignore - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: publish-frontend-metrics when: repo: @@ -1419,7 +1658,7 @@ steps: GCP_KEY: from_secret: gcp_key image: google/cloud-sdk - name: publish-images-grafana/grafana + name: publish-images-grafana volumes: - name: docker path: /var/run/docker.sock @@ -1439,7 +1678,7 @@ steps: GCP_KEY: from_secret: gcp_key image: google/cloud-sdk - name: publish-images-grafana/grafana-oss + name: publish-images-grafana-oss volumes: - name: docker path: /var/run/docker.sock @@ -1456,13 +1695,13 @@ steps: environment: NPM_TOKEN: from_secret: npm_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: release-canary-npm-packages when: repo: - grafana/grafana - commands: - - ./bin/grabpl upload-packages --edition oss + - ./bin/build upload-packages --edition oss depends_on: - end-to-end-tests-dashboards-suite - end-to-end-tests-panels-suite @@ -1479,7 +1718,7 @@ steps: repo: - grafana/grafana - commands: - - ./bin/grabpl upload-cdn --edition oss + - ./bin/build upload-cdn --edition oss depends_on: - grafana-server environment: @@ -1510,6 +1749,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -1543,7 +1784,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1556,7 +1797,7 @@ steps: name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -1565,13 +1806,13 @@ steps: in output.' - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - apt-get update @@ -1580,14 +1821,14 @@ steps: - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=30m {}' + -timeout=5m {}' depends_on: - wire-install environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: postgres-integration-tests - commands: - apt-get update @@ -1597,13 +1838,13 @@ steps: -prootpass - go clean -testcache - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=30m {}' + -timeout=5m {}' depends_on: - wire-install environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: mysql-integration-tests trigger: branch: main @@ -1633,6 +1874,8 @@ depends_on: - main-test-backend - main-build-e2e-publish - main-integration-tests +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -1649,7 +1892,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: windows-init @@ -1728,78 +1971,10 @@ type: docker clone: retries: 3 depends_on: -- main-test-frontend -- main-test-backend - main-build-e2e-publish - main-integration-tests -- main-windows -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: main-publish -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.19.4 - name: compile-build-cmd -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 - name: identify-runner -- commands: - - ./bin/build publish packages --edition oss --gcp-key /tmp/gcpkey.json --build-id - ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_COM_API_KEY: - from_secret: grafana_api_key - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-packages-oss -trigger: - branch: main - event: - - push - paths: - exclude: - - '*.md' - - docs/** - - latest.json - repo: - - grafana/grafana -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: -- main-publish +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -1845,7 +2020,6 @@ depends_on: - main-build-e2e-publish - main-integration-tests - main-windows -- main-publish kind: pipeline name: main-notify platform: @@ -1893,11 +2067,11 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1908,18 +2082,18 @@ steps: in output.' - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd @@ -1933,7 +2107,7 @@ steps: depends_on: - wire-install - compile-build-cmd - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-backend - commands: - ./bin/build build-frontend --jobs 8 --edition oss ${DRONE_TAG} @@ -1942,7 +2116,7 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-frontend - commands: - ./bin/build build-frontend-packages --jobs 8 --edition oss ${DRONE_TAG} @@ -1951,7 +2125,7 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-frontend-packages - commands: - ./bin/build build-plugins --jobs 8 --edition oss @@ -1961,7 +2135,7 @@ steps: environment: GRAFANA_API_KEY: from_secret: grafana_api_key - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-plugins - commands: - ./bin/build package --jobs 8 --edition oss --sign ${DRONE_TAG} @@ -1979,14 +2153,14 @@ steps: from_secret: packages_gpg_public_key GRAFANA_API_KEY: from_secret: grafana_api_key - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: package - commands: - ls dist/*.tar.gz* - cp dist/*.tar.gz* packaging/docker/ depends_on: - package - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: copy-packages-for-docker - commands: - ./bin/build build-docker --edition oss --shouldSave @@ -2025,7 +2199,7 @@ steps: environment: ARCH: linux-amd64 PORT: 3001 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: grafana-server - commands: - apt-get install -y netcat @@ -2066,8 +2240,6 @@ steps: - commands: - apt-get update - apt-get install -yq zip - - ls -lah ./e2e - - find ./e2e -type f -name "*.mp4" - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ @@ -2089,7 +2261,8 @@ steps: from_secret: gcp_upload_artifacts_key GITHUB_TOKEN: from_secret: github_token - image: google/cloud-sdk:367.0.0 + failure: ignore + image: google/cloud-sdk:406.0.0 name: e2e-tests-artifacts-upload when: status: @@ -2097,19 +2270,19 @@ steps: - failure - commands: - yarn storybook:build - - ./bin/grabpl verify-storybook + - ./bin/build verify-storybook depends_on: - build-frontend - build-frontend-packages environment: NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-storybook when: event: - tag - commands: - - ./bin/grabpl upload-cdn --edition oss + - ./bin/build upload-cdn --edition oss depends_on: - grafana-server environment: @@ -2120,7 +2293,7 @@ steps: image: grafana/grafana-ci-deploy:1.3.3 name: upload-cdn-assets - commands: - - ./bin/grabpl upload-packages --edition oss + - ./bin/build upload-packages --edition oss depends_on: - end-to-end-tests-dashboards-suite - end-to-end-tests-panels-suite @@ -2153,15 +2326,16 @@ steps: event: - tag - commands: - - ./bin/grabpl artifacts npm store --tag ${DRONE_TAG} + - ./bin/build artifacts npm store --tag ${DRONE_TAG} depends_on: + - compile-build-cmd - build-frontend-packages environment: GCP_KEY: from_secret: gcp_key PRERELEASE_BUCKET: from_secret: prerelease_bucket - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: store-npm-packages trigger: event: @@ -2169,9 +2343,6 @@ trigger: - promote ref: - refs/tags/v* - repo: - exclude: - - grafana/grafana type: docker volumes: - host: @@ -2187,6 +2358,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -2200,25 +2373,24 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install - commands: - yarn betterer ci depends_on: - yarn-install - failure: ignore - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: betterer-frontend - commands: - yarn run ci:test-frontend @@ -2226,7 +2398,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-frontend trigger: event: @@ -2234,9 +2406,6 @@ trigger: - promote ref: - refs/tags/v* - repo: - exclude: - - grafana/grafana type: docker volumes: - host: @@ -2261,7 +2430,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd @@ -2277,25 +2446,25 @@ steps: in output.' - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - - go test -short -covermode=atomic -timeout=30m ./pkg/... + - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-backend - commands: - - go test -run Integration -covermode=atomic -timeout=30m ./pkg/... + - go test -run Integration -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-backend-integration trigger: event: @@ -2303,9 +2472,6 @@ trigger: - promote ref: - refs/tags/v* - repo: - exclude: - - grafana/grafana type: docker volumes: - host: @@ -2350,13 +2516,13 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -2365,13 +2531,13 @@ steps: in output.' - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - apt-get update @@ -2380,14 +2546,14 @@ steps: - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=30m {}' + -timeout=5m {}' depends_on: - wire-install environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: postgres-integration-tests - commands: - apt-get update @@ -2397,13 +2563,13 @@ steps: -prootpass - go clean -testcache - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=30m {}' + -timeout=5m {}' depends_on: - wire-install environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: mysql-integration-tests trigger: event: @@ -2411,9 +2577,6 @@ trigger: - promote ref: - refs/tags/v* - repo: - exclude: - - grafana/grafana type: docker volumes: - host: @@ -2451,7 +2614,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: windows-init @@ -2484,9 +2647,6 @@ trigger: - promote ref: - refs/tags/v* - repo: - exclude: - - grafana/grafana type: docker volumes: - host: @@ -2511,13 +2671,13 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" @@ -2526,7 +2686,7 @@ steps: environment: GITHUB_TOKEN: from_secret: github_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: clone-enterprise - commands: - mv bin/grabpl /tmp/ @@ -2542,7 +2702,7 @@ steps: environment: GITHUB_TOKEN: from_secret: github_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: init-enterprise - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd @@ -2556,13 +2716,13 @@ steps: - make gen-go depends_on: - init-enterprise - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - yarn install --immutable depends_on: - init-enterprise - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -2572,14 +2732,14 @@ steps: - CODEGEN_VERIFY=1 make gen-cue depends_on: - init-enterprise - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - ./bin/build build-backend --jobs 8 --edition enterprise ${DRONE_TAG} depends_on: - wire-install - compile-build-cmd - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-backend - commands: - ./bin/build build-frontend --jobs 8 --edition enterprise ${DRONE_TAG} @@ -2588,7 +2748,7 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-frontend - commands: - ./bin/build build-frontend-packages --jobs 8 --edition enterprise ${DRONE_TAG} @@ -2597,7 +2757,7 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-frontend-packages - commands: - ./bin/build build-plugins --jobs 8 --edition enterprise @@ -2607,15 +2767,8 @@ steps: environment: GRAFANA_API_KEY: from_secret: grafana_api_key - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-plugins -- commands: - - ./bin/build build-backend --jobs 8 --edition enterprise2 ${DRONE_TAG} - depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.6.6 - name: build-backend-enterprise2 - commands: - ./bin/build package --jobs 8 --edition enterprise --sign ${DRONE_TAG} depends_on: @@ -2623,7 +2776,6 @@ steps: - build-backend - build-frontend - build-frontend-packages - - build-backend-enterprise2 environment: GPG_KEY_PASSWORD: from_secret: packages_gpg_passphrase @@ -2633,14 +2785,14 @@ steps: from_secret: packages_gpg_public_key GRAFANA_API_KEY: from_secret: grafana_api_key - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: package - commands: - ls dist/*.tar.gz* - cp dist/*.tar.gz* packaging/docker/ depends_on: - package - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: copy-packages-for-docker - commands: - ./bin/build build-docker --edition enterprise --shouldSave @@ -2680,7 +2832,7 @@ steps: ARCH: linux-amd64 PORT: 3001 RUNDIR: scripts/grafana-server/tmp-grafana-enterprise - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: grafana-server - commands: - apt-get install -y netcat @@ -2721,8 +2873,6 @@ steps: - commands: - apt-get update - apt-get install -yq zip - - ls -lah ./e2e - - find ./e2e -type f -name "*.mp4" - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ @@ -2744,14 +2894,15 @@ steps: from_secret: gcp_upload_artifacts_key GITHUB_TOKEN: from_secret: github_token - image: google/cloud-sdk:367.0.0 + failure: ignore + image: google/cloud-sdk:406.0.0 name: e2e-tests-artifacts-upload when: status: - success - failure - commands: - - ./bin/grabpl upload-cdn --edition enterprise + - ./bin/build upload-cdn --edition enterprise depends_on: - package environment: @@ -2762,7 +2913,7 @@ steps: image: grafana/grafana-ci-deploy:1.3.3 name: upload-cdn-assets - commands: - - ./bin/grabpl upload-packages --edition enterprise + - ./bin/build upload-packages --edition enterprise depends_on: - package environment: @@ -2772,67 +2923,12 @@ steps: from_secret: prerelease_bucket image: grafana/grafana-ci-deploy:1.3.3 name: upload-packages -- commands: - - ./bin/build package --jobs 8 --edition enterprise2 --sign ${DRONE_TAG} - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - - build-backend-enterprise2 - environment: - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.6.6 - name: package-enterprise2 -- commands: - - ./bin/grabpl upload-cdn --edition enterprise2 - depends_on: - - package-enterprise2 - environment: - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-cdn-assets-enterprise2 -- commands: - - ./bin/grabpl artifacts npm store --tag ${DRONE_TAG} - depends_on: - - build-frontend-packages - environment: - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/build-container:1.6.6 - name: store-npm-packages -- commands: - - ./bin/grabpl upload-packages --edition enterprise2 - depends_on: - - package-enterprise2 - environment: - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-packages-enterprise2 trigger: event: exclude: - promote ref: - refs/tags/v* - repo: - exclude: - - grafana/grafana type: docker volumes: - host: @@ -2848,6 +2944,8 @@ volumes: clone: disable: true depends_on: [] +environment: + EDITION: enterprise image_pull_secrets: - dockerconfigjson kind: pipeline @@ -2866,7 +2964,7 @@ steps: environment: GITHUB_TOKEN: from_secret: github_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: clone-enterprise - commands: - mv bin/grabpl /tmp/ @@ -2882,15 +2980,15 @@ steps: environment: GITHUB_TOKEN: from_secret: github_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: init-enterprise - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2898,15 +2996,14 @@ steps: - yarn install --immutable depends_on: - init-enterprise - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install - commands: - yarn betterer ci depends_on: - init-enterprise - yarn-install - failure: ignore - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: betterer-frontend - commands: - yarn run ci:test-frontend @@ -2915,7 +3012,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-frontend trigger: event: @@ -2923,9 +3020,6 @@ trigger: - promote ref: - refs/tags/v* - repo: - exclude: - - grafana/grafana type: docker volumes: - host: @@ -2955,11 +3049,11 @@ steps: environment: GITHUB_TOKEN: from_secret: github_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: clone-enterprise - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2977,11 +3071,11 @@ steps: environment: GITHUB_TOKEN: from_secret: github_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: init-enterprise - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd @@ -2999,25 +3093,25 @@ steps: - CODEGEN_VERIFY=1 make gen-cue depends_on: - init-enterprise - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - - go test -short -covermode=atomic -timeout=30m ./pkg/... + - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-backend - commands: - - go test -run Integration -covermode=atomic -timeout=30m ./pkg/... + - go test -run Integration -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-backend-integration trigger: event: @@ -3025,111 +3119,6 @@ trigger: - promote ref: - refs/tags/v* - repo: - exclude: - - grafana/grafana -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - disable: true -depends_on: [] -environment: - EDITION: enterprise2 -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-enterprise2-test-backend -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout ${DRONE_TAG} - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.6.6 - name: clone-enterprise -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - mv bin/grabpl /tmp/ - - rmdir bin - - mv grafana-enterprise /tmp/ - - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise - ${DRONE_TAG} - - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - - mkdir bin - - mv /tmp/grabpl bin/ - depends_on: - - clone-enterprise - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.6.6 - name: init-enterprise -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 - name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: - - init-enterprise - environment: - CGO_ENABLED: 0 - image: golang:1.19.4 - name: compile-build-cmd -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: - - init-enterprise - image: grafana/build-container:1.6.6 - name: verify-gen-cue -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.6.6 - name: wire-install -- commands: - - go test -tags=pro -covermode=atomic -timeout=30m ./pkg/... - depends_on: - - wire-install - image: grafana/build-container:1.6.6 - name: test-backend-enterprise2 -- commands: - - go test -run Integration -covermode=atomic -timeout=30m ./pkg/... - depends_on: - - wire-install - image: grafana/build-container:1.6.6 - name: test-backend-integration -trigger: - event: - exclude: - - promote - ref: - - refs/tags/v* - repo: - exclude: - - grafana/grafana type: docker volumes: - host: @@ -3180,13 +3169,13 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" @@ -3195,7 +3184,7 @@ steps: environment: GITHUB_TOKEN: from_secret: github_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: clone-enterprise - commands: - mv bin/grabpl /tmp/ @@ -3211,7 +3200,7 @@ steps: environment: GITHUB_TOKEN: from_secret: github_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: init-enterprise - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -3221,13 +3210,13 @@ steps: - CODEGEN_VERIFY=1 make gen-cue depends_on: - init-enterprise - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - apt-get update @@ -3236,14 +3225,14 @@ steps: - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=30m {}' + -timeout=5m {}' depends_on: - wire-install environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: postgres-integration-tests - commands: - apt-get update @@ -3253,31 +3242,35 @@ steps: -prootpass - go clean -testcache - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=30m {}' + -timeout=5m {}' depends_on: - wire-install environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: mysql-integration-tests - commands: - dockerize -wait tcp://redis:6379/0 -timeout 120s - - ./bin/grabpl integration-tests + - go clean -testcache + - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic + -timeout=5m {}' depends_on: - wire-install environment: REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: redis-integration-tests - commands: - dockerize -wait tcp://memcached:11211 -timeout 120s - - ./bin/grabpl integration-tests + - go clean -testcache + - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic + -timeout=5m {}' depends_on: - wire-install environment: MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: memcached-integration-tests trigger: event: @@ -3285,9 +3278,6 @@ trigger: - promote ref: - refs/tags/v* - repo: - exclude: - - grafana/grafana type: docker volumes: - host: @@ -3325,7 +3315,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/windows/grabpl.exe -OutFile grabpl.exe - git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git" - cd grafana-enterprise @@ -3378,18 +3368,514 @@ trigger: - promote ref: - refs/tags/v* - repo: - exclude: - - grafana/grafana type: docker volumes: - host: path: /var/run/docker.sock name: docker --- +clone: + disable: true +depends_on: [] +environment: + EDITION: enterprise2 +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: release-enterprise2-build-e2e-publish +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.17.1 + name: identify-runner +- commands: + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + - cd grafana-enterprise + - git checkout ${DRONE_TAG} + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:v1.7.1 + name: clone-enterprise +- commands: + - mv bin/grabpl /tmp/ + - rmdir bin + - mv grafana-enterprise /tmp/ + - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise + ${DRONE_TAG} + - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json + - mkdir bin + - mv /tmp/grabpl bin/ + depends_on: + - clone-enterprise + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:v1.7.1 + name: init-enterprise +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: + - init-enterprise + environment: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd +- commands: + - make gen-go + depends_on: + - init-enterprise + image: grafana/build-container:v1.7.1 + name: wire-install +- commands: + - yarn install --immutable + depends_on: + - init-enterprise + image: grafana/build-container:v1.7.1 + name: yarn-install +- commands: + - '# It is required that code generated from Thema/CUE be committed and in sync + with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - CODEGEN_VERIFY=1 make gen-cue + depends_on: + - init-enterprise + image: grafana/build-container:v1.7.1 + name: verify-gen-cue +- commands: + - ./bin/build build-frontend --jobs 8 --edition enterprise ${DRONE_TAG} + depends_on: + - compile-build-cmd + - yarn-install + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: grafana/build-container:v1.7.1 + name: build-frontend +- commands: + - ./bin/build build-frontend-packages --jobs 8 --edition enterprise ${DRONE_TAG} + depends_on: + - compile-build-cmd + - yarn-install + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: grafana/build-container:v1.7.1 + name: build-frontend-packages +- commands: + - ./bin/build build-plugins --jobs 8 --edition enterprise + depends_on: + - compile-build-cmd + - yarn-install + environment: + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:v1.7.1 + name: build-plugins +- commands: + - ./bin/build build-backend --jobs 8 --edition enterprise2 ${DRONE_TAG} + depends_on: + - wire-install + - compile-build-cmd + image: grafana/build-container:v1.7.1 + name: build-backend-enterprise2 +- commands: + - ./bin/build package --jobs 8 --edition enterprise2 --sign ${DRONE_TAG} + depends_on: + - build-plugins + - build-backend-enterprise2 + - build-frontend + - build-frontend-packages + environment: + GPG_KEY_PASSWORD: + from_secret: packages_gpg_passphrase + GPG_PRIV_KEY: + from_secret: packages_gpg_private_key + GPG_PUB_KEY: + from_secret: packages_gpg_public_key + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:v1.7.1 + name: package-enterprise2 +- commands: + - ./bin/build upload-cdn --edition enterprise2 + depends_on: + - package-enterprise2 + environment: + ENTERPRISE2_CDN_PATH: + from_secret: enterprise2-cdn-path + GCP_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.3 + name: upload-cdn-assets-enterprise2 +- commands: + - ls dist/*.tar.gz* + - cp dist/*.tar.gz* packaging/docker/ + depends_on: + - package-enterprise2 + image: grafana/build-container:v1.7.1 + name: copy-packages-for-docker +- commands: + - ./bin/build build-docker --edition enterprise2 --shouldSave + depends_on: + - copy-packages-for-docker + - compile-build-cmd + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: build-docker-images + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build build-docker --edition enterprise2 --shouldSave --ubuntu + depends_on: + - copy-packages-for-docker + - compile-build-cmd + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: build-docker-images-ubuntu + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build artifacts docker fetch --edition enterprise2 + depends_on: + - build-docker-images + - build-docker-images-ubuntu + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: fetch-images-enterprise2 + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build artifacts docker publish-enterprise2 --dockerhub-repo $${DOCKER_ENTERPRISE2_REPO} + depends_on: + - fetch-images-enterprise2 + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key_hg + image: google/cloud-sdk + name: publish-images-enterprise2 + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build upload-packages --edition enterprise2 + depends_on: + - package-enterprise2 + environment: + GCP_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.3 + name: upload-packages-enterprise2 +trigger: + event: + exclude: + - promote + ref: + - refs/tags/v* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory +--- +clone: + disable: true +depends_on: [] +environment: + EDITION: enterprise2 +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: custom-release-enterprise2-build-e2e-publish +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.17.1 + name: identify-runner +- commands: + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + - cd grafana-enterprise + - git checkout ${DRONE_TARGET_BRANCH} + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:v1.7.1 + name: clone-enterprise +- commands: + - mv bin/grabpl /tmp/ + - rmdir bin + - mv grafana-enterprise /tmp/ + - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise + ${DRONE_TAG} + - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json + - mkdir bin + - mv /tmp/grabpl bin/ + depends_on: + - clone-enterprise + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:v1.7.1 + name: init-enterprise +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: + - init-enterprise + environment: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd +- commands: + - make gen-go + depends_on: + - init-enterprise + image: grafana/build-container:v1.7.1 + name: wire-install +- commands: + - yarn install --immutable + depends_on: + - init-enterprise + image: grafana/build-container:v1.7.1 + name: yarn-install +- commands: + - '# It is required that code generated from Thema/CUE be committed and in sync + with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - CODEGEN_VERIFY=1 make gen-cue + depends_on: + - init-enterprise + image: grafana/build-container:v1.7.1 + name: verify-gen-cue +- commands: + - ./bin/build build-frontend --jobs 8 --edition enterprise ${DRONE_TAG} + depends_on: + - compile-build-cmd + - yarn-install + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: grafana/build-container:v1.7.1 + name: build-frontend +- commands: + - ./bin/build build-frontend-packages --jobs 8 --edition enterprise ${DRONE_TAG} + depends_on: + - compile-build-cmd + - yarn-install + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: grafana/build-container:v1.7.1 + name: build-frontend-packages +- commands: + - ./bin/build build-plugins --jobs 8 --edition enterprise + depends_on: + - compile-build-cmd + - yarn-install + environment: + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:v1.7.1 + name: build-plugins +- commands: + - ./bin/build build-backend --jobs 8 --edition enterprise2 ${DRONE_TAG} + depends_on: + - wire-install + - compile-build-cmd + image: grafana/build-container:v1.7.1 + name: build-backend-enterprise2 +- commands: + - ./bin/build package --jobs 8 --edition enterprise2 --sign ${DRONE_TAG} + depends_on: + - build-plugins + - build-backend-enterprise2 + - build-frontend + - build-frontend-packages + environment: + GPG_KEY_PASSWORD: + from_secret: packages_gpg_passphrase + GPG_PRIV_KEY: + from_secret: packages_gpg_private_key + GPG_PUB_KEY: + from_secret: packages_gpg_public_key + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:v1.7.1 + name: package-enterprise2 +- commands: + - ./bin/build upload-cdn --edition enterprise2 + depends_on: + - package-enterprise2 + environment: + ENTERPRISE2_CDN_PATH: + from_secret: enterprise2-cdn-path + GCP_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.3 + name: upload-cdn-assets-enterprise2 +- commands: + - ls dist/*.tar.gz* + - cp dist/*.tar.gz* packaging/docker/ + depends_on: + - package-enterprise2 + image: grafana/build-container:v1.7.1 + name: copy-packages-for-docker +- commands: + - ./bin/build build-docker --edition enterprise2 --shouldSave + depends_on: + - copy-packages-for-docker + - compile-build-cmd + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: build-docker-images + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build build-docker --edition enterprise2 --shouldSave --ubuntu + depends_on: + - copy-packages-for-docker + - compile-build-cmd + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: build-docker-images-ubuntu + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build artifacts docker fetch --edition enterprise2 + depends_on: + - build-docker-images + - build-docker-images-ubuntu + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: fetch-images-enterprise2 + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build artifacts docker publish-enterprise2 --dockerhub-repo $${DOCKER_ENTERPRISE2_REPO} + depends_on: + - fetch-images-enterprise2 + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key_hg + image: google/cloud-sdk + name: publish-images-enterprise2 + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build upload-packages --edition enterprise2 + depends_on: + - package-enterprise2 + environment: + GCP_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.3 + name: upload-packages-enterprise2 +trigger: + event: + - custom +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory +--- clone: retries: 3 depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -3401,9 +3887,13 @@ platform: os: linux services: [] steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.17.1 + name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -3419,6 +3909,8 @@ steps: depends_on: - compile-build-cmd environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo DOCKER_PASSWORD: from_secret: docker_password DOCKER_USER: @@ -3432,7 +3924,7 @@ steps: path: /var/run/docker.sock - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana --version-tag - ${TAG} + ${DRONE_TAG} depends_on: - fetch-images-oss environment: @@ -3443,13 +3935,13 @@ steps: GCP_KEY: from_secret: gcp_key image: google/cloud-sdk - name: publish-images-grafana/grafana + name: publish-images-grafana volumes: - name: docker path: /var/run/docker.sock - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana-oss --version-tag - ${TAG} + ${DRONE_TAG} depends_on: - fetch-images-oss environment: @@ -3460,7 +3952,7 @@ steps: GCP_KEY: from_secret: gcp_key image: google/cloud-sdk - name: publish-images-grafana/grafana-oss + name: publish-images-grafana-oss volumes: - name: docker path: /var/run/docker.sock @@ -3478,6 +3970,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: enterprise image_pull_secrets: - dockerconfigjson kind: pipeline @@ -3489,9 +3983,13 @@ platform: os: linux services: [] steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.17.1 + name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -3507,6 +4005,8 @@ steps: depends_on: - compile-build-cmd environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo DOCKER_PASSWORD: from_secret: docker_password DOCKER_USER: @@ -3520,7 +4020,7 @@ steps: path: /var/run/docker.sock - commands: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana-enterprise - --version-tag ${TAG} + --version-tag ${DRONE_TAG} depends_on: - fetch-images-enterprise environment: @@ -3531,7 +4031,7 @@ steps: GCP_KEY: from_secret: gcp_key image: google/cloud-sdk - name: publish-images-grafana/grafana-enterprise + name: publish-images-grafana-enterprise volumes: - name: docker path: /var/run/docker.sock @@ -3549,6 +4049,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: enterprise image_pull_secrets: - dockerconfigjson kind: pipeline @@ -3560,9 +4062,13 @@ platform: os: linux services: [] steps: +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.17.1 + name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -3578,6 +4084,8 @@ steps: depends_on: - compile-build-cmd environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo DOCKER_PASSWORD: from_secret: docker_password DOCKER_USER: @@ -3591,7 +4099,7 @@ steps: path: /var/run/docker.sock - commands: - ./bin/grabpl artifacts docker publish --security --dockerhub-repo grafana/grafana-enterprise - --version-tag ${TAG} + --version-tag ${DRONE_TAG} depends_on: - fetch-images-enterprise environment: @@ -3602,7 +4110,7 @@ steps: GCP_KEY: from_secret: gcp_key image: google/cloud-sdk - name: publish-images-grafana/grafana-enterprise + name: publish-images-grafana-enterprise volumes: - name: docker path: /var/run/docker.sock @@ -3620,6 +4128,204 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: enterprise2 +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: publish-github-public +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] + environment: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd +- commands: + - ./bin/build artifacts docker fetch --edition enterprise2 + depends_on: + - compile-build-cmd + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: fetch-images-enterprise2 + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build publish github --repo $${GH_REGISTRY} --create + depends_on: + - fetch-images-enterprise2 + environment: + GH_REGISTRY: + from_secret: gh_registry + GH_TOKEN: + from_secret: github_token + image: grafana/grafana-ci-deploy:1.3.3 + name: publish-github +trigger: + event: + - promote + target: + - public +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: [] +environment: + EDITION: enterprise2 +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: publish-github-security +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] + environment: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd +- commands: + - ./bin/build artifacts docker fetch --edition enterprise2 + depends_on: + - compile-build-cmd + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: fetch-images-enterprise2 + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build publish github --repo $${GH_REGISTRY} --create + depends_on: + - fetch-images-enterprise2 + environment: + GH_REGISTRY: + from_secret: gh_registry + GH_TOKEN: + from_secret: github_token + image: grafana/grafana-ci-deploy:1.3.3 + name: publish-github +trigger: + event: + - promote + target: + - security +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: +- publish-docker-enterprise-public +environment: + EDITION: enterprise2 +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: publish-aws-marketplace-public +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] + environment: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd +- commands: + - ./bin/build artifacts docker fetch --edition enterprise + depends_on: + - compile-build-cmd + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: fetch-images-enterprise + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build publish aws --image grafana/grafana-enterprise --repo grafana-labs/grafanaenterprise + --product 422b46fb-bea6-4f27-8bcc-832117bd627e + depends_on: + - fetch-images-enterprise + environment: + AWS_ACCESS_KEY_ID: + from_secret: aws_access_key_id + AWS_REGION: + from_secret: aws_region + AWS_SECRET_ACCESS_KEY: + from_secret: aws_secret_access_key + image: grafana/grafana-ci-deploy:1.3.3 + name: publish-aws-marketplace + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + event: + - promote + target: + - public +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: [] +environment: + EDITION: all image_pull_secrets: - dockerconfigjson kind: pipeline @@ -3632,20 +4338,27 @@ platform: services: [] steps: - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - ./bin/grabpl artifacts publish --security --tag ${TAG} --src-bucket $${PRERELEASE_BUCKET} - depends_on: - - grabpl + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] environment: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd +- commands: + - ./bin/build artifacts publish --security --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET} + depends_on: + - compile-build-cmd + environment: + ENTERPRISE2_SECURITY_PREFIX: + from_secret: enterprise2_security_prefix GCP_KEY: from_secret: gcp_key PRERELEASE_BUCKET: from_secret: prerelease_bucket + SECURITY_DEST_BUCKET: + from_secret: security_dest_bucket + STATIC_ASSET_EDITIONS: + from_secret: static_asset_editions image: grafana/grafana-ci-deploy:1.3.3 name: publish-artifacts trigger: @@ -3662,6 +4375,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: all image_pull_secrets: - dockerconfigjson kind: pipeline @@ -3674,20 +4389,27 @@ platform: services: [] steps: - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - ./bin/grabpl artifacts publish --tag ${TAG} --src-bucket $${PRERELEASE_BUCKET} - depends_on: - - grabpl + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] environment: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd +- commands: + - ./bin/build artifacts publish --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET} + depends_on: + - compile-build-cmd + environment: + ENTERPRISE2_SECURITY_PREFIX: + from_secret: enterprise2_security_prefix GCP_KEY: from_secret: gcp_key PRERELEASE_BUCKET: from_secret: prerelease_bucket + SECURITY_DEST_BUCKET: + from_secret: security_dest_bucket + STATIC_ASSET_EDITIONS: + from_secret: static_asset_editions image: grafana/grafana-ci-deploy:1.3.3 name: publish-artifacts trigger: @@ -3704,6 +4426,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: all image_pull_secrets: - dockerconfigjson kind: pipeline @@ -3716,19 +4440,21 @@ platform: services: [] steps: - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] + environment: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install - commands: - - ./bin/grabpl artifacts npm retrieve --tag v${TAG} + - ./bin/build artifacts npm retrieve --tag ${DRONE_TAG} depends_on: + - compile-build-cmd - yarn-install environment: GCP_KEY: @@ -3739,14 +4465,15 @@ steps: image: grafana/grafana-ci-deploy:1.3.3 name: retrieve-npm-packages - commands: - - ./bin/grabpl artifacts npm release --tag v${TAG} + - ./bin/build artifacts npm release --tag ${DRONE_TAG} depends_on: + - compile-build-cmd - retrieve-npm-packages environment: NPM_TOKEN: from_secret: npm_token failure: ignore - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: release-npm-packages trigger: event: @@ -3765,6 +4492,8 @@ depends_on: - publish-artifacts-public - publish-docker-oss-public - publish-docker-enterprise-public +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -3778,7 +4507,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -3789,37 +4518,8 @@ steps: CGO_ENABLED: 0 image: golang:1.19.4 name: compile-build-cmd -- commands: - - ./bin/build publish packages --edition oss --gcp-key /tmp/gcpkey.json ${DRONE_TAG} - depends_on: - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_COM_API_KEY: - from_secret: grafana_api_key - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-packages-oss -- commands: - - ./bin/build publish grafana-com --edition oss ${DRONE_TAG} - depends_on: - - publish-packages-oss - environment: - GCP_KEY: - from_secret: gcp_key - GRAFANA_COM_API_KEY: - from_secret: grafana_api_key - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-grafanacom-oss - depends_on: - grabpl - failure: ignore image: us.gcr.io/kubernetes-dev/package-publish:latest name: publish-linux-packages-deb privileged: true @@ -3841,7 +4541,6 @@ steps: target_bucket: grafana-packages - depends_on: - grabpl - failure: ignore image: us.gcr.io/kubernetes-dev/package-publish:latest name: publish-linux-packages-rpm privileged: true @@ -3861,6 +4560,18 @@ steps: service_account_json: from_secret: packages_service_account target_bucket: grafana-packages +- commands: + - ./bin/build publish grafana-com --edition oss ${DRONE_TAG} + depends_on: + - publish-linux-packages-deb + - publish-linux-packages-rpm + environment: + GCP_KEY: + from_secret: gcp_key + GRAFANA_COM_API_KEY: + from_secret: grafana_api_key + image: grafana/grafana-ci-deploy:1.3.3 + name: publish-grafanacom-oss trigger: event: - promote @@ -3878,6 +4589,8 @@ depends_on: - publish-artifacts-public - publish-docker-oss-public - publish-docker-enterprise-public +environment: + EDITION: enterprise image_pull_secrets: - dockerconfigjson kind: pipeline @@ -3891,7 +4604,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -3902,37 +4615,8 @@ steps: CGO_ENABLED: 0 image: golang:1.19.4 name: compile-build-cmd -- commands: - - ./bin/build publish packages --edition enterprise --gcp-key /tmp/gcpkey.json ${DRONE_TAG} - depends_on: - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_COM_API_KEY: - from_secret: grafana_api_key - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-packages-enterprise -- commands: - - ./bin/build publish grafana-com --edition enterprise ${DRONE_TAG} - depends_on: - - publish-packages-enterprise - environment: - GCP_KEY: - from_secret: gcp_key - GRAFANA_COM_API_KEY: - from_secret: grafana_api_key - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-grafanacom-enterprise - depends_on: - grabpl - failure: ignore image: us.gcr.io/kubernetes-dev/package-publish:latest name: publish-linux-packages-deb privileged: true @@ -3954,7 +4638,6 @@ steps: target_bucket: grafana-packages - depends_on: - grabpl - failure: ignore image: us.gcr.io/kubernetes-dev/package-publish:latest name: publish-linux-packages-rpm privileged: true @@ -3974,6 +4657,18 @@ steps: service_account_json: from_secret: packages_service_account target_bucket: grafana-packages +- commands: + - ./bin/build publish grafana-com --edition enterprise ${DRONE_TAG} + depends_on: + - publish-linux-packages-deb + - publish-linux-packages-rpm + environment: + GCP_KEY: + from_secret: gcp_key + GRAFANA_COM_API_KEY: + from_secret: grafana_api_key + image: grafana/grafana-ci-deploy:1.3.3 + name: publish-grafanacom-enterprise trigger: event: - promote @@ -3988,6 +4683,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: enterprise image_pull_secrets: - dockerconfigjson kind: pipeline @@ -4001,18 +4698,51 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - - ./bin/grabpl artifacts-page + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + - cd grafana-enterprise + - git checkout ${DRONE_TAG} + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:v1.7.1 + name: clone-enterprise +- commands: + - mv bin/grabpl /tmp/ + - rmdir bin + - mv grafana-enterprise /tmp/ + - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise + ${DRONE_TAG} + - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json + - mkdir bin + - mv /tmp/grabpl bin/ depends_on: - - grabpl + - clone-enterprise + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:v1.7.1 + name: init-enterprise +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: + - init-enterprise + environment: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd +- commands: + - ./bin/build artifacts-page + depends_on: + - compile-build-cmd environment: GCP_KEY: from_secret: gcp_key - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: artifacts-page trigger: event: @@ -4042,11 +4772,11 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -4057,18 +4787,18 @@ steps: in output.' - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd @@ -4082,7 +4812,7 @@ steps: depends_on: - wire-install - compile-build-cmd - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-backend - commands: - ./bin/build build-frontend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} @@ -4091,7 +4821,7 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-frontend - commands: - ./bin/build build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} @@ -4100,7 +4830,7 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-frontend-packages - commands: - ./bin/build build-plugins --jobs 8 --edition oss @@ -4110,7 +4840,7 @@ steps: environment: GRAFANA_API_KEY: from_secret: grafana_api_key - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-plugins - commands: - ./bin/build package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --sign @@ -4128,14 +4858,14 @@ steps: from_secret: packages_gpg_public_key GRAFANA_API_KEY: from_secret: grafana_api_key - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: package - commands: - ls dist/*.tar.gz* - cp dist/*.tar.gz* packaging/docker/ depends_on: - package - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: copy-packages-for-docker - commands: - ./bin/build build-docker --edition oss --shouldSave @@ -4174,7 +4904,7 @@ steps: environment: ARCH: linux-amd64 PORT: 3001 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: grafana-server - commands: - apt-get install -y netcat @@ -4215,8 +4945,6 @@ steps: - commands: - apt-get update - apt-get install -yq zip - - ls -lah ./e2e - - find ./e2e -type f -name "*.mp4" - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ @@ -4238,7 +4966,8 @@ steps: from_secret: gcp_upload_artifacts_key GITHUB_TOKEN: from_secret: github_token - image: google/cloud-sdk:367.0.0 + failure: ignore + image: google/cloud-sdk:406.0.0 name: e2e-tests-artifacts-upload when: status: @@ -4246,20 +4975,20 @@ steps: - failure - commands: - yarn storybook:build - - ./bin/grabpl verify-storybook + - ./bin/build verify-storybook depends_on: - build-frontend - build-frontend-packages environment: NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-storybook when: paths: include: - packages/grafana-ui/** - commands: - - ./bin/grabpl upload-cdn --edition oss + - ./bin/build upload-cdn --edition oss depends_on: - grafana-server environment: @@ -4273,7 +5002,7 @@ steps: repo: - grafana/grafana - commands: - - ./bin/grabpl upload-packages --edition oss + - ./bin/build upload-packages --edition oss depends_on: - end-to-end-tests-dashboards-suite - end-to-end-tests-panels-suite @@ -4307,6 +5036,8 @@ volumes: clone: retries: 3 depends_on: [] +environment: + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -4320,25 +5051,24 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - yarn install --immutable depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install - commands: - yarn betterer ci depends_on: - yarn-install - failure: ignore - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: betterer-frontend - commands: - yarn run ci:test-frontend @@ -4346,7 +5076,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-frontend trigger: ref: @@ -4375,7 +5105,7 @@ services: [] steps: - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd @@ -4391,25 +5121,25 @@ steps: in output.' - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - - go test -short -covermode=atomic -timeout=30m ./pkg/... + - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-backend - commands: - - go test -run Integration -covermode=atomic -timeout=30m ./pkg/... + - go test -run Integration -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-backend-integration trigger: ref: @@ -4458,13 +5188,13 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -4473,13 +5203,13 @@ steps: in output.' - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - apt-get update @@ -4488,14 +5218,14 @@ steps: - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=30m {}' + -timeout=5m {}' depends_on: - wire-install environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: postgres-integration-tests - commands: - apt-get update @@ -4505,13 +5235,13 @@ steps: -prootpass - go clean -testcache - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=30m {}' + -timeout=5m {}' depends_on: - wire-install environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: mysql-integration-tests trigger: ref: @@ -4553,7 +5283,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: windows-init @@ -4603,13 +5333,13 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" @@ -4618,7 +5348,7 @@ steps: environment: GITHUB_TOKEN: from_secret: github_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: clone-enterprise - commands: - mv bin/grabpl /tmp/ @@ -4633,7 +5363,7 @@ steps: environment: GITHUB_TOKEN: from_secret: github_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: init-enterprise - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd @@ -4647,13 +5377,13 @@ steps: - make gen-go depends_on: - init-enterprise - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - yarn install --immutable depends_on: - init-enterprise - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -4663,14 +5393,14 @@ steps: - CODEGEN_VERIFY=1 make gen-cue depends_on: - init-enterprise - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - ./bin/build build-backend --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} depends_on: - wire-install - compile-build-cmd - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-backend - commands: - ./bin/build build-frontend --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} @@ -4679,7 +5409,7 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-frontend - commands: - ./bin/build build-frontend-packages --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} @@ -4688,7 +5418,7 @@ steps: - yarn-install environment: NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-frontend-packages - commands: - ./bin/build build-plugins --jobs 8 --edition enterprise @@ -4698,16 +5428,8 @@ steps: environment: GRAFANA_API_KEY: from_secret: grafana_api_key - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: build-plugins -- commands: - - ./bin/build build-backend --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER} - --variants linux-amd64 - depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.6.6 - name: build-backend-enterprise2 - commands: - ./bin/build package --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} --sign @@ -4716,7 +5438,6 @@ steps: - build-backend - build-frontend - build-frontend-packages - - build-backend-enterprise2 environment: GPG_KEY_PASSWORD: from_secret: packages_gpg_passphrase @@ -4726,14 +5447,14 @@ steps: from_secret: packages_gpg_public_key GRAFANA_API_KEY: from_secret: grafana_api_key - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: package - commands: - ls dist/*.tar.gz* - cp dist/*.tar.gz* packaging/docker/ depends_on: - package - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: copy-packages-for-docker - commands: - ./bin/build build-docker --edition enterprise --shouldSave @@ -4773,7 +5494,7 @@ steps: ARCH: linux-amd64 PORT: 3001 RUNDIR: scripts/grafana-server/tmp-grafana-enterprise - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: grafana-server - commands: - apt-get install -y netcat @@ -4814,8 +5535,6 @@ steps: - commands: - apt-get update - apt-get install -yq zip - - ls -lah ./e2e - - find ./e2e -type f -name "*.mp4" - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ @@ -4837,14 +5556,15 @@ steps: from_secret: gcp_upload_artifacts_key GITHUB_TOKEN: from_secret: github_token - image: google/cloud-sdk:367.0.0 + failure: ignore + image: google/cloud-sdk:406.0.0 name: e2e-tests-artifacts-upload when: status: - success - failure - commands: - - ./bin/grabpl upload-cdn --edition enterprise + - ./bin/build upload-cdn --edition enterprise depends_on: - package environment: @@ -4858,7 +5578,7 @@ steps: repo: - grafana/grafana - commands: - - ./bin/grabpl upload-packages --edition enterprise + - ./bin/build upload-packages --edition enterprise depends_on: - package environment: @@ -4871,48 +5591,6 @@ steps: when: repo: - grafana/grafana -- commands: - - ./bin/build package --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER} - --variants linux-amd64 --sign - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - - build-backend-enterprise2 - environment: - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.6.6 - name: package-enterprise2 -- commands: - - ./bin/grabpl upload-cdn --edition enterprise2 - depends_on: - - package-enterprise2 - environment: - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-cdn-assets-enterprise2 -- commands: - - ./bin/grabpl upload-packages --edition enterprise2 - depends_on: - - package-enterprise2 - environment: - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-packages-enterprise2 trigger: ref: - refs/heads/v[0-9]* @@ -4931,6 +5609,8 @@ volumes: clone: disable: true depends_on: [] +environment: + EDITION: enterprise image_pull_secrets: - dockerconfigjson kind: pipeline @@ -4949,7 +5629,7 @@ steps: environment: GITHUB_TOKEN: from_secret: github_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: clone-enterprise - commands: - mv bin/grabpl /tmp/ @@ -4964,15 +5644,15 @@ steps: environment: GITHUB_TOKEN: from_secret: github_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: init-enterprise - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -4980,15 +5660,14 @@ steps: - yarn install --immutable depends_on: - init-enterprise - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: yarn-install - commands: - yarn betterer ci depends_on: - init-enterprise - yarn-install - failure: ignore - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: betterer-frontend - commands: - yarn run ci:test-frontend @@ -4997,7 +5676,7 @@ steps: - yarn-install environment: TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-frontend trigger: ref: @@ -5031,11 +5710,11 @@ steps: environment: GITHUB_TOKEN: from_secret: github_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: clone-enterprise - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -5052,11 +5731,11 @@ steps: environment: GITHUB_TOKEN: from_secret: github_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: init-enterprise - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd @@ -5074,120 +5753,25 @@ steps: - CODEGEN_VERIFY=1 make gen-cue depends_on: - init-enterprise - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - - go test -short -covermode=atomic -timeout=30m ./pkg/... + - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-backend - commands: - - go test -run Integration -covermode=atomic -timeout=30m ./pkg/... + - go test -run Integration -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: grafana/build-container:1.6.6 - name: test-backend-integration -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - disable: true -depends_on: [] -environment: - EDITION: enterprise2 -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-enterprise2-test-backend -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout ${DRONE_BRANCH} - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.6.6 - name: clone-enterprise -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - mv bin/grabpl /tmp/ - - rmdir bin - - mv grafana-enterprise /tmp/ - - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise - - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - - mkdir bin - - mv /tmp/grabpl bin/ - depends_on: - - clone-enterprise - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.6.6 - name: init-enterprise -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 - name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: - - init-enterprise - environment: - CGO_ENABLED: 0 - image: golang:1.19.4 - name: compile-build-cmd -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: - - init-enterprise - image: grafana/build-container:1.6.6 - name: verify-gen-cue -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.6.6 - name: wire-install -- commands: - - go test -tags=pro -covermode=atomic -timeout=30m ./pkg/... - depends_on: - - wire-install - image: grafana/build-container:1.6.6 - name: test-backend-enterprise2 -- commands: - - go test -run Integration -covermode=atomic -timeout=30m ./pkg/... - depends_on: - - wire-install - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: test-backend-integration trigger: ref: @@ -5242,13 +5826,13 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - echo $DRONE_RUNNER_NAME - image: alpine:3.15.6 + image: alpine:3.17.1 name: identify-runner - commands: - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" @@ -5257,7 +5841,7 @@ steps: environment: GITHUB_TOKEN: from_secret: github_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: clone-enterprise - commands: - mv bin/grabpl /tmp/ @@ -5272,7 +5856,7 @@ steps: environment: GITHUB_TOKEN: from_secret: github_token - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: init-enterprise - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -5282,13 +5866,13 @@ steps: - CODEGEN_VERIFY=1 make gen-cue depends_on: - init-enterprise - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: verify-gen-cue - commands: - make gen-go depends_on: - verify-gen-cue - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: wire-install - commands: - apt-get update @@ -5297,14 +5881,14 @@ steps: - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - go clean -testcache - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=30m {}' + -timeout=5m {}' depends_on: - wire-install environment: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: postgres-integration-tests - commands: - apt-get update @@ -5314,31 +5898,35 @@ steps: -prootpass - go clean -testcache - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=30m {}' + -timeout=5m {}' depends_on: - wire-install environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: mysql-integration-tests - commands: - dockerize -wait tcp://redis:6379/0 -timeout 120s - - ./bin/grabpl integration-tests + - go clean -testcache + - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic + -timeout=5m {}' depends_on: - wire-install environment: REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: redis-integration-tests - commands: - dockerize -wait tcp://memcached:11211 -timeout 120s - - ./bin/grabpl integration-tests + - go clean -testcache + - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic + -timeout=5m {}' depends_on: - wire-install environment: MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.6.6 + image: grafana/build-container:v1.7.1 name: memcached-integration-tests trigger: ref: @@ -5380,7 +5968,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.15/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/windows/grabpl.exe -OutFile grabpl.exe - git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git" - cd grafana-enterprise @@ -5432,6 +6020,254 @@ volumes: path: /var/run/docker.sock name: docker --- +clone: + disable: true +depends_on: [] +environment: + EDITION: enterprise2 +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: release-branch-enterprise2-build-e2e-publish +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.21/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.17.1 + name: identify-runner +- commands: + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + - cd grafana-enterprise + - git checkout ${DRONE_BRANCH} + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:v1.7.1 + name: clone-enterprise +- commands: + - mv bin/grabpl /tmp/ + - rmdir bin + - mv grafana-enterprise /tmp/ + - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise + - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json + - mkdir bin + - mv /tmp/grabpl bin/ + depends_on: + - clone-enterprise + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:v1.7.1 + name: init-enterprise +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: + - init-enterprise + environment: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd +- commands: + - make gen-go + depends_on: + - init-enterprise + image: grafana/build-container:v1.7.1 + name: wire-install +- commands: + - yarn install --immutable + depends_on: + - init-enterprise + image: grafana/build-container:v1.7.1 + name: yarn-install +- commands: + - '# It is required that code generated from Thema/CUE be committed and in sync + with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - CODEGEN_VERIFY=1 make gen-cue + depends_on: + - init-enterprise + image: grafana/build-container:v1.7.1 + name: verify-gen-cue +- commands: + - ./bin/build build-frontend --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} + depends_on: + - compile-build-cmd + - yarn-install + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: grafana/build-container:v1.7.1 + name: build-frontend +- commands: + - ./bin/build build-frontend-packages --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} + depends_on: + - compile-build-cmd + - yarn-install + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: grafana/build-container:v1.7.1 + name: build-frontend-packages +- commands: + - ./bin/build build-plugins --jobs 8 --edition enterprise + depends_on: + - compile-build-cmd + - yarn-install + environment: + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:v1.7.1 + name: build-plugins +- commands: + - ./bin/build build-backend --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER} + --variants linux-amd64 + depends_on: + - wire-install + - compile-build-cmd + image: grafana/build-container:v1.7.1 + name: build-backend-enterprise2 +- commands: + - ./bin/build package --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER} + --sign + depends_on: + - build-plugins + - build-backend-enterprise2 + - build-frontend + - build-frontend-packages + environment: + GPG_KEY_PASSWORD: + from_secret: packages_gpg_passphrase + GPG_PRIV_KEY: + from_secret: packages_gpg_private_key + GPG_PUB_KEY: + from_secret: packages_gpg_public_key + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:v1.7.1 + name: package-enterprise2 +- commands: + - ./bin/build upload-cdn --edition enterprise2 + depends_on: + - package-enterprise2 + environment: + ENTERPRISE2_CDN_PATH: + from_secret: enterprise2-cdn-path + GCP_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.3 + name: upload-cdn-assets-enterprise2 +- commands: + - ls dist/*.tar.gz* + - cp dist/*.tar.gz* packaging/docker/ + depends_on: + - package-enterprise2 + image: grafana/build-container:v1.7.1 + name: copy-packages-for-docker +- commands: + - ./bin/build build-docker --edition enterprise2 --shouldSave + depends_on: + - copy-packages-for-docker + - compile-build-cmd + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: build-docker-images + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build build-docker --edition enterprise2 --shouldSave --ubuntu + depends_on: + - copy-packages-for-docker + - compile-build-cmd + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: build-docker-images-ubuntu + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build artifacts docker fetch --edition enterprise2 + depends_on: + - build-docker-images + - build-docker-images-ubuntu + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: fetch-images-enterprise2 + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build artifacts docker publish-enterprise2 --dockerhub-repo $${DOCKER_ENTERPRISE2_REPO} + depends_on: + - fetch-images-enterprise2 + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key_hg + image: google/cloud-sdk + name: publish-images-enterprise2 + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build upload-packages --edition enterprise2 + depends_on: + - package-enterprise2 + environment: + GCP_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.3 + name: upload-packages-enterprise2 +trigger: + ref: + - refs/heads/v[0-9]* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory +--- clone: retries: 3 kind: pipeline @@ -5554,6 +6390,37 @@ trigger: event: cron type: docker --- +clone: + retries: 3 +kind: pipeline +name: grafana-com-nightly +platform: + arch: amd64 + os: linux +steps: +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] + environment: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd +- commands: + - ./bin/build publish grafana-com --edition oss + depends_on: + - compile-build-cmd + environment: + GCP_KEY: + from_secret: gcp_key + GRAFANA_COM_API_KEY: + from_secret: grafana_api_key + image: grafana/grafana-ci-deploy:1.3.3 + name: post-to-grafana-com +trigger: + cron: grafana-com-nightly + event: cron +type: docker +--- get: name: .dockerconfigjson path: secret/data/common/gcr @@ -5657,6 +6524,6 @@ kind: secret name: aws_secret_access_key --- kind: signature -hmac: 613244b7c6b51cdcbfb71f124850e27da526b72b2c07c7c660356b484b3e7cb4 +hmac: d77fb0a3aa52ab5e6d17f5626a3988288cc193524a0bbc2d3e1fa98e779ae340 ... diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b55814b198c..3377c28c7da 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,205 +12,601 @@ # This should make it easy to add new rules without breaking existing ones. # Documentation -/docs/ @grafana/docs-squad -/contribute/ @grafana/docs-squad -/docs/sources/developers/plugins/ @grafana/docs-squad @grafana/plugins-platform-frontend @grafana/plugins-platform-backend -/docs/sources/developers/plugins/backend @grafana/docs-squad @grafana/plugins-platform-backend -# Administration, data sources, querying, release updates: Garrett Guillotte +/docs/ @grafana/docs-grafana +/contribute/ @grafana/docs-grafana +/docs/sources/developers/plugins/ @grafana/docs-grafana @grafana/plugins-platform-frontend @grafana/plugins-platform-backend +/docs/sources/developers/plugins/backend/ @grafana/docs-grafana @grafana/plugins-platform-backend +/.changelog-archive @grafana/docs-grafana +CHANGELOG.md @grafana/docs-grafana +CODE_OF_CONDUCT.md @grafana/docs-grafana +CONTRIBUTING.md @grafana/docs-grafana +GOVERNANCE.md @RichiH +HALL_OF_FAME.md @grafana/docs-grafana +ISSUE_TRIAGE.md @grafana/grafana-community-support +LICENSE @torkelo +LICENSING.md @torkelo +MAINTAINERS.md @RichiH +NOTICE.md @torkelo +README.md @grafana/docs-grafana +ROADMAP.md @torkelo +SECURITY.md @grafana/security-team +SUPPORT.md @torkelo +UPGRADING_DEPENDENCIES.md @grafana/docs-grafana +WORKFLOW.md @torkelo + + # Set up, dashboards/visualization, best practices: Chris Moyer # Alerting: Brenda Muir -/docs/sources/administration/ @gguillotte-grafana -/docs/sources/alerting @brendamuir -/docs/sources/best-practices/ @chri2547 +/docs/sources/administration/ @Eve832 @GrafanaWriter +/docs/sources/alerting/ @brendamuir /docs/sources/dashboards/ @chri2547 -/docs/sources/datasources/ @gguillotte-grafana -/docs/sources/enterprise/ @gguillotte-grafana -/docs/sources/explore/ @gguillotte-grafana +/docs/sources/datasources/ @Eve832 @GrafanaWriter +/docs/sources/explore/ @Eve832 @GrafanaWriter /docs/sources/getting-started/ @chri2547 -/docs/sources/old-alerting @brendamuir -/docs/sources/panels/ @chri2547 -/docs/sources/release-notes/ @gguillotte-grafana -/docs/sources/visualization/ @chri2547 -/docs/sources/whatsnew/ @gguillotte-grafana +/docs/sources/old-alerting/ @brendamuir +/docs/sources/release-notes/ @Eve832 @GrafanaWriter +/docs/sources/setup-grafana/ @chri2547 +/docs/sources/whatsnew/ @Eve832 @GrafanaWriter # Backend code -*.go @grafana/backend-platform -go.mod @grafana/backend-platform -go.sum @grafana/backend-platform -/.bingo @grafana/backend-platform +/go.mod @grafana/backend-platform +/go.sum @grafana/backend-platform +/.bingo/ @grafana/backend-platform +/pkg/README.md @grafana/backend-platform +/pkg/ruleguard.rules.go @grafana/backend-platform +/.bra.toml @grafana/backend-platform +/.golangci.toml @grafana/backend-platform +/build.go @grafana/backend-platform +/pkg/api/ @grafana/backend-platform +/pkg/bus/ @grafana/backend-platform +/pkg/cmd/ @grafana/backend-platform +/pkg/components/apikeygen/ @grafana/grafana-authnz-team +/pkg/components/apikeygenprefixed/ @grafana/grafana-authnz-team +/pkg/components/dashdiffs/ @grafana/backend-platform +/pkg/components/imguploader/ @grafana/backend-platform +/pkg/components/loki/ @grafana/backend-platform +/pkg/components/null/ @grafana/backend-platform +/pkg/components/simplejson/ @grafana/backend-platform +/pkg/events/ @grafana/backend-platform +/pkg/extensions/ @grafana/backend-platform +/pkg/ifaces/ @grafana/backend-platform +/pkg/infra/appcontext/ @grafana/backend-platform +/pkg/infra/db/ @grafana/backend-platform +/pkg/infra/grn/ @grafana/backend-platform +/pkg/infra/localcache/ @grafana/backend-platform +/pkg/infra/log/ @grafana/backend-platform +/pkg/infra/metrics/ @grafana/backend-platform +/pkg/infra/network/ @grafana/backend-platform +/pkg/infra/process/ @grafana/backend-platform +/pkg/infra/remotecache/ @grafana/backend-platform +/pkg/infra/serverlock/ @grafana/backend-platform +/pkg/infra/slugify/ @grafana/backend-platform +/pkg/infra/tracing/ @grafana/backend-platform +/pkg/infra/usagestats/ @grafana/backend-platform +/pkg/middleware/ @grafana/backend-platform +/pkg/mocks/ @grafana/backend-platform +/pkg/models/ @grafana/backend-platform +/pkg/server/ @grafana/backend-platform +/pkg/services/annotations/ @grafana/backend-platform +/pkg/services/apikey/ @grafana/backend-platform +/pkg/services/cleanup/ @grafana/backend-platform +/pkg/services/comments/ @grafana/backend-platform +/pkg/services/contexthandler/ @grafana/backend-platform +/pkg/services/correlations/ @grafana/backend-platform +/pkg/services/dashboardimport/ @grafana/backend-platform +/pkg/services/dashboards/ @grafana/backend-platform +/pkg/services/dashboardsnapshots/ @grafana/backend-platform +/pkg/services/dashboardversion/ @grafana/backend-platform +/pkg/services/encryption/ @grafana/backend-platform +/pkg/services/featuremgmt/ @grafana/backend-platform +/pkg/services/folder/ @grafana/backend-platform +/pkg/services/hooks/ @grafana/backend-platform +/pkg/services/kmsproviders/ @grafana/backend-platform +/pkg/services/licensing/ @grafana/backend-platform +/pkg/services/navtree/ @grafana/backend-platform +/pkg/services/notifications/ @grafana/backend-platform +/pkg/services/org/ @grafana/backend-platform +/pkg/services/playlist/ @grafana/backend-platform +/pkg/services/plugindashboards/ @grafana/backend-platform +/pkg/services/pluginsettings/ @grafana/backend-platform +/pkg/services/preference/ @grafana/backend-platform +/pkg/services/provisioning/ @grafana/backend-platform +/pkg/services/publicdashboards/ @grafana/dashboards-squad +/pkg/services/query/ @grafana/backend-platform +/pkg/services/queryhistory/ @grafana/backend-platform +/pkg/services/quota/ @grafana/backend-platform +/pkg/services/rendering/ @grafana/backend-platform +/pkg/services/screenshot/ @grafana/backend-platform +/pkg/services/search/ @grafana/backend-platform +/pkg/services/searchusers/ @grafana/backend-platform +/pkg/services/secrets/ @grafana/backend-platform +/pkg/services/shorturls/ @grafana/backend-platform +/pkg/services/sqlstore/ @grafana/backend-platform +/pkg/services/star/ @grafana/backend-platform +/pkg/services/stats/ @grafana/backend-platform +/pkg/services/tag/ @grafana/backend-platform +/pkg/services/team/ @grafana/grafana-authnz-team +/pkg/services/temp_user/ @grafana/backend-platform +/pkg/services/updatechecker/ @grafana/backend-platform +/pkg/services/user/ @grafana/backend-platform +/pkg/services/validations/ @grafana/backend-platform +/pkg/setting/ @grafana/backend-platform +/pkg/tests/ @grafana/backend-platform +/pkg/tsdb/grafanads/ @grafana/backend-platform +/pkg/tsdb/intervalv2/ @grafana/backend-platform +/pkg/tsdb/legacydata/ @grafana/backend-platform +/pkg/tsdb/opentsdb/ @grafana/backend-platform +/pkg/tsdb/sqleng/ @grafana/backend-platform +/pkg/util/ @grafana/backend-platform +/pkg/web/ @grafana/backend-platform + +/pkg/services/grpcserver/ @grafana/backend-platform +/pkg/infra/kvstore/ @grafana/backend-platform +/pkg/infra/fs/ @grafana/backend-platform +/pkg/infra/x/ @grafana/backend-platform + + +# devenv # Backend code, developers environment -/devenv/docker/blocks/auth @grafana/grafana-authnz-team +/devenv/docker/blocks/auth/ @grafana/grafana-authnz-team # Logs code, developers environment /devenv/docker/blocks/loki* @grafana/observability-logs /devenv/docker/blocks/elastic* @grafana/observability-logs +# Performance tests +/devenv/docker/loadtest-ts/ @grafana/multitenancy-squad + +/devenv/bulk-dashboards/ @grafana/dashboards-squad +/devenv/bulk_alerting_dashboards/ @grafana/alerting-squad-backend +/devenv/create_docker_compose.sh @grafana/backend-platform +/devenv/dashboards.yaml @grafana/dashboards-squad +/devenv/datasources.yaml @grafana/backend-platform +/devenv/datasources_docker.yaml @grafana/backend-platform +/devenv/dev-dashboards-without-uid/ @grafana/dashboards-squad +/devenv/dev-dashboards/ @grafana/dashboards-squad +/devenv/docker/blocks/alert_webhook_listener/ @grafana/alerting-squad-backend +/devenv/docker/blocks/clickhouse/ @grafana/partner-plugins +/devenv/docker/blocks/collectd/ @grafana/observability-metrics +/devenv/docker/blocks/grafana/ @grafana/grafana-as-code +/devenv/docker/blocks/graphite/ @grafana/observability-metrics +/devenv/docker/blocks/graphite09/ @grafana/observability-metrics +/devenv/docker/blocks/graphite1/ @grafana/observability-metrics +/devenv/docker/blocks/influxdb/ @grafana/observability-metrics +/devenv/docker/blocks/influxdb1/ @grafana/observability-metrics +/devenv/docker/blocks/jaeger/ @grafana/observability-traces-and-profiling +/devenv/docker/blocks/maildev/ @grafana/alerting-squad-frontend +/devenv/docker/blocks/memcached/ @grafana/backend-platform +/devenv/docker/blocks/mssql/ @grafana/grafana-bi-squad +/devenv/docker/blocks/mssql_arm64/ @grafana/grafana-bi-squad +/devenv/docker/blocks/mssql_tests/ @grafana/grafana-bi-squad +/devenv/docker/blocks/mssql_tls/ @grafana/grafana-bi-squad +/devenv/docker/blocks/mysql/ @grafana/grafana-bi-squad +/devenv/docker/blocks/mysql_exporter/ @grafana/grafana-bi-squad +/devenv/docker/blocks/mysql_opendata/ @grafana/grafana-bi-squad +/devenv/docker/blocks/mysql_tests/ @grafana/grafana-bi-squad +/devenv/docker/blocks/opentsdb/ @grafana/observability-metrics +/devenv/docker/blocks/phlare/ @grafana/observability-traces-and-profiling +/devenv/docker/blocks/postgres/ @grafana/grafana-bi-squad +/devenv/docker/blocks/postgres_tests/ @grafana/grafana-bi-squad +/devenv/docker/blocks/prometheus/ @grafana/observability-metrics +/devenv/docker/blocks/prometheus_random_data/ @grafana/observability-metrics +/devenv/docker/blocks/redis/ @bergquist +/devenv/docker/blocks/sensugo/ @grafana/backend-platform +/devenv/docker/blocks/slow_proxy/ @bergquist +/devenv/docker/blocks/smtp/ @bergquist +/devenv/docker/blocks/tempo/ @grafana/observability-traces-and-profiling +/devenv/docker/blocks/traefik/ @mckn +/devenv/docker/blocks/zipkin/ @grafana/observability-traces-and-profiling +/devenv/docker/buildcontainer/ @bergquist +/devenv/docker/compose_header.yml @grafana/backend-platform +/devenv/docker/debtest/ @bergquist +/devenv/docker/ha-test-unified-alerting/ @grafana/alerting-squad-backend +/devenv/docker/ha_test/ @grafana/backend-platform +/devenv/docker/loadtest/ @grafana/backend-platform +/devenv/docker/rpmtest/ @grafana/backend-platform +/devenv/jsonnet/ @grafana/grafana-edge-squad +/devenv/local-npm/ @grafana/frontend-ops +/devenv/vscode/ @grafana/frontend-ops +/devenv/setup.sh @grafana/backend-platform + +# Emails +/emails/ @grafana/alerting-squad-frontend + +#Packaging +/packaging/ @DanCech + + # Continuous Integration .drone.yml @grafana/grafana-release-eng .drone.star @grafana/grafana-release-eng /scripts/drone/ @grafana/grafana-release-eng /pkg/build/ @grafana/grafana-release-eng +/.dockerignore @grafana/grafana-release-eng +/Dockerfile @grafana/grafana-release-eng +/Makefile @grafana/grafana-release-eng +/scripts/build/ @grafana/grafana-release-eng -# Cloud Datasources backend code -/pkg/tsdb/cloudwatch @grafana/aws-plugins -/pkg/tsdb/azuremonitor @grafana/cloud-provider-plugins -/pkg/tsdb/cloudmonitoring @grafana/cloud-provider-plugins +# OSS Plugin Partnerships backend code +/pkg/tsdb/cloudwatch/ @grafana/aws-plugins +/pkg/tsdb/azuremonitor/ @grafana/partner-plugins +/pkg/tsdb/cloudmonitoring/ @grafana/partner-plugins # Observability backend code -/pkg/tsdb/prometheus @grafana/observability-metrics -/pkg/tsdb/influxdb @grafana/observability-metrics -/pkg/tsdb/elasticsearch @grafana/observability-logs -/pkg/tsdb/graphite @grafana/observability-metrics -/pkg/tsdb/jaeger @grafana/observability-traces-and-profiling -/pkg/tsdb/loki @grafana/observability-logs -/pkg/tsdb/zipkin @grafana/observability-traces-and-profiling -/pkg/tsdb/tempo @grafana/observability-traces-and-profiling +/pkg/tsdb/prometheus/ @grafana/observability-metrics +/pkg/tsdb/influxdb/ @grafana/observability-metrics +/pkg/tsdb/elasticsearch/ @grafana/observability-logs +/pkg/tsdb/graphite/ @grafana/observability-metrics +/pkg/tsdb/loki/ @grafana/observability-logs +/pkg/tsdb/tempo/ @grafana/observability-traces-and-profiling +/pkg/tsdb/phlare/ @grafana/observability-traces-and-profiling +/pkg/tsdb/parca/ @grafana/observability-traces-and-profiling # BI backend code -/pkg/tsdb/mysql @grafana/grafana-bi-squad -/pkg/tsdb/postgres @grafana/grafana-bi-squad -/pkg/tsdb/mssql @grafana/grafana-bi-squad +/pkg/tsdb/mysql/ @grafana/grafana-bi-squad +/pkg/tsdb/postgres/ @grafana/grafana-bi-squad +/pkg/tsdb/mssql/ @grafana/grafana-bi-squad # Database migrations -/pkg/services/sqlstore/migrations @grafana/backend-platform @grafana/hosted-grafana-team +/pkg/services/sqlstore/migrations/ @grafana/backend-platform @grafana/hosted-grafana-team *_mig.go @grafana/backend-platform @grafana/hosted-grafana-team -# Grafana edge -/pkg/services/live/ @grafana/grafana-edge-squad -/pkg/services/searchV2/ @grafana/grafana-edge-squad -/pkg/services/store/ @grafana/grafana-edge-squad -/pkg/services/export/ @grafana/grafana-edge-squad -/pkg/infra/filestore/ @grafana/grafana-edge-squad -/pkg/tsdb/testdatasource/sims/ @grafana/grafana-edge-squad +# Grafana multitenancy +/pkg/services/live/ @grafana/multitenancy-squad +/pkg/services/searchV2/ @grafana/multitenancy-squad +/pkg/services/store/ @grafana/multitenancy-squad +/pkg/services/querylibrary/ @grafana/multitenancy-squad +/pkg/services/export/ @grafana/multitenancy-squad +/pkg/infra/filestorage/ @grafana/multitenancy-squad +/pkg/util/converter/ @grafana/multitenancy-squad # Alerting -/pkg/services/ngalert @grafana/alerting-squad-backend -/pkg/services/sqlstore/migrations/ualert @grafana/alerting-squad-backend -/pkg/services/alerting @grafana/alerting-squad-backend -/pkg/tests/api/alerting @grafana/alerting-squad-backend -/public/app/features/alerting @grafana/alerting-squad-frontend +/pkg/services/ngalert/ @grafana/alerting-squad-backend +/pkg/services/sqlstore/migrations/ualert/ @grafana/alerting-squad-backend +/pkg/services/alerting/ @grafana/alerting-squad-backend +/pkg/tests/api/alerting/ @grafana/alerting-squad-backend +/public/app/features/alerting/ @grafana/alerting-squad-frontend # Library Services -/pkg/services/libraryelements @grafana/user-essentials -/pkg/services/librarypanels @grafana/user-essentials +/pkg/services/libraryelements/ @grafana/user-essentials +/pkg/services/librarypanels/ @grafana/user-essentials # Plugins -/pkg/api/pluginproxy @grafana/plugins-platform-backend -/pkg/plugins @grafana/plugins-platform-backend -/pkg/services/datasourceproxy @grafana/plugins-platform-backend -/pkg/services/datasources @grafana/plugins-platform-backend -/pkg/plugins/pfs @grafana/plugins-platform-backend @grafana/grafana-as-code +/pkg/api/pluginproxy/ @grafana/plugins-platform-backend +/pkg/infra/httpclient/ @grafana/plugins-platform-backend +/pkg/plugins/ @grafana/plugins-platform-backend +/pkg/services/datasourceproxy/ @grafana/plugins-platform-backend +/pkg/services/datasources/ @grafana/plugins-platform-backend +/pkg/services/pluginsintegration/ @grafana/plugins-platform-backend +/pkg/plugins/pfs/ @grafana/plugins-platform-backend @grafana/grafana-as-code +/pkg/tsdb/testdatasource/ @grafana/plugins-platform-backend # Dashboard previews / crawler (behind feature flag) -/pkg/services/thumbs @grafana/grafana-edge-squad +/pkg/services/thumbs/ @grafana/multitenancy-squad # Backend code docs -/contribute/style-guides/backend.md @grafana/backend-platform -/contribute/architecture/backend @grafana/backend-platform -/contribute/engineering/backend @grafana/backend-platform +/contribute/backend/ @grafana/backend-platform -/e2e @grafana/user-essentials -/packages @grafana/user-essentials @grafana/plugins-platform-frontend @grafana/grafana-bi-squad -/packages/grafana-e2e-selectors @grafana/user-essentials -/packages/grafana-e2e @grafana/user-essentials -/packages/grafana-toolkit @grafana/plugins-platform-frontend -/packages/grafana-ui/.storybook @grafana/plugins-platform-frontend -/packages/grafana-ui/src/components/DateTimePickers @grafana/grafana-bi-squad -/packages/grafana-ui/src/components/GraphNG @grafana/grafana-bi-squad -/packages/grafana-ui/src/components/Logs @grafana/observability-logs -/packages/grafana-ui/src/components/Table @grafana/grafana-bi-squad -/packages/grafana-ui/src/components/TimeSeries @grafana/grafana-bi-squad -/packages/grafana-ui/src/components/uPlot @grafana/grafana-bi-squad -/packages/grafana-ui/src/utils/storybook @grafana/plugins-platform-frontend -/packages/jaeger-ui-components/ @grafana/observability-traces-and-profiling -/plugins-bundled @grafana/plugins-platform-frontend -# public folder -/public/app/core/components/TimePicker @grafana/grafana-bi-squad -/public/app/core/components/Layers @grafana/grafana-edge-squad -/public/app/features/canvas/ @grafana/grafana-edge-squad -/public/app/features/comments/ @grafana/grafana-edge-squad -/public/app/features/dimensions/ @grafana/grafana-edge-squad -/public/app/features/geo/ @grafana/grafana-edge-squad -/public/app/features/storage/ @grafana/grafana-edge-squad -/public/app/features/live/ @grafana/grafana-edge-squad -/public/app/features/explore/ @grafana/observability-experience-squad -/public/app/features/plugins @grafana/plugins-platform-frontend -/public/app/features/transformers/spatial @grafana/grafana-edge-squad -/public/app/plugins/panel/alertlist @grafana/alerting-squad-frontend -/public/app/plugins/panel/barchart @grafana/grafana-bi-squad -/public/app/plugins/panel/heatmap @grafana/grafana-bi-squad -/public/app/plugins/panel/histogram @grafana/grafana-bi-squad -/public/app/plugins/panel/logs @grafana/observability-logs -/public/app/plugins/panel/nodeGraph @grafana/observability-traces-and-profiling -/public/app/plugins/panel/traces @grafana/observability-traces-and-profiling -/public/app/plugins/panel/piechart @grafana/grafana-bi-squad -/public/app/plugins/panel/state-timeline @grafana/grafana-bi-squad -/public/app/plugins/panel/status-history @grafana/grafana-bi-squad -/public/app/plugins/panel/table @grafana/grafana-bi-squad -/public/app/plugins/panel/timeseries @grafana/grafana-bi-squad -/public/app/plugins/panel/geomap @grafana/grafana-edge-squad -/public/app/plugins/panel/canvas @grafana/grafana-edge-squad -/public/app/plugins/panel/candlestick @grafana/grafana-edge-squad -/public/app/plugins/panel/icon @grafana/grafana-edge-squad -/scripts/build/release-packages.sh @grafana/plugins-platform-frontend -/scripts/circle-release-next-packages.sh @grafana/plugins-platform-frontend -/scripts/ci-frontend-metrics.sh @grafana/user-essentials @grafana/plugins-platform-frontend @grafana/grafana-bi-squad -/scripts/grunt @grafana/frontend-ops -/scripts/webpack @grafana/frontend-ops -/scripts/generate-a11y-report.sh @grafana/user-essentials + +/crowdin.yml @grafana/user-essentials +/public/locales/ @grafana/user-essentials +/public/app/core/internationalization/ @grafana/user-essentials +/e2e/ @grafana/user-essentials +/e2e/cloud-plugins-suite/ @grafana/partner-plugins +/packages/ @grafana/user-essentials @grafana/plugins-platform-frontend @grafana/grafana-bi-squad +/packages/grafana-e2e-selectors/ @grafana/user-essentials +/packages/grafana-e2e/ @grafana/user-essentials +/packages/grafana-toolkit/ @grafana/plugins-platform-frontend +/packages/grafana-ui/.storybook/ @grafana/plugins-platform-frontend +/packages/grafana-ui/src/components/DateTimePickers/ @grafana/user-essentials +/packages/grafana-ui/src/components/GraphNG/ @grafana/grafana-bi-squad +/packages/grafana-ui/src/components/Logs/ @grafana/observability-logs +/packages/grafana-ui/src/components/Table/ @grafana/grafana-bi-squad +/packages/grafana-ui/src/components/TimeSeries/ @grafana/grafana-bi-squad +/packages/grafana-ui/src/components/uPlot/ @grafana/grafana-bi-squad +/packages/grafana-ui/src/utils/storybook/ @grafana/plugins-platform-frontend +/packages/grafana-data/src/**/*logs* @grafana/observability-logs +/plugins-bundled/ @grafana/plugins-platform-frontend + + +# root files, mostly frontend +.browserslistrc @grafana/frontend-ops package.json @grafana/frontend-ops tsconfig.json @grafana/frontend-ops +/.editorconfig @grafana/frontend-ops +/.eslintignore @grafana/frontend-ops +/.gitattributes @grafana/frontend-ops +/.gitignore @grafana/frontend-ops +/.husky/pre-commit @grafana/frontend-ops +/.nvmrc @grafana/frontend-ops +/.prettierignore @grafana/frontend-ops +/.yarn @grafana/frontend-ops +/.yarnrc.yml @grafana/frontend-ops +/yarn.lock @grafana/frontend-ops +/.linguirc @grafana/user-essentials +/babel.config.json @grafana/frontend-ops lerna.json @grafana/frontend-ops -.babelrc @grafana/frontend-ops -.prettierrc.js @grafana/frontend-ops -.eslintrc @grafana/frontend-ops +/.prettierrc.js @grafana/frontend-ops +/.eslintrc @grafana/frontend-ops +/.vim @zoltanbedi +/jest.config.js @grafana/frontend-ops +/latest.json @grafana/frontend-ops +/metadata.md @grafana/plugins-platform +/stylelint.config.js @grafana/frontend-ops +/tools/ @grafana/frontend-ops + + +# public folder +/public/app/core/ @grafana/user-essentials +/public/app/core/components/TimePicker/ @grafana/user-essentials +/public/app/core/components/Layers/ @grafana/grafana-edge-squad +/public/app/features/all.ts @grafana/user-essentials +/public/app/features/admin/ @grafana/grafana-authnz-team +/public/app/features/annotations/ @grafana/user-essentials +/public/app/features/api-keys/ @grafana/user-essentials +/public/app/features/canvas/ @grafana/grafana-edge-squad +/public/app/features/commandPalette/ @grafana/user-essentials +/public/app/features/comments/ @grafana/grafana-edge-squad +/public/app/features/connections/ @grafana/plugins-platform-frontend +/public/app/features/correlations/ @grafana/explore-squad +/public/app/features/dashboard/ @grafana/dashboards-squad +/public/app/features/datasources/ @grafana/user-essentials +/public/app/features/dimensions/ @grafana/grafana-edge-squad +/public/app/features/explore/ @grafana/explore-squad +/public/app/features/expressions/ @grafana/observability-metrics +/public/app/features/folders/ @grafana/user-essentials +/public/app/features/inspector/ @grafana/user-essentials +/public/app/features/invites/ @grafana/user-essentials +/public/app/features/geo/ @grafana/grafana-edge-squad +/public/app/features/library-panels/ @grafana/user-essentials +/public/app/features/logs/ @grafana/observability-logs +/public/app/features/live/ @grafana/multitenancy-squad +/public/app/features/manage-dashboards/ @grafana/dashboards-squad +/public/app/features/notifications/ @grafana/user-essentials +/public/app/features/org/ @grafana/user-essentials +/public/app/features/panel/ @grafana/user-essentials +/public/app/features/playlist/ @grafana/dashboards-squad +/public/app/features/plugins/ @grafana/plugins-platform-frontend +/public/app/features/profile/ @grafana/user-essentials +/public/app/features/runtime/ @ryantxu +/public/app/features/query/ @grafana/dashboards-squad +/public/app/features/query-library/ @grafana/grafana-edge-squad +/public/app/features/sandbox/ @grafana/user-essentials +/public/app/features/scenes/ @grafana/dashboards-squad +/public/app/features/search/ @grafana/user-essentials +/public/app/features/serviceaccounts/ @grafana/grafana-authnz-team +/public/app/features/storage/ @grafana/multitenancy-squad +/public/app/features/teams/ @grafana/grafana-authnz-team +/public/app/features/templating/ @grafana/dashboards-squad +/public/app/features/transformers/ @grafana/grafana-edge-squad +/public/app/features/users/ @grafana/grafana-authnz-team +/public/app/features/variables/ @grafana/dashboards-squad +/public/app/plugins/panel/alertGroups/ @grafana/alerting-squad-frontend +/public/app/plugins/panel/alertlist/ @grafana/alerting-squad-frontend +/public/app/plugins/panel/annolist/ @grafana/user-essentials +/public/app/plugins/panel/barchart/ @grafana/grafana-bi-squad +/public/app/plugins/panel/bargauge/ @grafana/user-essentials +/public/app/plugins/panel/dashlist/ @grafana/user-essentials +/public/app/plugins/panel/debug/ @ryantxu +/public/app/plugins/panel/gauge/ @grafana/user-essentials +/public/app/plugins/panel/gettingstarted/ @grafana/user-essentials +/public/app/plugins/panel/graph/ @grafana/user-essentials +/public/app/plugins/panel/heatmap/ @grafana/grafana-bi-squad +/public/app/plugins/panel/histogram/ @grafana/grafana-bi-squad +/public/app/plugins/panel/logs/ @grafana/observability-logs +/public/app/plugins/panel/nodeGraph/ @grafana/observability-traces-and-profiling +/public/app/plugins/panel/traces/ @grafana/observability-traces-and-profiling +/public/app/plugins/panel/flamegraph/ @grafana/observability-traces-and-profiling +/public/app/plugins/panel/piechart/ @grafana/grafana-bi-squad +/public/app/plugins/panel/state-timeline/ @grafana/grafana-bi-squad +/public/app/plugins/panel/status-history/ @grafana/grafana-bi-squad +/public/app/plugins/panel/table/ @grafana/grafana-bi-squad +/public/app/plugins/panel/table-old/ @grafana/grafana-bi-squad +/public/app/plugins/panel/timeseries/ @grafana/grafana-bi-squad +/public/app/plugins/panel/geomap/ @grafana/grafana-edge-squad +/public/app/plugins/panel/canvas/ @grafana/grafana-edge-squad +/public/app/plugins/panel/candlestick/ @grafana/grafana-edge-squad +/public/app/plugins/panel/icon/ @grafana/grafana-edge-squad +/public/app/plugins/panel/live/ @grafana/multitenancy-squad +/public/app/plugins/panel/news/ @grafana/user-essentials +/public/app/plugins/panel/stat/ @grafana/user-essentials +/public/app/plugins/panel/text/ @grafana/user-essentials +/public/app/plugins/panel/welcome/ @grafana/user-essentials +/public/app/plugins/panel/xychart/ @grafana/grafana-bi-squad +/public/app/plugins/sdk.ts @grafana/plugins-platform-frontend +/public/app/polyfills/old-mediaquerylist.ts @grafana/user-essentials +/public/app/routes/ @grafana/user-essentials +/public/app/store/ @grafana/user-essentials +/public/app/types/ @grafana/user-essentials +/public/dashboards/ @grafana/dashboards-squad +/public/fonts/ @grafana/alerting-squad-frontend +/public/emails/ @grafana/user-essentials +/public/gazetteer/ @ryantxu +/public/img/ @grafana/user-essentials +/public/lib/ @grafana/user-essentials +/public/maps/ @ryantxu +/public/robots.txt @grafana/frontend-ops +/public/sass/ @grafana/user-essentials +/public/test/ @grafana/user-essentials +/public/testdata/ @grafana/user-essentials +/public/views/ @grafana/user-essentials + +/public/app/features/explore/Logs.tsx @grafana/observability-logs +/public/app/features/explore/LogsContainer.tsx @grafana/observability-logs + +/public/app/features/explore/TraceView/ @grafana/observability-traces-and-profiling + +/public/api-merged.json @grafana/backend-platform +/public/api-spec.json @grafana/backend-platform +/public/openapi3.json @grafana/backend-platform +/public/app/angular/ @torkelo +/public/app/app.ts @grafana/frontend-ops +/public/app/dev.ts @grafana/frontend-ops +/public/app/index.ts @grafana/frontend-ops +/public/app/AppWrapper.tsx @grafana/frontend-ops +/public/app/partials/ @grafana/user-essentials + + + + +/scripts/benchmark-access-control.sh @grafana/grafana-authnz-team +/scripts/check-breaking-changes.sh @grafana/plugins-platform-frontend +/scripts/ci-* @grafana/grafana-release-eng +/scripts/circle-* @grafana/grafana-release-eng +/scripts/ci-frontend-metrics.sh @grafana/user-essentials @grafana/plugins-platform-frontend @grafana/grafana-bi-squad +/scripts/cli/ @grafana/user-essentials +/scripts/clean-git-or-error.sh @grafana/grafana-as-code +/scripts/grafana-server/ @grafana/user-essentials +/scripts/helpers/ @grafana/grafana-release-eng +/scripts/import_many_dashboards.sh @torkelo +/scripts/mixin-check.sh @bergquist +/scripts/openapi3/ @grafana/grafana-partnerships-team +/scripts/prepare-packagejson.js @grafana/frontend-ops +/scripts/protobuf-check.sh @grafana/plugins-platform-backend +/scripts/stripnulls.sh @grafana/grafana-as-code +/scripts/tag_release.sh @grafana/grafana-release-eng +/scripts/trigger_docker_build.sh @grafana/grafana-release-eng +/scripts/trigger_grafana_packer.sh @grafana/grafana-release-eng +/scripts/trigger_windows_build.sh @grafana/grafana-release-eng +/scripts/validate-devenv-dashboards.sh @grafana/grafana-release-eng +/scripts/verify-repo-update/ @grafana/grafana-release-eng + +/scripts/webpack/ @grafana/frontend-ops +/scripts/generate-a11y-report.sh @grafana/user-essentials .pa11yci.conf.js @grafana/user-essentials .pa11yci-pr.conf.js @grafana/user-essentials .betterer.results @joshhunt +.betterer.ts @joshhunt # @grafana/ui component documentation -*.mdx @jessover9000 @grafana/plugins-platform-frontend +*.mdx @grafana/plugins-platform-frontend + +# Design system +/public/img/icons/unicons/ @grafana/design-system # Core datasources -/public/app/plugins/datasource/cloudwatch @grafana/aws-plugins -/public/app/plugins/datasource/elasticsearch @grafana/observability-logs -/public/app/plugins/datasource/grafana-azure-monitor-datasource @grafana/cloud-provider-plugins -/public/app/plugins/datasource/graphite @grafana/observability-metrics -/public/app/plugins/datasource/influxdb @grafana/observability-metrics -/public/app/plugins/datasource/jaeger @grafana/observability-traces-and-profiling -/public/app/plugins/datasource/loki @grafana/observability-logs -/public/app/plugins/datasource/mssql @grafana/grafana-bi-squad -/public/app/plugins/datasource/mysql @grafana/grafana-bi-squad -/public/app/plugins/datasource/opentsdb @grafana/backend-platform -/public/app/plugins/datasource/postgres @grafana/grafana-bi-squad -/public/app/plugins/datasource/prometheus @grafana/observability-metrics -/public/app/plugins/datasource/cloud-monitoring @grafana/cloud-provider-plugins -/public/app/plugins/datasource/zipkin @grafana/observability-traces-and-profiling -/public/app/plugins/datasource/tempo @grafana/observability-traces-and-profiling -/public/app/plugins/datasource/alertmanager @grafana/alerting-squad +/public/app/plugins/datasource/dashboard/ @grafana/dashboards-squad +/public/app/plugins/datasource/cloudwatch/ @grafana/aws-plugins +/public/app/plugins/datasource/elasticsearch/ @grafana/observability-logs +/public/app/plugins/datasource/grafana/ @grafana/user-essentials +/public/app/plugins/datasource/testdata/ @grafana/plugins-platform-frontend +/public/app/plugins/datasource/grafana-azure-monitor-datasource/ @grafana/partner-plugins +/public/app/plugins/datasource/graphite/ @grafana/observability-metrics +/public/app/plugins/datasource/influxdb/ @grafana/observability-metrics +/public/app/plugins/datasource/jaeger/ @grafana/observability-traces-and-profiling +/public/app/plugins/datasource/loki/ @grafana/observability-logs +/public/app/plugins/datasource/mixed/ @grafana/dashboards-squad +/public/app/plugins/datasource/mssql/ @grafana/grafana-bi-squad +/public/app/plugins/datasource/mysql/ @grafana/grafana-bi-squad +/public/app/plugins/datasource/opentsdb/ @grafana/backend-platform +/public/app/plugins/datasource/postgres/ @grafana/grafana-bi-squad +/public/app/plugins/datasource/prometheus/ @grafana/observability-metrics +/public/app/plugins/datasource/cloud-monitoring/ @grafana/partner-plugins +/public/app/plugins/datasource/zipkin/ @grafana/observability-traces-and-profiling +/public/app/plugins/datasource/tempo/ @grafana/observability-traces-and-profiling +/public/app/plugins/datasource/phlare/ @grafana/observability-traces-and-profiling +/public/app/plugins/datasource/parca/ @grafana/observability-traces-and-profiling +/public/app/plugins/datasource/alertmanager/ @grafana/alerting-squad + +# SSE - Server Side Expressions +/pkg/expr/ @grafana/observability-metrics # Cloud middleware /grafana-mixin/ @grafana/hosted-grafana-team # Grafana authentication and authorization -/pkg/services/accesscontrol @grafana/grafana-authnz-team -/pkg/services/auth @grafana/grafana-authnz-team +/pkg/login/ @grafana/grafana-authnz-team +/pkg/services/accesscontrol/ @grafana/grafana-authnz-team +/pkg/services/auth/ @grafana/grafana-authnz-team /pkg/services/dashboards/accesscontrol.go @grafana/grafana-authnz-team -/pkg/services/datasources/permissions @grafana/grafana-authnz-team -/pkg/services/datasources/permissions/accesscontrol.go @grafana/grafana-authnz-team -/pkg/services/guardian @grafana/grafana-authnz-team -/pkg/services/ldap @grafana/grafana-authnz-team -/pkg/services/login @grafana/grafana-authnz-team -/pkg/services/multildap @grafana/grafana-authnz-team -/pkg/services/oauthtoken @grafana/grafana-authnz-team -/pkg/services/teamguardian @grafana/grafana-authnz-team -/pkg/services/serviceaccounts @grafana/grafana-authnz-team +/pkg/services/datasources/permissions/ @grafana/grafana-authnz-team +/pkg/services/guardian/ @grafana/grafana-authnz-team +/pkg/services/ldap/ @grafana/grafana-authnz-team +/pkg/services/login/ @grafana/grafana-authnz-team +/pkg/services/multildap/ @grafana/grafana-authnz-team +/pkg/services/oauthtoken/ @grafana/grafana-authnz-team +/pkg/services/teamguardian/ @grafana/grafana-authnz-team +/pkg/services/serviceaccounts/ @grafana/grafana-authnz-team +/pkg/services/loginattempt/ @grafana/grafana-authnz-team +/pkg/services/authn/ @grafana/grafana-authnz-team + +# Support bundles +/public/app/features/support-bundles/ @grafana/grafana-authnz-team +/pkg/services/supportbundles/ @grafana/grafana-authnz-team # Grafana Partnerships Team /pkg/infra/httpclient/httpclientprovider/sigv4_middleware.go @grafana/grafana-partnerships-team +/pkg/infra/httpclient/httpclientprovider/sigv4_middleware_test.go @grafana/grafana-partnerships-team -# Schema framework and code generation -/pkg/codegen @grafana/grafana-as-code -/pkg/framework/coremodel @grafana/grafana-as-code +# Kind definitions +/kinds/dashboard @grafana/dashboards-squad +/kinds/ @grafana/grafana-as-code + +# Kind system and code generation +embed.go @grafana/grafana-as-code +/pkg/kinds/ @grafana/grafana-as-code +/pkg/cuectx/ @grafana/grafana-as-code +/pkg/registry/ @grafana/grafana-as-code +/pkg/codegen/ @grafana/grafana-as-code +/pkg/kindsys/ @grafana/grafana-as-code +/pkg/kindsys/kindcat_custom.cue @grafana/cloud-app-platform-squad +/pkg/kinds/*/*_gen.go @grafana/grafana-as-code +/pkg/registry/corekind/ @grafana/grafana-as-code /public/app/plugins/*gen.go @grafana/grafana-as-code +/cue.mod/ @grafana/grafana-as-code + +# GitHub Workflows and Templates +/.github/CODEOWNERS @tolzhabayev +/.github/ISSUE_TEMPLATE/ @torkelo +/.github/PULL_REQUEST_TEMPLATE.md @torkelo +/.github/bot.md @torkelo +/.github/commands.json @torkelo +/.github/dependabot.yml @grafana/frontend-ops +/.github/metrics-collector.json @torkelo +/.github/pr-checks.json @marefr +/.github/pr-commands.json @marefr +/.github/renovate.json5 @grafana/frontend-ops +/.github/teams.yml @armandgrillet +/.github/workflows/backport.yml @grafana/grafana-release-eng +/.github/workflows/bump-version.yml @grafana/grafana-release-eng +/.github/workflows/close-milestone.yml @grafana/grafana-release-eng +/.github/workflows/cloud-data-sources-code-coverage.yml @grafana/partner-plugins @grafana/aws-plugins +/.github/workflows/codeowners-validator.yml @tolzhabayev +/.github/workflows/codeql-analysis.yml @DanCech +/.github/workflows/commands.yml @torkelo +/.github/workflows/detect-breaking-changes-* @grafana/plugins-platform-frontend +/.github/workflows/doc-validator.yml @grafana/docs-grafana +/.github/workflows/epic-add-to-platform-ux-parent-project.yml @meanmina +/.github/workflows/github-release.yml @torkelo +/.github/workflows/issue-labeled.yml @armandgrillet +/.github/workflows/metrics-collector.yml @torkelo +/.github/workflows/milestone.yml @marefr +/.github/workflows/ox-code-coverage.yml @grafana/explore-squad +/.github/workflows/pr-checks.yml @marefr +/.github/workflows/pr-codeql-analysis-go.yml @DanCech +/.github/workflows/pr-codeql-analysis-javascript.yml @DanCech +/.github/workflows/pr-codeql-analysis-python.yml @DanCech +/.github/workflows/pr-commands-closed.yml @tolzhabayev +/.github/workflows/pr-commands.yml @marefr +/.github/workflows/publish-technical-documentation-next.yml @grafana/docs-grafana +/.github/workflows/publish-technical-documentation-release.yml @grafana/docs-grafana +/.github/workflows/remove-milestone.yml @grafana/user-essentials +/.github/workflows/sbom-report.yml @grafana/security-team +/.github/workflows/scripts/json-file-to-job-output.js @grafana/plugins-platform-frontend +/.github/workflows/scripts/pr-get-job-link.js @grafana/plugins-platform-frontend +/.github/workflows/stale.yml @grafana/user-essentials +/.github/workflows/update-changelog.yml @grafana/grafana-release-eng + + +# Conf +/conf/defaults.ini @torkelo +/conf/sample.ini @torkelo +/conf/ldap.toml @grafana/grafana-authnz-team +/conf/ldap_multiple.toml @grafana/grafana-authnz-team +/conf/provisioning/access-control/ @grafana/grafana-authnz-team +/conf/provisioning/alerting/ @grafana/alerting-squad-backend +/conf/provisioning/dashboards/ @grafana/dashboards-squad +/conf/provisioning/datasources/ @grafana/plugins-platform-backend +/conf/provisioning/notifiers/ @bergquist +/conf/provisioning/plugins/ @grafana/plugins-platform-backend diff --git a/.github/workflows/enterprise-pr-check.yml b/.github/workflows/enterprise-pr-check.yml deleted file mode 100644 index f5552fe9437..00000000000 --- a/.github/workflows/enterprise-pr-check.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Enterprise PR check -on: - pull_request: - branches: - - main - - 'v[0-9]+.[0-9]+.x' -jobs: - dispatch: - runs-on: ubuntu-latest - steps: - - name: Checkout Actions - uses: actions/checkout@v2 - with: - repository: "grafana/grafana-github-actions" - path: ./actions - ref: main - - name: Install Actions - run: npm install --production --prefix ./actions - - name: Repository Dispatch - uses: ./actions/repository-dispatch - with: - token: ${{ secrets.GH_BOT_ACCESS_TOKEN }} - repository: grafana/grafana-enterprise - event_type: oss-pull-request - client_payload: - '{"source_branch": "${{ github.head_ref }}", "target_branch": "${{ github.base_ref }}", "pr_number": "${{ github.event.number }}"}' diff --git a/Dockerfile b/Dockerfile index 972ca3ad8ac..5a1f9a9feaa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ COPY emails emails ENV NODE_ENV production RUN yarn build -FROM golang:1.19.4-alpine3.15 as go-builder +FROM golang:1.19.4-alpine3.17 as go-builder RUN apk add --no-cache gcc g++ make diff --git a/go.mod b/go.mod index 42b6b1bbda4..b625ca0bc61 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/go-sql-driver/mysql v1.6.0 github.com/go-stack/stack v1.8.1 github.com/gobwas/glob v0.2.3 - github.com/gofrs/uuid v4.3.0+incompatible // indirect + github.com/gofrs/uuid v4.3.0+incompatible github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 github.com/golang/snappy v0.0.4 @@ -242,13 +242,16 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.13.2 github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.4.0 github.com/Azure/go-autorest/autorest/adal v0.9.17 + github.com/Masterminds/semver/v3 v3.1.1 github.com/armon/go-radix v1.0.0 github.com/blugelabs/bluge v0.1.9 github.com/blugelabs/bluge_segment_api v0.2.0 github.com/dlmiddlecote/sqlstats v1.0.2 + github.com/docker/docker v20.10.8+incompatible github.com/drone/drone-cli v1.5.0 github.com/getkin/kin-openapi v0.94.0 github.com/golang-migrate/migrate/v4 v4.7.0 + github.com/google/go-github v17.0.0+incompatible github.com/google/go-github/v45 v45.2.0 github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f github.com/jmoiron/sqlx v1.3.5 @@ -267,14 +270,16 @@ require ( github.com/bmatcuk/doublestar v1.1.1 // indirect github.com/buildkite/yaml v2.1.0+incompatible // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect github.com/drone/drone-yaml v0.0.0-20190729072335-70fa398b3560 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/memberlist v0.4.0 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/segmentio/asm v1.1.4 // indirect go.starlark.net v0.0.0-20201118183435-e55f603d8c79 // indirect @@ -334,10 +339,6 @@ require ( go.opentelemetry.io/proto/otlp v0.15.0 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - k8s.io/api v0.22.5 // indirect - k8s.io/apimachinery v0.22.5 // indirect - k8s.io/klog/v2 v2.30.0 // indirect - k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect ) // Use fork of crewjam/saml with fixes for some issues until changes get merged into upstream diff --git a/go.sum b/go.sum index d3fc570e3ac..447b30e9115 100644 --- a/go.sum +++ b/go.sum @@ -135,6 +135,7 @@ github.com/Azure/azure-storage-blob-go v0.13.0/go.mod h1:pA9kNqtjUeQF2zOSu4s//nU github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= github.com/Azure/go-amqp v0.16.0/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg= github.com/Azure/go-amqp v0.16.4/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v11.2.8+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -225,6 +226,7 @@ github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Masterminds/squirrel v0.0.0-20161115235646-20f192218cf5/go.mod h1:xnKTFzjGUiZtiOagBsfnvomW+nJg2usB1ZpordQWqNM= @@ -923,7 +925,6 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -1245,6 +1246,7 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI= github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28= @@ -1257,10 +1259,8 @@ github.com/google/go-replayers/httpreplay v1.1.1/go.mod h1:gN9GeLIs7l6NUoVaSSnv2 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE= github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -1830,8 +1830,6 @@ github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vq github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA= -github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= @@ -1909,6 +1907,7 @@ github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2J github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -1922,6 +1921,7 @@ github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3P github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mozillazg/go-cos v0.13.0/go.mod h1:Zp6DvvXn0RUOXGJ2chmWt2bLEqRAnJnS3DnAZsJsoaE= @@ -2848,7 +2848,6 @@ golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -3533,6 +3532,7 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -3563,7 +3563,6 @@ k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU= k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s= k8s.io/api v0.22.1/go.mod h1:bh13rkTp3F1XEaLGykbyRD2QaTTzPm0e/BMd8ptFONY= k8s.io/api v0.22.5 h1:xk7C+rMjF/EGELiD560jdmwzrB788mfcHiNbMQLIVI8= -k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs= k8s.io/apimachinery v0.0.0-20181201231028-18a5ff3097b4/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/apimachinery v0.0.0-20190809020650-423f5d784010/go.mod h1:Waf/xTS2FGRrgXCkO5FP3XxTOWh0qLf2QhL1qFZZ/R8= k8s.io/apimachinery v0.0.0-20191115015347-3c7067801da2/go.mod h1:dXFS2zaQR8fyzuvRdJDHw2Aerij/yVGJSre0bZQSVJA= @@ -3583,7 +3582,6 @@ k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswP k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= k8s.io/apimachinery v0.22.5 h1:cIPwldOYm1Slq9VLBRPtEYpyhjIm1C6aAMAoENuvN9s= -k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= @@ -3603,6 +3601,7 @@ k8s.io/klog v0.1.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= @@ -3613,7 +3612,6 @@ k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw= -k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20190709113604-33be087ad058/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4= k8s.io/kube-openapi v0.0.0-20190722073852-5e22f3d471e6/go.mod h1:RZvgC8MSN6DjiMV6oIfEE9pDL9CYXokkfaCKZeHm3nc= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= @@ -3623,7 +3621,6 @@ k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20190809000727-6c36bc71fc4a/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114200735-6ca3b61696b6/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= @@ -3631,7 +3628,6 @@ k8s.io/utils v0.0.0-20200414100711-2df71ebbae66/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs= -k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/pkg/build/cmd/argcount_wrapper.go b/pkg/build/cmd/argcount_wrapper.go index d3a3cfd67f6..690695cd350 100644 --- a/pkg/build/cmd/argcount_wrapper.go +++ b/pkg/build/cmd/argcount_wrapper.go @@ -2,7 +2,22 @@ package main import "github.com/urfave/cli/v2" -func ArgCountWrapper(max int, action cli.ActionFunc) cli.ActionFunc { +// ArgCountWrapper will cause the action to fail if there were not exactly `num` args provided. +func ArgCountWrapper(num int, action cli.ActionFunc) cli.ActionFunc { + return func(ctx *cli.Context) error { + if ctx.NArg() != num { + if err := cli.ShowSubcommandHelp(ctx); err != nil { + return cli.Exit(err.Error(), 1) + } + return cli.Exit("", 1) + } + + return action(ctx) + } +} + +// ArgCountWrapper will cause the action to fail if there were more than `num` args provided. +func MaxArgCountWrapper(max int, action cli.ActionFunc) cli.ActionFunc { return func(ctx *cli.Context) error { if ctx.NArg() > max { if err := cli.ShowSubcommandHelp(ctx); err != nil { diff --git a/pkg/build/cmd/buildbackend.go b/pkg/build/cmd/buildbackend.go index 29ffb307d63..39f0d707fe4 100644 --- a/pkg/build/cmd/buildbackend.go +++ b/pkg/build/cmd/buildbackend.go @@ -13,7 +13,7 @@ import ( ) func BuildBackend(ctx *cli.Context) error { - metadata, err := GenerateMetadata(ctx) + metadata, err := config.GenerateMetadata(ctx) if err != nil { return err } diff --git a/pkg/build/cmd/builddocker.go b/pkg/build/cmd/builddocker.go index f17037664a7..ac1858a65ba 100644 --- a/pkg/build/cmd/builddocker.go +++ b/pkg/build/cmd/builddocker.go @@ -14,7 +14,7 @@ func BuildDocker(c *cli.Context) error { return err } - metadata, err := GenerateMetadata(c) + metadata, err := config.GenerateMetadata(c) if err != nil { return err } diff --git a/pkg/build/cmd/buildfrontend.go b/pkg/build/cmd/buildfrontend.go index 2874a119403..d7825f419cd 100644 --- a/pkg/build/cmd/buildfrontend.go +++ b/pkg/build/cmd/buildfrontend.go @@ -3,6 +3,7 @@ package main import ( "log" + "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/errutil" "github.com/grafana/grafana/pkg/build/frontend" "github.com/grafana/grafana/pkg/build/syncutil" @@ -10,7 +11,7 @@ import ( ) func BuildFrontend(c *cli.Context) error { - metadata, err := GenerateMetadata(c) + metadata, err := config.GenerateMetadata(c) if err != nil { return err } diff --git a/pkg/build/cmd/buildinternalplugins.go b/pkg/build/cmd/buildinternalplugins.go index 2871106c933..fad80b818d5 100644 --- a/pkg/build/cmd/buildinternalplugins.go +++ b/pkg/build/cmd/buildinternalplugins.go @@ -17,7 +17,7 @@ func BuildInternalPlugins(c *cli.Context) error { } const grafanaDir = "." - metadata, err := GenerateMetadata(c) + metadata, err := config.GenerateMetadata(c) if err != nil { return err } diff --git a/pkg/build/cmd/enterprisecheck.go b/pkg/build/cmd/enterprisecheck.go new file mode 100644 index 00000000000..f2c36b0fc2e --- /dev/null +++ b/pkg/build/cmd/enterprisecheck.go @@ -0,0 +1,132 @@ +package main + +import ( + "fmt" + "log" + "os" + "strconv" + + "github.com/grafana/grafana/pkg/build/env" + "github.com/grafana/grafana/pkg/build/git" + "github.com/urfave/cli/v2" +) + +// checkOpts are options used to create a new GitHub check for the enterprise downstream test. +type checkOpts struct { + SHA string + URL string + Branch string + PR int +} + +func getCheckOpts(args []string) (*checkOpts, error) { + branch, ok := env.Lookup("DRONE_SOURCE_BRANCH", args) + if !ok { + return nil, cli.Exit("Unable to retrieve build source branch", 1) + } + + var ( + rgx = git.PRCheckRegexp() + matches = rgx.FindStringSubmatch(branch) + ) + + sha, ok := env.Lookup("SOURCE_COMMIT", args) + if !ok { + if matches == nil || len(matches) <= 1 { + return nil, cli.Exit("Unable to retrieve source commit", 1) + } + sha = matches[2] + } + + url, ok := env.Lookup("DRONE_BUILD_LINK", args) + if !ok { + return nil, cli.Exit(`missing environment variable "DRONE_BUILD_LINK"`, 1) + } + + prStr, ok := env.Lookup("OSS_PULL_REQUEST", args) + if !ok { + if matches == nil || len(matches) <= 1 { + return nil, cli.Exit("Unable to retrieve PR number", 1) + } + + prStr = matches[1] + } + + pr, err := strconv.Atoi(prStr) + if err != nil { + return nil, err + } + + return &checkOpts{ + Branch: branch, + PR: pr, + SHA: sha, + URL: url, + }, nil +} + +// EnterpriseCheckBegin creates the GitHub check and signals the beginning of the downstream build / test process +func EnterpriseCheckBegin(c *cli.Context) error { + var ( + ctx = c.Context + client = git.NewGitHubClient(ctx, c.String("github-token")) + ) + + opts, err := getCheckOpts(os.Environ()) + if err != nil { + return err + } + + if _, err = git.CreateEnterpriseStatus(ctx, client.Repositories, opts.SHA, opts.URL, "pending"); err != nil { + return err + } + + return nil +} + +func EnterpriseCheckSuccess(c *cli.Context) error { + return completeEnterpriseCheck(c, true) +} + +func EnterpriseCheckFail(c *cli.Context) error { + return completeEnterpriseCheck(c, false) +} + +func completeEnterpriseCheck(c *cli.Context, success bool) error { + var ( + ctx = c.Context + client = git.NewGitHubClient(ctx, c.String("github-token")) + ) + + // Update the pull request labels + opts, err := getCheckOpts(os.Environ()) + if err != nil { + return err + } + + status := "failure" + if success { + status = "success" + } + + // Update the GitHub check... + if _, err := git.CreateEnterpriseStatus(ctx, client.Repositories, opts.SHA, opts.URL, status); err != nil { + return err + } + + // Delete branch if needed + log.Printf("Checking branch '%s' against '%s'", git.PRCheckRegexp().String(), opts.Branch) + if git.PRCheckRegexp().MatchString(opts.Branch) { + log.Println("Deleting branch", opts.Branch) + if err := git.DeleteEnterpriseBranch(ctx, client.Git, opts.Branch); err != nil { + return fmt.Errorf("error deleting enterprise branch: %w", err) + } + } + + label := "enterprise-failed" + if success { + label = "enterprise-ok" + } + + return git.AddLabelToPR(ctx, client.Issues, opts.PR, label) +} diff --git a/pkg/build/cmd/enterprisecheck_test.go b/pkg/build/cmd/enterprisecheck_test.go new file mode 100644 index 00000000000..0eeb5bd5741 --- /dev/null +++ b/pkg/build/cmd/enterprisecheck_test.go @@ -0,0 +1,69 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetCheckOpts(t *testing.T) { + t.Run("it should return the checkOpts if the correct environment variables are set", func(t *testing.T) { + args := []string{ + "SOURCE_COMMIT=1234", + "DRONE_SOURCE_BRANCH=test", + "DRONE_BUILD_LINK=http://example.com", + "OSS_PULL_REQUEST=1", + } + + opts, err := getCheckOpts(args) + require.NoError(t, err) + require.Equal(t, opts.SHA, "1234") + require.Equal(t, opts.URL, "http://example.com") + }) + t.Run("it should return an error if SOURCE_COMMIT is not set", func(t *testing.T) { + args := []string{ + "DRONE_BUILD_LINK=http://example.com", + "DRONE_SOURCE_BRANCH=test", + "DRONE_BUILD_LINK=http://example.com", + "OSS_PULL_REQUEST=1", + } + + opts, err := getCheckOpts(args) + require.Nil(t, opts) + require.Error(t, err) + }) + t.Run("it should return an error if DRONE_BUILD_LINK is not set", func(t *testing.T) { + args := []string{ + "SOURCE_COMMIT=1234", + "DRONE_SOURCE_BRANCH=test", + "OSS_PULL_REQUEST=1", + } + + opts, err := getCheckOpts(args) + require.Nil(t, opts) + require.Error(t, err) + }) + t.Run("it should return an error if OSS_PULL_REQUEST is not set", func(t *testing.T) { + args := []string{ + "SOURCE_COMMIT=1234", + "DRONE_SOURCE_BRANCH=test", + "DRONE_BUILD_LINK=http://example.com", + } + + opts, err := getCheckOpts(args) + require.Nil(t, opts) + require.Error(t, err) + }) + t.Run("it should return an error if OSS_PULL_REQUEST is not an integer", func(t *testing.T) { + args := []string{ + "SOURCE_COMMIT=1234", + "DRONE_SOURCE_BRANCH=test", + "DRONE_BUILD_LINK=http://example.com", + "OSS_PULL_REQUEST=http://example.com", + } + + opts, err := getCheckOpts(args) + require.Nil(t, opts) + require.Error(t, err) + }) +} diff --git a/pkg/build/cmd/exportversion.go b/pkg/build/cmd/exportversion.go index afbe05114af..a3a54309c73 100644 --- a/pkg/build/cmd/exportversion.go +++ b/pkg/build/cmd/exportversion.go @@ -4,11 +4,12 @@ import ( "os" "path/filepath" + "github.com/grafana/grafana/pkg/build/config" "github.com/urfave/cli/v2" ) func ExportVersion(c *cli.Context) error { - metadata, err := GenerateMetadata(c) + metadata, err := config.GenerateMetadata(c) if err != nil { return err } diff --git a/pkg/build/cmd/fetchimages.go b/pkg/build/cmd/fetchimages.go index 3845daed5a7..bd1ec6a5102 100644 --- a/pkg/build/cmd/fetchimages.go +++ b/pkg/build/cmd/fetchimages.go @@ -20,12 +20,12 @@ const ( func FetchImages(c *cli.Context) error { if c.NArg() > 0 { if err := cli.ShowSubcommandHelp(c); err != nil { - return cli.NewExitError(err.Error(), 1) + return cli.Exit(err.Error(), 1) } - return cli.NewExitError("", 1) + return cli.Exit("", 1) } - metadata, err := GenerateMetadata(c) + metadata, err := config.GenerateMetadata(c) if err != nil { return err } diff --git a/pkg/build/cmd/flags.go b/pkg/build/cmd/flags.go index 7aa6d98d026..413ad66efa1 100644 --- a/pkg/build/cmd/flags.go +++ b/pkg/build/cmd/flags.go @@ -46,4 +46,14 @@ var ( Usage: "Google Cloud Platform key file", Required: true, } + gitHubTokenFlag = cli.StringFlag{ + Name: "github-token", + Value: "", + EnvVars: []string{"GITHUB_TOKEN"}, + Usage: "GitHub token", + } + tagFlag = cli.StringFlag{ + Name: "tag", + Usage: "Grafana version tag", + } ) diff --git a/pkg/build/cmd/genversions.go b/pkg/build/cmd/genversions.go deleted file mode 100644 index 81879f42b55..00000000000 --- a/pkg/build/cmd/genversions.go +++ /dev/null @@ -1,92 +0,0 @@ -package main - -import ( - "fmt" - "os" - "strings" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/droneutil" - "github.com/urfave/cli/v2" -) - -func GenerateMetadata(c *cli.Context) (config.Metadata, error) { - var metadata config.Metadata - version := "" - - event, err := droneutil.GetDroneEventFromEnv() - if err != nil { - return config.Metadata{}, err - } - - var releaseMode config.ReleaseMode - switch event { - case string(config.PullRequestMode): - releaseMode = config.ReleaseMode{Mode: config.PullRequestMode} - case config.Push: - mode, err := config.CheckDroneTargetBranch() - if err != nil { - return config.Metadata{}, err - } - releaseMode = config.ReleaseMode{Mode: mode} - case config.Custom: - mode, err := config.CheckDroneTargetBranch() - if err != nil { - return config.Metadata{}, err - } - // if there is a custom event targeting the main branch, that's an enterprise downstream build - if mode == config.MainBranch { - releaseMode = config.ReleaseMode{Mode: config.CustomMode} - } else { - releaseMode = config.ReleaseMode{Mode: mode} - } - case config.Tag, config.Promote: - tag, ok := os.LookupEnv("DRONE_TAG") - if !ok || tag == "" { - return config.Metadata{}, err - } - version = strings.TrimPrefix(tag, "v") - mode, err := config.CheckSemverSuffix() - if err != nil { - return config.Metadata{}, err - } - releaseMode = mode - case config.Cronjob: - releaseMode = config.ReleaseMode{Mode: config.CronjobMode} - } - - if version == "" { - version, err = generateVersionFromBuildID() - if err != nil { - return config.Metadata{}, err - } - } - - currentCommit, err := config.GetDroneCommit() - if err != nil { - return config.Metadata{}, err - } - metadata = config.Metadata{ - GrafanaVersion: version, - ReleaseMode: releaseMode, - GrabplVersion: c.App.Version, - CurrentCommit: currentCommit, - } - - fmt.Printf("building Grafana version: %s, release mode: %+v", metadata.GrafanaVersion, metadata.ReleaseMode) - - return metadata, nil -} - -func generateVersionFromBuildID() (string, error) { - buildID, ok := os.LookupEnv("DRONE_BUILD_NUMBER") - if !ok { - return "", fmt.Errorf("unable to get DRONE_BUILD_NUMBER environmental variable") - } - var err error - version, err := config.GetGrafanaVersion(buildID, ".") - if err != nil { - return "", err - } - return version, nil -} diff --git a/pkg/build/cmd/genversions_test.go b/pkg/build/cmd/genversions_test.go deleted file mode 100644 index c34b8915da2..00000000000 --- a/pkg/build/cmd/genversions_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package main - -import ( - "flag" - "os" - "testing" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/stretchr/testify/require" - "github.com/urfave/cli/v2" -) - -const ( - DroneBuildEvent = "DRONE_BUILD_EVENT" - DroneTargetBranch = "DRONE_TARGET_BRANCH" - DroneTag = "DRONE_TAG" - DroneSemverPrerelease = "DRONE_SEMVER_PRERELEASE" - DroneBuildNumber = "DRONE_BUILD_NUMBER" -) - -const ( - hashedGrafanaVersion = "9.2.0-12345pre" - versionedBranch = "v9.2.x" -) - -func TestGetMetadata(t *testing.T) { - tcs := []struct { - envMap map[string]string - expVersion string - mode config.ReleaseMode - }{ - {map[string]string{DroneBuildEvent: config.PullRequest, DroneTargetBranch: "", DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, config.ReleaseMode{Mode: config.PullRequestMode}}, - {map[string]string{DroneBuildEvent: config.Push, DroneTargetBranch: versionedBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, config.ReleaseMode{Mode: config.ReleaseBranchMode}}, - {map[string]string{DroneBuildEvent: config.Push, DroneTargetBranch: config.MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, config.ReleaseMode{Mode: config.MainMode}}, - {map[string]string{DroneBuildEvent: config.Custom, DroneTargetBranch: versionedBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, config.ReleaseMode{Mode: config.ReleaseBranchMode}}, - {map[string]string{DroneBuildEvent: config.Custom, DroneTargetBranch: config.MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, config.ReleaseMode{Mode: config.Custom}}, - {map[string]string{DroneBuildEvent: config.Tag, DroneTargetBranch: "", DroneTag: "v9.2.0", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, "9.2.0", config.ReleaseMode{Mode: config.TagMode, IsBeta: false, IsTest: false}}, - {map[string]string{DroneBuildEvent: config.Tag, DroneTargetBranch: "", DroneTag: "v9.2.0-beta", DroneSemverPrerelease: "beta", DroneBuildNumber: "12345"}, "9.2.0-beta", config.ReleaseMode{Mode: config.TagMode, IsBeta: true, IsTest: false}}, - {map[string]string{DroneBuildEvent: config.Tag, DroneTargetBranch: "", DroneTag: "v9.2.0-test", DroneSemverPrerelease: "test", DroneBuildNumber: "12345"}, "9.2.0-test", config.ReleaseMode{Mode: config.TagMode, IsBeta: false, IsTest: true}}, - {map[string]string{DroneBuildEvent: config.Promote, DroneTargetBranch: "", DroneTag: "v9.2.0", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, "9.2.0", config.ReleaseMode{Mode: config.TagMode, IsBeta: false, IsTest: false}}, - {map[string]string{DroneBuildEvent: config.Promote, DroneTargetBranch: "", DroneTag: "v9.2.0-beta", DroneSemverPrerelease: "beta", DroneBuildNumber: "12345"}, "9.2.0-beta", config.ReleaseMode{Mode: config.TagMode, IsBeta: true, IsTest: false}}, - {map[string]string{DroneBuildEvent: config.Promote, DroneTargetBranch: "", DroneTag: "v9.2.0-test", DroneSemverPrerelease: "test", DroneBuildNumber: "12345"}, "9.2.0-test", config.ReleaseMode{Mode: config.TagMode, IsBeta: false, IsTest: true}}, - } - - ctx := cli.NewContext(cli.NewApp(), &flag.FlagSet{}, nil) - for _, tc := range tcs { - t.Run("Should return valid metadata, ", func(t *testing.T) { - setUpEnv(t, tc.envMap) - testMetadata(t, ctx, tc.expVersion, tc.mode) - }) - } -} - -func testMetadata(t *testing.T, ctx *cli.Context, version string, releaseMode config.ReleaseMode) { - t.Helper() - - metadata, err := GenerateMetadata(ctx) - require.NoError(t, err) - t.Run("with a valid version", func(t *testing.T) { - expVersion := metadata.GrafanaVersion - require.Equal(t, expVersion, version) - }) - - t.Run("with a valid release mode from the built-in list", func(t *testing.T) { - expMode := metadata.ReleaseMode - require.NoError(t, err) - require.Equal(t, expMode, releaseMode) - }) -} - -func setUpEnv(t *testing.T, envMap map[string]string) { - t.Helper() - - os.Clearenv() - err := os.Setenv("DRONE_COMMIT", "abcd12345") - require.NoError(t, err) - for k, v := range envMap { - err := os.Setenv(k, v) - require.NoError(t, err) - } -} diff --git a/pkg/build/cmd/grafanacom.go b/pkg/build/cmd/grafanacom.go index 9d81d8b62f9..43f5bc9eb0f 100644 --- a/pkg/build/cmd/grafanacom.go +++ b/pkg/build/cmd/grafanacom.go @@ -15,11 +15,12 @@ import ( "strings" "time" + "github.com/urfave/cli/v2" + "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/gcloud" "github.com/grafana/grafana/pkg/build/gcloud/storage" "github.com/grafana/grafana/pkg/build/packaging" - "github.com/urfave/cli/v2" ) const grafanaAPI = "https://grafana.com/api" @@ -33,7 +34,7 @@ func GrafanaCom(c *cli.Context) error { return fmt.Errorf("couldn't activate service account, err: %w", err) } - metadata, err := GenerateMetadata(c) + metadata, err := config.GenerateMetadata(c) if err != nil { return err } @@ -66,11 +67,11 @@ func GrafanaCom(c *cli.Context) error { grafanaAPIKey := strings.TrimSpace(os.Getenv("GRAFANA_COM_API_KEY")) if grafanaAPIKey == "" { - return cli.NewExitError("the environment variable GRAFANA_COM_API_KEY must be set", 1) + return cli.Exit("the environment variable GRAFANA_COM_API_KEY must be set", 1) } whatsNewURL, releaseNotesURL, err := getReleaseURLs() if err != nil { - return cli.NewExitError(err.Error(), 1) + return cli.Exit(err.Error(), 1) } // TODO: Verify config values @@ -89,7 +90,7 @@ func GrafanaCom(c *cli.Context) error { } if err := publishPackages(cfg); err != nil { - return cli.NewExitError(err.Error(), 1) + return cli.Exit(err.Error(), 1) } log.Println("Successfully published packages to grafana.com!") @@ -146,7 +147,7 @@ func publishPackages(cfg packaging.PublishConfig) error { } switch cfg.ReleaseMode.Mode { - case config.MainMode, config.CustomMode, config.CronjobMode: + case config.MainMode, config.DownstreamMode, config.CronjobMode: pth = path.Join(pth, packaging.MainFolder) default: pth = path.Join(pth, packaging.ReleaseFolder) @@ -177,7 +178,7 @@ func publishPackages(cfg packaging.PublishConfig) error { Version: cfg.Version, ReleaseDate: time.Now().UTC(), Builds: builds, - Stable: cfg.ReleaseMode.Mode == config.TagMode, + Stable: cfg.ReleaseMode.Mode == config.TagMode && !cfg.ReleaseMode.IsBeta && !cfg.ReleaseMode.IsTest, Beta: cfg.ReleaseMode.IsBeta, Nightly: cfg.ReleaseMode.Mode == config.CronjobMode, } diff --git a/pkg/build/cmd/main.go b/pkg/build/cmd/main.go index a045a933076..2d2f9cbe4c6 100644 --- a/pkg/build/cmd/main.go +++ b/pkg/build/cmd/main.go @@ -5,12 +5,18 @@ import ( "os" "strings" - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/docker" - "github.com/grafana/grafana/pkg/build/packaging" "github.com/urfave/cli/v2" + + "github.com/grafana/grafana/pkg/build/docker" ) +var additionalCommands []*cli.Command = make([]*cli.Command, 0, 5) + +//nolint:unused +func registerAppCommand(c *cli.Command) { + additionalCommands = append(additionalCommands, c) +} + func main() { app := cli.NewApp() app.Commands = cli.Commands{ @@ -18,7 +24,7 @@ func main() { Name: "build-backend", Usage: "Build one or more variants of back-end binaries", ArgsUsage: "[version]", - Action: ArgCountWrapper(1, BuildBackend), + Action: MaxArgCountWrapper(1, BuildBackend), Flags: []cli.Flag{ &jobsFlag, &variantsFlag, @@ -58,13 +64,18 @@ func main() { Value: "grafana-server", Usage: "Specify the server host", }, + &cli.StringFlag{ + Name: "video", + Value: "true", + Usage: "Specify if videos should be recorded", + }, }, }, { Name: "build-frontend", Usage: "Build front-end artifacts", ArgsUsage: "[version]", - Action: ArgCountWrapper(1, BuildFrontend), + Action: MaxArgCountWrapper(1, BuildFrontend), Flags: []cli.Flag{ &jobsFlag, &editionFlag, @@ -74,7 +85,7 @@ func main() { { Name: "build-docker", Usage: "Build Grafana Docker images", - Action: ArgCountWrapper(1, BuildDocker), + Action: MaxArgCountWrapper(1, BuildDocker), Flags: []cli.Flag{ &jobsFlag, &editionFlag, @@ -93,6 +104,14 @@ func main() { }, }, }, + { + Name: "upload-cdn", + Usage: "Upload public/* to a cdn bucket", + Action: UploadCDN, + Flags: []cli.Flag{ + &editionFlag, + }, + }, { Name: "shellcheck", Usage: "Run shellcheck on shell scripts", @@ -101,7 +120,7 @@ func main() { { Name: "build-plugins", Usage: "Build internal plug-ins", - Action: ArgCountWrapper(1, BuildInternalPlugins), + Action: MaxArgCountWrapper(1, BuildInternalPlugins), Flags: []cli.Flag{ &jobsFlag, &editionFlag, @@ -114,13 +133,19 @@ func main() { Name: "publish-metrics", Usage: "Publish a set of metrics from stdin", ArgsUsage: "", - Action: ArgCountWrapper(1, PublishMetrics), + Action: MaxArgCountWrapper(1, PublishMetrics), }, { Name: "verify-drone", Usage: "Verify Drone configuration", Action: VerifyDrone, }, + { + Name: "verify-starlark", + Usage: "Verify Starlark configuration", + ArgsUsage: "", + Action: VerifyStarlark, + }, { Name: "export-version", Usage: "Exports version in dist/grafana.version", @@ -130,7 +155,7 @@ func main() { Name: "package", Usage: "Package one or more Grafana variants", ArgsUsage: "[version]", - Action: ArgCountWrapper(1, Package), + Action: MaxArgCountWrapper(1, Package), Flags: []cli.Flag{ &jobsFlag, &variantsFlag, @@ -141,7 +166,7 @@ func main() { }, { Name: "store-storybook", - Usage: "Integrity check for storybook build", + Usage: "Stores storybook to GCS buckets", Action: StoreStorybook, Flags: []cli.Flag{ &cli.StringFlag{ @@ -150,10 +175,81 @@ func main() { }, }, }, + { + Name: "verify-storybook", + Usage: "Integrity check for storybook build", + Action: VerifyStorybook, + }, + { + Name: "upload-packages", + Usage: "Upload Grafana packages", + Action: UploadPackages, + Flags: []cli.Flag{ + &jobsFlag, + &editionFlag, + &cli.BoolFlag{ + Name: "enterprise2", + Usage: "Declare if the edition is enterprise2", + }, + }, + }, { Name: "artifacts", Usage: "Handle Grafana artifacts", Subcommands: cli.Commands{ + { + Name: "publish", + Usage: "Publish Grafana artifacts", + Action: PublishArtifactsAction, + Flags: []cli.Flag{ + &editionFlag, + &cli.BoolFlag{ + Name: "security", + Usage: "Security release", + }, + &cli.StringFlag{ + Name: "security-dest-bucket", + Usage: "Google Cloud Storage bucket for security packages (or $SECURITY_DEST_BUCKET)", + }, + &cli.StringFlag{ + Name: "tag", + Usage: "Grafana version tag", + }, + &cli.StringFlag{ + Name: "src-bucket", + Value: "grafana-prerelease", + Usage: "Google Cloud Storage bucket", + }, + &cli.StringFlag{ + Name: "dest-bucket", + Value: "grafana-downloads", + Usage: "Google Cloud Storage bucket for published packages", + }, + &cli.StringFlag{ + Name: "enterprise2-dest-bucket", + Value: "grafana-downloads-enterprise2", + Usage: "Google Cloud Storage bucket for published packages", + }, + &cli.StringFlag{ + Name: "enterprise2-security-prefix", + Usage: "Bucket path prefix for enterprise2 security releases (or $ENTERPRISE2_SECURITY_PREFIX)", + }, + &cli.StringFlag{ + Name: "static-assets-bucket", + Value: "grafana-static-assets", + Usage: "Google Cloud Storage bucket for static assets", + }, + &cli.StringSliceFlag{ + Name: "static-asset-editions", + Usage: "All the editions of the static assets (or $STATIC_ASSET_EDITIONS)", + }, + &cli.StringFlag{ + Name: "storybook-bucket", + Value: "grafana-storybook", + Usage: "Google Cloud Storage bucket for storybooks", + }, + }, + }, { Name: "docker", Usage: "Handle Grafana Docker images", @@ -162,11 +258,54 @@ func main() { Name: "fetch", Usage: "Fetch Grafana Docker images", ArgsUsage: "[version]", - Action: ArgCountWrapper(1, FetchImages), + Action: MaxArgCountWrapper(1, FetchImages), Flags: []cli.Flag{ &editionFlag, }, }, + { + Name: "publish-enterprise2", + Usage: "Handle Grafana Enterprise2 Docker images", + ArgsUsage: "[version]", + Action: Enterprise2, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "dockerhub-repo", + Usage: "DockerHub repo to push images", + }, + }, + }, + }, + }, + { + Name: "npm", + Usage: "Handle Grafana npm packages", + Subcommands: cli.Commands{ + { + Name: "release", + Usage: "Release npm packages", + ArgsUsage: "[version]", + Action: NpmReleaseAction, + Flags: []cli.Flag{ + &tagFlag, + }, + }, + { + Name: "store", + Usage: "Store npm packages tarball", + Action: NpmStoreAction, + Flags: []cli.Flag{ + &tagFlag, + }, + }, + { + Name: "retrieve", + Usage: "Retrieve npm packages tarball", + Action: NpmRetrieveAction, + Flags: []cli.Flag{ + &tagFlag, + }, + }, }, }, }, @@ -175,48 +314,6 @@ func main() { Name: "publish", Usage: "Publish packages to Grafana com and repositories", Subcommands: cli.Commands{ - { - Name: "packages", - Usage: "publish Grafana packages", - ArgsUsage: "[version]", - Action: PublishPackages, - Flags: []cli.Flag{ - &jobsFlag, - &editionFlag, - &buildIDFlag, - &dryRunFlag, - &gcpKeyFlag, - &cli.StringFlag{ - Name: "packages-bucket", - Value: config.PublicBucket, - Usage: "Google Cloud Storage Debian database bucket", - }, - &cli.StringFlag{ - Name: "deb-db-bucket", - Value: packaging.DefaultDebDBBucket, - Usage: "Google Cloud Storage Debian database bucket", - }, - &cli.StringFlag{ - Name: "deb-repo-bucket", - Value: packaging.DefaultDebRepoBucket, - Usage: "Google Cloud Storage Debian repo bucket", - }, - &cli.StringFlag{ - Name: "rpm-repo-bucket", - Value: packaging.DefaultRPMRepoBucket, - Usage: "Google Cloud Storage RPM repo bucket", - }, - &cli.StringFlag{ - Name: "ttl", - Value: packaging.DefaultTTLSeconds, - Usage: "Cache time to live for uploaded packages", - }, - &cli.BoolFlag{ - Name: "simulate-release", - Usage: "Only simulate creating release at grafana.com", - }, - }, - }, { Name: "grafana-com", Usage: "Publish packages to grafana.com", @@ -232,10 +329,94 @@ func main() { }, }, }, + { + Name: "github", + Usage: "Publish packages to GitHub releases", + Action: PublishGithub, + Flags: []cli.Flag{ + &dryRunFlag, + &cli.StringFlag{ + Name: "path", + Usage: "Path to the asset to be published", + }, + &cli.StringFlag{ + Name: "repo", + Required: true, + Usage: "GitHub repository", + }, + &cli.StringFlag{ + Name: "tag", + Usage: "Release tag (default from metadata)", + }, + &cli.BoolFlag{ + Name: "create", + Usage: "Create release if it doesn't exist", + }, + }, + }, + { + Name: "aws", + Usage: "Publish image to AWS Marketplace releases", + Action: PublishAwsMarketplace, + Flags: []cli.Flag{ + &dryRunFlag, + &cli.StringFlag{ + Name: "version", + Usage: "Release version (default from metadata)", + }, + &cli.StringFlag{ + Name: "image", + Required: true, + Usage: "Name of the image to be released", + }, + &cli.StringFlag{ + Name: "repo", + Required: true, + Usage: "AWS Marketplace ECR repository", + }, + &cli.StringFlag{ + Name: "product", + Required: true, + Usage: "AWS Marketplace product identifier", + }, + }, + }, + }, + }, + { + Name: "enterprise-check", + Usage: "Commands for testing against Grafana Enterprise", + Subcommands: cli.Commands{ + { + Name: "begin", + Usage: "Creates the GitHub check in a pull request and begins the tests", + Action: EnterpriseCheckBegin, + Flags: []cli.Flag{ + &gitHubTokenFlag, + }, + }, + { + Name: "success", + Usage: "Updates the GitHub check in a pull request to show a successful build and updates the pull request labels", + Action: EnterpriseCheckSuccess, + Flags: []cli.Flag{ + &gitHubTokenFlag, + }, + }, + { + Name: "fail", + Usage: "Updates the GitHub check in a pull request to show a failed build and updates the pull request labels", + Action: EnterpriseCheckFail, + Flags: []cli.Flag{ + &gitHubTokenFlag, + }, + }, }, }, } + app.Commands = append(app.Commands, additionalCommands...) + if err := app.Run(os.Args); err != nil { log.Fatalln(err) } diff --git a/pkg/build/cmd/npm.go b/pkg/build/cmd/npm.go new file mode 100644 index 00000000000..07c2fe1365f --- /dev/null +++ b/pkg/build/cmd/npm.go @@ -0,0 +1,88 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "strings" + + "github.com/grafana/grafana/pkg/build/npm" + "github.com/urfave/cli/v2" +) + +func NpmRetrieveAction(c *cli.Context) error { + if c.NArg() > 0 { + if err := cli.ShowSubcommandHelp(c); err != nil { + return cli.Exit(err.Error(), 1) + } + return cli.Exit("", 1) + } + + tag := c.String("tag") + if tag == "" { + return fmt.Errorf("no tag version specified, exitting") + } + + prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET")) + if prereleaseBucket == "" { + return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1) + } + + err := npm.FetchNpmPackages(c.Context, tag, prereleaseBucket) + if err != nil { + return err + } + return nil +} + +func NpmStoreAction(c *cli.Context) error { + if c.NArg() > 0 { + if err := cli.ShowSubcommandHelp(c); err != nil { + return cli.Exit(err.Error(), 1) + } + return cli.Exit("", 1) + } + + tag := c.String("tag") + if tag == "" { + return fmt.Errorf("no tag version specified, exiting") + } + + prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET")) + if prereleaseBucket == "" { + return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1) + } + + err := npm.StoreNpmPackages(c.Context, tag, prereleaseBucket) + if err != nil { + return err + } + return nil +} + +func NpmReleaseAction(c *cli.Context) error { + if c.NArg() > 0 { + if err := cli.ShowSubcommandHelp(c); err != nil { + return cli.Exit(err.Error(), 1) + } + return cli.Exit("", 1) + } + + tag := c.String("tag") + if tag == "" { + return fmt.Errorf("no tag version specified, exitting") + } + + cmd := exec.Command("git", "checkout", ".") + if err := cmd.Run(); err != nil { + fmt.Println("command failed to run, err: ", err) + return err + } + + err := npm.PublishNpmPackages(c.Context, tag) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/build/cmd/package.go b/pkg/build/cmd/package.go index 37b8d2e81fd..7f1b2da25bc 100644 --- a/pkg/build/cmd/package.go +++ b/pkg/build/cmd/package.go @@ -13,7 +13,7 @@ import ( ) func Package(c *cli.Context) error { - metadata, err := GenerateMetadata(c) + metadata, err := config.GenerateMetadata(c) if err != nil { return err } @@ -22,12 +22,12 @@ func Package(c *cli.Context) error { releaseMode, err := metadata.GetReleaseMode() if err != nil { - return cli.NewExitError(err.Error(), 1) + return cli.Exit(err.Error(), 1) } releaseModeConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode) if err != nil { - return cli.NewExitError(err.Error(), 1) + return cli.Exit(err.Error(), 1) } cfg := config.Config{ @@ -37,14 +37,17 @@ func Package(c *cli.Context) error { ctx := context.Background() - variantStrs := strings.Split(c.String("variants"), ",") variants := []config.Variant{} - for _, varStr := range variantStrs { - if varStr == "" { - continue + variantStrs := strings.Split(c.String("variants"), ",") + if c.String("variants") != "" { + for _, varStr := range variantStrs { + if varStr == "" { + continue + } + variants = append(variants, config.Variant(varStr)) } - - variants = append(variants, config.Variant(varStr)) + } else { + variants = releaseModeConfig.Variants } if len(variants) == 0 { diff --git a/pkg/build/cmd/publishartifacts.go b/pkg/build/cmd/publishartifacts.go new file mode 100644 index 00000000000..bd158d6d3ad --- /dev/null +++ b/pkg/build/cmd/publishartifacts.go @@ -0,0 +1,211 @@ +package main + +import ( + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/grafana/grafana/pkg/build/gcloud" + "github.com/grafana/grafana/pkg/build/versions" + "github.com/urfave/cli/v2" +) + +type publishConfig struct { + tag string + srcBucket string + destBucket string + enterprise2DestBucket string + enterprise2SecurityPrefix string + staticAssetsBucket string + staticAssetEditions []string + storybookBucket string + security bool +} + +// requireListWithEnvFallback first checks the CLI for a flag with the required +// name. If this is empty, it falls back to taking the environment variable. +// Sadly, we cannot use cli.Flag.EnvVars for this due to it potentially leaking +// environment variables as default values in usage-errors. +func requireListWithEnvFallback(cctx *cli.Context, name string, envName string) ([]string, error) { + result := cctx.StringSlice(name) + if len(result) == 0 { + for _, v := range strings.Split(os.Getenv(envName), ",") { + value := strings.TrimSpace(v) + if value != "" { + result = append(result, value) + } + } + } + if len(result) == 0 { + return nil, cli.Exit(fmt.Sprintf("Required flag (%s) or environment variable (%s) not set", name, envName), 1) + } + return result, nil +} + +func requireStringWithEnvFallback(cctx *cli.Context, name string, envName string) (string, error) { + result := cctx.String(name) + if result == "" { + result = os.Getenv(envName) + } + if result == "" { + return "", cli.Exit(fmt.Sprintf("Required flag (%s) or environment variable (%s) not set", name, envName), 1) + } + return result, nil +} + +// Action implements the sub-command "publish-artifacts". +func PublishArtifactsAction(c *cli.Context) error { + if c.NArg() > 0 { + if err := cli.ShowSubcommandHelp(c); err != nil { + return cli.Exit(err.Error(), 1) + } + return cli.Exit("", 1) + } + + staticAssetEditions, err := requireListWithEnvFallback(c, "static-asset-editions", "STATIC_ASSET_EDITIONS") + if err != nil { + return err + } + securityDestBucket, err := requireStringWithEnvFallback(c, "security-dest-bucket", "SECURITY_DEST_BUCKET") + if err != nil { + return err + } + enterprise2SecurityPrefix, err := requireStringWithEnvFallback(c, "enterprise2-security-prefix", "ENTERPRISE2_SECURITY_PREFIX") + if err != nil { + return err + } + + if err := gcloud.ActivateServiceAccount(); err != nil { + return fmt.Errorf("error connecting to gcp, %q", err) + } + + cfg := publishConfig{ + srcBucket: c.String("src-bucket"), + destBucket: c.String("dest-bucket"), + enterprise2DestBucket: c.String("enterprise2-dest-bucket"), + enterprise2SecurityPrefix: enterprise2SecurityPrefix, + staticAssetsBucket: c.String("static-assets-bucket"), + staticAssetEditions: staticAssetEditions, + storybookBucket: c.String("storybook-bucket"), + security: c.Bool("security"), + tag: strings.TrimPrefix(c.String("tag"), "v"), + } + + if cfg.security { + cfg.destBucket = securityDestBucket + } + + err = copyStaticAssets(cfg) + if err != nil { + return err + } + err = copyStorybook(cfg) + if err != nil { + return err + } + err = copyDownloads(cfg) + if err != nil { + return err + } + err = copyEnterprise2Downloads(cfg) + if err != nil { + return err + } + return nil +} + +func copyStaticAssets(cfg publishConfig) error { + for _, edition := range cfg.staticAssetEditions { + log.Printf("Copying static assets for %s", edition) + srcURL := fmt.Sprintf("%s/artifacts/static-assets/%s/%s/*", cfg.srcBucket, edition, cfg.tag) + destURL := fmt.Sprintf("%s/%s/%s/", cfg.staticAssetsBucket, edition, cfg.tag) + err := gcsCopy("static assets", srcURL, destURL) + if err != nil { + return fmt.Errorf("error copying static assets, %q", err) + } + } + log.Printf("Successfully copied static assets!") + return nil +} + +func copyStorybook(cfg publishConfig) error { + if cfg.security { + log.Printf("skipping storybook copy - not needed for a security release") + return nil + } + log.Printf("Copying storybooks...") + srcURL := fmt.Sprintf("%s/artifacts/storybook/v%s/*", cfg.srcBucket, cfg.tag) + destURL := fmt.Sprintf("%s/%s", cfg.storybookBucket, cfg.tag) + err := gcsCopy("storybook", srcURL, destURL) + if err != nil { + return fmt.Errorf("error copying storybook. %q", err) + } + stableVersion, err := versions.GetLatestVersion(versions.LatestStableVersionURL) + if err != nil { + return err + } + isLatest, err := versions.IsGreaterThanOrEqual(cfg.tag, stableVersion) + if err != nil { + return err + } + if isLatest { + log.Printf("Copying storybooks to latest...") + srcURL := fmt.Sprintf("%s/artifacts/storybook/v%s/*", cfg.srcBucket, cfg.tag) + destURL := fmt.Sprintf("%s/latest", cfg.storybookBucket) + err := gcsCopy("storybook (latest)", srcURL, destURL) + if err != nil { + return fmt.Errorf("error copying storybook to latest. %q", err) + } + } + + log.Printf("Successfully copied storybook!") + return nil +} + +func copyDownloads(cfg publishConfig) error { + for _, edition := range []string{ + "oss", "enterprise", + } { + destURL := fmt.Sprintf("%s/%s/", cfg.destBucket, edition) + srcURL := fmt.Sprintf("%s/artifacts/downloads/v%s/%s/release/*", cfg.srcBucket, cfg.tag, edition) + if !cfg.security { + destURL = filepath.Join(destURL, "release") + } + log.Printf("Copying downloads for %s, from %s bucket to %s bucket", edition, srcURL, destURL) + err := gcsCopy("downloads", srcURL, destURL) + if err != nil { + return fmt.Errorf("error copying downloads, %q", err) + } + } + log.Printf("Successfully copied downloads.") + return nil +} + +func copyEnterprise2Downloads(cfg publishConfig) error { + var prefix string + if cfg.security { + prefix = cfg.enterprise2SecurityPrefix + } + srcURL := fmt.Sprintf("%s/artifacts/downloads-enterprise2/v%s/enterprise2/release/*", cfg.srcBucket, cfg.tag) + destURL := fmt.Sprintf("%s/enterprise2/%srelease", cfg.enterprise2DestBucket, prefix) + log.Printf("Copying downloads for enterprise2, from %s bucket to %s bucket", srcURL, destURL) + err := gcsCopy("enterprise2 downloads", srcURL, destURL) + if err != nil { + return fmt.Errorf("error copying ") + } + return nil +} + +func gcsCopy(desc, src, dest string) error { + args := strings.Split(fmt.Sprintf("-m cp -r gs://%s gs://%s", src, dest), " ") + // nolint:gosec + cmd := exec.Command("gsutil", args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to publish %s: %w\n%s", desc, err, out) + } + return nil +} diff --git a/pkg/build/cmd/publishaws.go b/pkg/build/cmd/publishaws.go new file mode 100644 index 00000000000..e499c6b27fa --- /dev/null +++ b/pkg/build/cmd/publishaws.go @@ -0,0 +1,308 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ecr" + "github.com/aws/aws-sdk-go/service/marketplacecatalog" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "github.com/grafana/grafana/pkg/build/config" + "github.com/urfave/cli/v2" +) + +const ( + marketplaceChangeSetName = "Add new version" + marketplaceCatalogId = "AWSMarketplace" + marketplaceRegistryId = "709825985650" + marketplaceRegistryRegion = "us-east-1" + marketplaceRegistryUrl = "709825985650.dkr.ecr.us-east-1.amazonaws.com" + marketplaceRequestsUrl = "https://aws.amazon.com/marketplace/management/requests/" + releaseNotesTemplateUrl = "https://grafana.com/docs/grafana/latest/release-notes/release-notes-${TAG}/" + helmChartsUrl = "https://grafana.github.io/helm-charts/" + docsUrl = "https://grafana.com/docs/grafana/latest/enterprise/license/" + imagePlatform = "linux/amd64" + + publishAwsMarketplaceTestKey publishAwsMarketplaceTestKeyType = "test-client" +) + +var ( + errEmptyVersion = errors.New(`failed to retrieve release version from metadata, use "--version" to set it manually`) +) + +type publishAwsMarketplaceTestKeyType string + +type publishAwsMarketplaceFlags struct { + dryRun bool + version string + repo string + image string + product string +} + +type AwsMarketplacePublishingService struct { + auth string + docker AwsMarketplaceDocker + ecr AwsMarketplaceRegistry + mkt AwsMarketplaceCatalog +} + +type AwsMarketplaceDocker interface { + ImagePull(ctx context.Context, refStr string, options types.ImagePullOptions) (io.ReadCloser, error) + ImageTag(ctx context.Context, source string, target string) error + ImagePush(ctx context.Context, image string, options types.ImagePushOptions) (io.ReadCloser, error) +} + +type AwsMarketplaceRegistry interface { + GetAuthorizationTokenWithContext(ctx context.Context, input *ecr.GetAuthorizationTokenInput, opts ...request.Option) (*ecr.GetAuthorizationTokenOutput, error) +} + +type AwsMarketplaceCatalog interface { + DescribeEntityWithContext(ctx context.Context, input *marketplacecatalog.DescribeEntityInput, opts ...request.Option) (*marketplacecatalog.DescribeEntityOutput, error) + StartChangeSetWithContext(ctx context.Context, input *marketplacecatalog.StartChangeSetInput, opts ...request.Option) (*marketplacecatalog.StartChangeSetOutput, error) +} + +func PublishAwsMarketplace(ctx *cli.Context) error { + f, err := getPublishAwsMarketplaceFlags(ctx) + if err != nil { + return err + } + + if f.version == "" { + return errEmptyVersion + } + + svc, err := getAwsMarketplacePublishingService() + if err != nil { + return err + } + + if ctx.Context.Value(publishAwsMarketplaceTestKey) != nil { + svc = ctx.Context.Value(publishAwsMarketplaceTestKey).(*AwsMarketplacePublishingService) + } + + fmt.Println("Logging in to AWS Marketplace registry") + err = svc.Login(ctx.Context) + if err != nil { + return err + } + + fmt.Printf("Retrieving image '%s:%s' from Docker Hub\n", f.image, f.version) + err = svc.PullImage(ctx.Context, f.image, f.version) + if err != nil { + return err + } + + fmt.Printf("Renaming image '%s:%s' to '%s/%s:%s'\n", f.image, f.version, marketplaceRegistryUrl, f.repo, f.version) + err = svc.TagImage(ctx.Context, f.image, f.repo, f.version) + if err != nil { + return err + } + + if !f.dryRun { + fmt.Printf("Pushing image '%s/%s:%s' to the AWS Marketplace ECR\n", marketplaceRegistryUrl, f.repo, f.version) + err = svc.PushToMarketplace(ctx.Context, f.repo, f.version) + if err != nil { + return err + } + } else { + fmt.Printf("Dry-Run: Pushing image '%s/%s:%s' to the AWS Marketplace ECR\n", marketplaceRegistryUrl, f.repo, f.version) + } + + fmt.Printf("Retrieving product identifier for product '%s'\n", f.product) + pid, err := svc.GetProductIdentifier(ctx.Context, f.product) + if err != nil { + return err + } + + if !f.dryRun { + fmt.Printf("Releasing to product, you can view the progress of the release on %s\n", marketplaceRequestsUrl) + return svc.ReleaseToProduct(ctx.Context, pid, f.repo, f.version) + } else { + fmt.Printf("Dry-Run: Releasing to product, you can view the progress of the release on %s\n", marketplaceRequestsUrl) + } + + return nil +} + +func getAwsMarketplacePublishingService() (*AwsMarketplacePublishingService, error) { + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return nil, err + } + + mySession := session.Must(session.NewSession()) + ecr := ecr.New(mySession, aws.NewConfig().WithRegion(marketplaceRegistryRegion)) + mkt := marketplacecatalog.New(mySession, aws.NewConfig().WithRegion(marketplaceRegistryRegion)) + return &AwsMarketplacePublishingService{ + docker: cli, + ecr: ecr, + mkt: mkt, + }, nil +} + +func (s *AwsMarketplacePublishingService) Login(ctx context.Context) error { + out, err := s.ecr.GetAuthorizationTokenWithContext(ctx, &ecr.GetAuthorizationTokenInput{}) + if err != nil { + return err + } + s.auth = *out.AuthorizationData[0].AuthorizationToken + authData, err := base64.StdEncoding.DecodeString(s.auth) + if err != nil { + return err + } + authString := strings.Split(string(authData), ":") + authData, err = json.Marshal(types.AuthConfig{ + Username: authString[0], + Password: authString[1], + }) + s.auth = base64.StdEncoding.EncodeToString(authData) + return err +} + +func (s *AwsMarketplacePublishingService) PullImage(ctx context.Context, image string, version string) error { + reader, err := s.docker.ImagePull(ctx, fmt.Sprintf("%s:%s", image, version), types.ImagePullOptions{ + Platform: imagePlatform, + }) + if err != nil { + return err + } + + _, err = io.Copy(os.Stdout, reader) + if err != nil { + return err + } + + err = reader.Close() + if err != nil { + return err + } + return nil +} + +func (s *AwsMarketplacePublishingService) TagImage(ctx context.Context, image string, repo string, version string) error { + err := s.docker.ImageTag(ctx, fmt.Sprintf("%s:%s", image, version), fmt.Sprintf("%s/%s:%s", marketplaceRegistryUrl, repo, version)) + if err != nil { + return err + } + return nil +} + +func (s *AwsMarketplacePublishingService) PushToMarketplace(ctx context.Context, repo string, version string) error { + reader, err := s.docker.ImagePush(ctx, fmt.Sprintf("%s/%s:%s", marketplaceRegistryUrl, repo, version), types.ImagePushOptions{ + RegistryAuth: s.auth, + }) + if err != nil { + return err + } + + _, err = io.Copy(os.Stdout, reader) + if err != nil { + return err + } + + err = reader.Close() + if err != nil { + return err + } + return nil +} + +func (s *AwsMarketplacePublishingService) GetProductIdentifier(ctx context.Context, product string) (string, error) { + out, err := s.mkt.DescribeEntityWithContext(ctx, &marketplacecatalog.DescribeEntityInput{ + EntityId: aws.String(product), + Catalog: aws.String(marketplaceCatalogId), + }) + if err != nil { + return "", err + } + return *out.EntityIdentifier, nil +} + +func (s *AwsMarketplacePublishingService) ReleaseToProduct(ctx context.Context, pid string, repo string, version string) error { + _, err := s.mkt.StartChangeSetWithContext(ctx, &marketplacecatalog.StartChangeSetInput{ + Catalog: aws.String(marketplaceCatalogId), + ChangeSetName: aws.String(marketplaceChangeSetName), + ChangeSet: []*marketplacecatalog.Change{ + buildAwsMarketplaceChangeSet(pid, repo, version), + }, + }) + return err +} + +func getPublishAwsMarketplaceFlags(ctx *cli.Context) (*publishAwsMarketplaceFlags, error) { + metadata, err := config.GenerateMetadata(ctx) + if err != nil { + return nil, err + } + version := ctx.String("version") + if version == "" && metadata.GrafanaVersion != "" { + version = metadata.GrafanaVersion + } + image := ctx.String("image") + repo := ctx.String("repo") + product := ctx.String("product") + dryRun := ctx.Bool("dry-run") + return &publishAwsMarketplaceFlags{ + dryRun: dryRun, + version: version, + image: image, + repo: repo, + product: product, + }, nil +} + +func buildAwsMarketplaceReleaseNotesUrl(version string) string { + sanitizedVersion := strings.ReplaceAll(version, ".", "-") + return strings.ReplaceAll(releaseNotesTemplateUrl, "${TAG}", sanitizedVersion) +} + +func buildAwsMarketplaceChangeSet(entityId string, repo string, version string) *marketplacecatalog.Change { + return &marketplacecatalog.Change{ + ChangeType: aws.String("AddDeliveryOptions"), + Entity: &marketplacecatalog.Entity{ + Type: aws.String("ContainerProduct@1.0"), + Identifier: aws.String(entityId), + }, + Details: aws.String(buildAwsMarketplaceVersionDetails(repo, version)), + } +} + +func buildAwsMarketplaceVersionDetails(repo string, version string) string { + releaseNotesUrl := buildAwsMarketplaceReleaseNotesUrl(version) + return fmt.Sprintf(`{ + "Version": { + "ReleaseNotes": "Release notes are available on the website %s", + "VersionTitle": "v%s" + }, + "DeliveryOptions": [ + { + "Details": { + "EcrDeliveryOptionDetails": { + "DeploymentResources": [ + { + "Name": "Helm Charts", + "Url": "%s" + } + ], + "CompatibleServices": ["EKS", "ECS", "ECS-Anywhere", "EKS-Anywhere"], + "ContainerImages": ["%s/%s:%s"], + "Description": "Grafana Enterprise can be installed using the official Grafana Helm chart repository. The repository is available on Github: %s", + "UsageInstructions": "You can apply your Grafana Enterprise license to a new or existing Grafana Enterprise deployment by updating a configuration setting or environment variable. Your Grafana instance must be deployed on AWS, or have network access to AWS. For more information, see %s" + } + }, + "DeliveryOptionTitle": "Helm Chart" + } + ] + }`, releaseNotesUrl, version, helmChartsUrl, marketplaceRegistryUrl, repo, version, helmChartsUrl, docsUrl) +} diff --git a/pkg/build/cmd/publishaws_test.go b/pkg/build/cmd/publishaws_test.go new file mode 100644 index 00000000000..fc8f2615336 --- /dev/null +++ b/pkg/build/cmd/publishaws_test.go @@ -0,0 +1,207 @@ +package main + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "io" + "os" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/ecr" + "github.com/aws/aws-sdk-go/service/marketplacecatalog" + "github.com/docker/docker/api/types" + "github.com/stretchr/testify/assert" + "github.com/urfave/cli/v2" +) + +type awsPublishTestCase struct { + name string + args []string + expectedError error + errorContains string + expectedOutput string + mockedService *AwsMarketplacePublishingService +} + +func TestPublishAwsMarketplace(t *testing.T) { + t.Setenv("DRONE_BUILD_EVENT", "promote") + t.Setenv("DRONE_TAG", "v1.0.0") + t.Setenv("DRONE_COMMIT", "abcdefgh") + testApp := setupPublishAwsMarketplaceTests(t) + errShouldNotCallMock := errors.New("shouldn't call") + + testCases := []awsPublishTestCase{ + { + name: "try to publish without required flags", + errorContains: `Required flags "image, repo, product" not set`, + }, + { + name: "try to publish without credentials", + args: []string{"--image", "test/test", "--repo", "test/test", "--product", "test", "--version", "1.0.0"}, + mockedService: &AwsMarketplacePublishingService{ + ecr: &mockAwsMarketplaceRegistry{ + GetAuthorizationTokenWithContextError: credentials.ErrNoValidProvidersFoundInChain, + }, + }, + expectedError: credentials.ErrNoValidProvidersFoundInChain, + }, + { + name: "try to publish with valid credentials and nonexisting version", + args: []string{"--image", "test/test", "--repo", "test/test", "--product", "test", "--version", "1.0.0"}, + mockedService: &AwsMarketplacePublishingService{ + ecr: &mockAwsMarketplaceRegistry{}, + docker: &mockAwsMarketplaceDocker{}, + mkt: &mockAwsMarketplaceCatalog{}, + }, + expectedOutput: "Releasing to product", + }, + { + name: "try to publish with valid credentials and existing version", + args: []string{"--image", "test/test", "--repo", "test/test", "--product", "test", "--version", "1.0.0"}, + mockedService: &AwsMarketplacePublishingService{ + ecr: &mockAwsMarketplaceRegistry{}, + docker: &mockAwsMarketplaceDocker{}, + mkt: &mockAwsMarketplaceCatalog{}, + }, + expectedOutput: "Releasing to product", + }, + { + name: "dry run with invalid credentials", + args: []string{"--dry-run", "--image", "test/test", "--repo", "test/test", "--product", "test", "--version", "1.0.0"}, + mockedService: &AwsMarketplacePublishingService{ + ecr: &mockAwsMarketplaceRegistry{ + GetAuthorizationTokenWithContextError: credentials.ErrNoValidProvidersFoundInChain, + }, + }, + expectedError: credentials.ErrNoValidProvidersFoundInChain, + }, + { + name: "dry run with valid credentials", + args: []string{"--dry-run", "--image", "test/test", "--repo", "test/test", "--product", "test", "--version", "1.0.0"}, + mockedService: &AwsMarketplacePublishingService{ + ecr: &mockAwsMarketplaceRegistry{}, + docker: &mockAwsMarketplaceDocker{ + ImagePushError: errShouldNotCallMock, + }, + mkt: &mockAwsMarketplaceCatalog{ + StartChangeSetWithContextError: errShouldNotCallMock, + }, + }, + expectedOutput: "Dry-Run: Releasing to product", + }, + } + + if os.Getenv("DRONE_COMMIT") == "" { + // this test only works locally due to Drone environment + testCases = append(testCases, + awsPublishTestCase{ + name: "try to publish without version", + args: []string{"--image", "test/test", "--repo", "test/test", "--product", "test"}, + expectedError: errEmptyVersion, + }, + ) + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + ctx := context.WithValue(context.Background(), publishAwsMarketplaceTestKey, test.mockedService) + args := []string{"run"} + args = append(args, test.args...) + out, err := captureStdout(t, func() error { + return testApp.RunContext(ctx, args) + }) + if test.expectedOutput != "" { + assert.Contains(t, out, test.expectedOutput) + } + if test.expectedError != nil || test.errorContains != "" { + assert.Error(t, err) + if test.expectedError != nil { + assert.ErrorIs(t, err, test.expectedError) + } + if test.errorContains != "" { + assert.ErrorContains(t, err, test.errorContains) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +func setupPublishAwsMarketplaceTests(t *testing.T) *cli.App { + t.Helper() + testApp := cli.NewApp() + testApp.Action = PublishAwsMarketplace + testApp.Flags = []cli.Flag{ + &dryRunFlag, + &cli.StringFlag{ + Name: "version", + Usage: "Release version (default from metadata)", + }, + &cli.StringFlag{ + Name: "image", + Required: true, + Usage: "Name of the image to be released", + }, + &cli.StringFlag{ + Name: "repo", + Required: true, + Usage: "AWS Marketplace ECR repository", + }, + &cli.StringFlag{ + Name: "product", + Required: true, + Usage: "AWS Marketplace product identifier", + }, + } + return testApp +} + +type mockAwsMarketplaceDocker struct { + ImagePullError error + ImageTagError error + ImagePushError error +} + +func (m *mockAwsMarketplaceDocker) ImagePull(ctx context.Context, refStr string, options types.ImagePullOptions) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader([]byte(""))), m.ImagePullError +} +func (m *mockAwsMarketplaceDocker) ImageTag(ctx context.Context, source string, target string) error { + return m.ImageTagError +} +func (m *mockAwsMarketplaceDocker) ImagePush(ctx context.Context, image string, options types.ImagePushOptions) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader([]byte(""))), m.ImagePushError +} + +type mockAwsMarketplaceRegistry struct { + GetAuthorizationTokenWithContextError error +} + +func (m *mockAwsMarketplaceRegistry) GetAuthorizationTokenWithContext(ctx context.Context, input *ecr.GetAuthorizationTokenInput, opts ...request.Option) (*ecr.GetAuthorizationTokenOutput, error) { + return &ecr.GetAuthorizationTokenOutput{ + AuthorizationData: []*ecr.AuthorizationData{ + { + AuthorizationToken: aws.String(base64.StdEncoding.EncodeToString([]byte("username:password"))), + }, + }, + }, m.GetAuthorizationTokenWithContextError +} + +type mockAwsMarketplaceCatalog struct { + DescribeEntityWithContextError error + StartChangeSetWithContextError error +} + +func (m *mockAwsMarketplaceCatalog) DescribeEntityWithContext(ctx context.Context, input *marketplacecatalog.DescribeEntityInput, opts ...request.Option) (*marketplacecatalog.DescribeEntityOutput, error) { + return &marketplacecatalog.DescribeEntityOutput{ + EntityIdentifier: aws.String("productid"), + }, m.DescribeEntityWithContextError +} +func (m *mockAwsMarketplaceCatalog) StartChangeSetWithContext(ctx context.Context, input *marketplacecatalog.StartChangeSetInput, opts ...request.Option) (*marketplacecatalog.StartChangeSetOutput, error) { + return &marketplacecatalog.StartChangeSetOutput{}, m.StartChangeSetWithContextError +} diff --git a/pkg/build/cmd/publishgithub.go b/pkg/build/cmd/publishgithub.go new file mode 100644 index 00000000000..49366cecd27 --- /dev/null +++ b/pkg/build/cmd/publishgithub.go @@ -0,0 +1,162 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "strings" + + "github.com/google/go-github/github" + "github.com/grafana/grafana/pkg/build/config" + "github.com/urfave/cli/v2" + "golang.org/x/oauth2" +) + +type githubRepositoryService interface { + GetReleaseByTag(ctx context.Context, owner string, repo string, tag string) (*github.RepositoryRelease, *github.Response, error) + CreateRelease(ctx context.Context, owner string, repo string, release *github.RepositoryRelease) (*github.RepositoryRelease, *github.Response, error) + UploadReleaseAsset(ctx context.Context, owner string, repo string, id int64, opt *github.UploadOptions, file *os.File) (*github.ReleaseAsset, *github.Response, error) +} + +type githubRepo struct { + owner string + name string +} + +type publishGithubFlags struct { + create bool + dryRun bool + tag string + repo *githubRepo + artifactPath string +} + +var ( + newGithubClient = githubRepositoryClient + errTokenIsEmpty = errors.New("the environment variable GH_TOKEN must be set") + errTagIsEmpty = errors.New(`failed to retrieve release tag from metadata, use "--tag" to set it manually`) + errReleaseNotFound = errors.New(`release not found, use "--create" to create the release`) +) + +func PublishGithub(ctx *cli.Context) error { + token := os.Getenv("GH_TOKEN") + f, err := getPublishGithubFlags(ctx) + if err != nil { + return err + } + + if f.tag == "" { + return errTagIsEmpty + } + + if token == "" { + return errTokenIsEmpty + } + + if f.dryRun { + return runPublishGithubDryRun(f, token, ctx) + } + + client := newGithubClient(ctx.Context, token) + release, res, err := client.GetReleaseByTag(ctx.Context, f.repo.owner, f.repo.name, f.tag) + if err != nil && res.StatusCode != 404 { + return err + } + + if release == nil { + if f.create { + release, _, err = client.CreateRelease(ctx.Context, f.repo.owner, f.repo.name, &github.RepositoryRelease{TagName: &f.tag}) + if err != nil { + return err + } + } else { + return errReleaseNotFound + } + } + + artifactName := path.Base(f.artifactPath) + file, err := os.Open(f.artifactPath) + if err != nil { + return err + } + + asset, _, err := client.UploadReleaseAsset(ctx.Context, f.repo.owner, f.repo.name, *release.ID, &github.UploadOptions{Name: artifactName}, file) + if err != nil { + return err + } + fmt.Printf("Asset '%s' uploaded to release '%s' on repository '%s/%s'\nDownload: %s\n", *asset.Name, f.tag, f.repo.owner, f.repo.name, *asset.BrowserDownloadURL) + return nil +} + +func githubRepositoryClient(ctx context.Context, token string) githubRepositoryService { + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + tc := oauth2.NewClient(ctx, ts) + + client := github.NewClient(tc) + return client.Repositories +} + +func getPublishGithubFlags(ctx *cli.Context) (*publishGithubFlags, error) { + metadata, err := config.GenerateMetadata(ctx) + if err != nil { + return nil, err + } + tag := ctx.Value("tag").(string) + if tag == "" && metadata.GrafanaVersion != "" { + tag = fmt.Sprintf("v%s", metadata.GrafanaVersion) + } + fullRepo := ctx.Value("repo").(string) + dryRun := ctx.Value("dry-run").(bool) + owner := strings.Split(fullRepo, "/")[0] + name := strings.Split(fullRepo, "/")[1] + create := ctx.Value("create").(bool) + artifactPath := ctx.Value("path").(string) + if artifactPath == "" { + artifactPath = fmt.Sprintf("grafana-enterprise2-%s-amd64.img", metadata.GrafanaVersion) + fmt.Printf("path argument is not provided, resolving to default %s...\n", artifactPath) + } + return &publishGithubFlags{ + artifactPath: artifactPath, + create: create, + dryRun: dryRun, + tag: tag, + repo: &githubRepo{ + owner: owner, + name: name, + }, + }, nil +} + +func runPublishGithubDryRun(f *publishGithubFlags, token string, ctx *cli.Context) error { + client := newGithubClient(ctx.Context, token) + fmt.Println("Dry-Run: Retrieving release on repository by tag") + release, res, err := client.GetReleaseByTag(ctx.Context, f.repo.owner, f.repo.name, f.tag) + if err != nil && res.StatusCode != 404 { + fmt.Println("Dry-Run: Github communication error:\n", err) + return nil + } + + if release == nil { + if f.create { + fmt.Println("Dry-Run: Release doesn't exist and --create is enabled, so it would try to create the release") + } else { + fmt.Println("Dry-Run: Release doesn't exist and --create is disabled, so it would fail with error") + return nil + } + } + + artifactName := path.Base(f.artifactPath) + fmt.Printf("Dry-Run: Opening file for release: %s\n", f.artifactPath) + _, err = os.Open(f.artifactPath) + if err != nil { + fmt.Println("Dry-Run: Error opening file\n", err) + return nil + } + + fmt.Printf("Dry-Run: Would upload asset '%s' to release '%s' on repo '%s/%s' and return download URL if successful\n", artifactName, f.tag, f.repo.owner, f.repo.name) + return nil +} diff --git a/pkg/build/cmd/publishgithub_test.go b/pkg/build/cmd/publishgithub_test.go new file mode 100644 index 00000000000..6c9b0b18934 --- /dev/null +++ b/pkg/build/cmd/publishgithub_test.go @@ -0,0 +1,226 @@ +package main + +import ( + "context" + "errors" + "io" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/google/go-github/github" + "github.com/stretchr/testify/assert" + "github.com/urfave/cli/v2" +) + +type githubPublishTestCases struct { + name string + args []string + token string + expectedError error + errorContains string + expectedOutput string + mockedService *mockGithubRepositoryServiceImpl +} + +var mockGithubRepositoryService = &mockGithubRepositoryServiceImpl{} + +func mockGithubRepositoryClient(context.Context, string) githubRepositoryService { + return mockGithubRepositoryService +} + +func TestPublishGithub(t *testing.T) { + t.Setenv("DRONE_BUILD_EVENT", "promote") + t.Setenv("DRONE_TAG", "v1.0.0") + t.Setenv("DRONE_COMMIT", "abcdefgh") + testApp, testPath := setupPublishGithubTests(t) + mockErrUnauthorized := errors.New("401") + + testCases := []githubPublishTestCases{ + { + name: "try to publish without required flags", + errorContains: `Required flags "path, repo" not set`, + }, + { + name: "try to publish without token", + args: []string{"--path", testPath, "--repo", "test/test", "--tag", "v1.0.0"}, + expectedError: errTokenIsEmpty, + }, + { + name: "try to publish with invalid token", + token: "invalid", + args: []string{"--path", testPath, "--repo", "test/test", "--tag", "v1.0.0"}, + mockedService: &mockGithubRepositoryServiceImpl{tagErr: mockErrUnauthorized}, + expectedError: mockErrUnauthorized, + }, + { + name: "try to publish with valid token and nonexisting tag with create disabled", + token: "valid", + args: []string{"--path", testPath, "--repo", "test/test", "--tag", "v1.0.0"}, + mockedService: &mockGithubRepositoryServiceImpl{tagErr: errReleaseNotFound}, + expectedError: errReleaseNotFound, + }, + { + name: "try to publish with valid token and nonexisting tag with create enabled", + token: "valid", + args: []string{"--path", testPath, "--repo", "test/test", "--tag", "v1.0.0", "--create"}, + mockedService: &mockGithubRepositoryServiceImpl{tagErr: errReleaseNotFound}, + }, + { + name: "try to publish with valid token and existing tag", + token: "valid", + args: []string{"--path", testPath, "--repo", "test/test", "--tag", "v1.0.0"}, + }, + { + name: "dry run with invalid token", + token: "invalid", + args: []string{"--dry-run", "--path", testPath, "--repo", "test/test", "--tag", "v1.0.0"}, + mockedService: &mockGithubRepositoryServiceImpl{tagErr: mockErrUnauthorized}, + expectedOutput: "Github communication error", + }, + { + name: "dry run with valid token and nonexisting tag with create disabled", + token: "valid", + args: []string{"--dry-run", "--path", testPath, "--repo", "test/test", "--tag", "v1.0.0"}, + mockedService: &mockGithubRepositoryServiceImpl{tagErr: errReleaseNotFound}, + expectedOutput: "Release doesn't exist", + }, + { + name: "dry run with valid token and nonexisting tag with create enabled", + token: "valid", + args: []string{"--dry-run", "--path", testPath, "--repo", "test/test", "--tag", "v1.0.0", "--create"}, + mockedService: &mockGithubRepositoryServiceImpl{tagErr: errReleaseNotFound}, + expectedOutput: "Would upload asset", + }, + { + name: "dry run with valid token and existing tag", + token: "valid", + args: []string{"--dry-run", "--path", testPath, "--repo", "test/test", "--tag", "v1.0.0"}, + expectedOutput: "Would upload asset", + }, + } + + if os.Getenv("DRONE_COMMIT") == "" { + // this test only works locally due to Drone environment + testCases = append(testCases, + githubPublishTestCases{ + name: "try to publish without tag", + args: []string{"--path", testPath, "--repo", "test/test"}, + expectedError: errTagIsEmpty, + }, + ) + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + if test.token != "" { + t.Setenv("GH_TOKEN", test.token) + } + if test.mockedService != nil { + mockGithubRepositoryService = test.mockedService + } else { + mockGithubRepositoryService = &mockGithubRepositoryServiceImpl{} + } + args := []string{"run"} + args = append(args, test.args...) + out, err := captureStdout(t, func() error { + return testApp.Run(args) + }) + if test.expectedOutput != "" { + assert.Contains(t, out, test.expectedOutput) + } + if test.expectedError != nil || test.errorContains != "" { + assert.Error(t, err) + if test.expectedError != nil { + assert.ErrorIs(t, err, test.expectedError) + } + if test.errorContains != "" { + assert.ErrorContains(t, err, test.errorContains) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +func setupPublishGithubTests(t *testing.T) (*cli.App, string) { + t.Helper() + ex, err := os.Executable() + if err != nil { + panic(err) + } + testPath := filepath.Dir(ex) + + newGithubClient = mockGithubRepositoryClient + + testApp := cli.NewApp() + testApp.Action = PublishGithub + testApp.Flags = []cli.Flag{ + &dryRunFlag, + &cli.StringFlag{ + Name: "path", + Required: true, + Usage: "Path to the asset to be published", + }, + &cli.StringFlag{ + Name: "repo", + Required: true, + Usage: "Github repository", + }, + &cli.StringFlag{ + Name: "tag", + Usage: "Release tag (default from metadata)ß", + }, + &cli.BoolFlag{ + Name: "create", + Usage: "Create release if it doesn't exist", + }, + } + return testApp, testPath +} + +func captureStdout(t *testing.T, fn func() error) (string, error) { + t.Helper() + rescueStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + err := fn() + werr := w.Close() + if werr != nil { + return "", err + } + out, _ := io.ReadAll(r) + os.Stdout = rescueStdout + return string(out), err +} + +type mockGithubRepositoryServiceImpl struct { + tagErr error + createErr error + uploadErr error +} + +func (m *mockGithubRepositoryServiceImpl) GetReleaseByTag(ctx context.Context, owner string, repo string, tag string) (*github.RepositoryRelease, *github.Response, error) { + var release *github.RepositoryRelease + res := &github.Response{Response: &http.Response{}} + if m.tagErr == nil { + releaseID := int64(1) + release = &github.RepositoryRelease{ID: &releaseID} + } else if errors.Is(m.tagErr, errReleaseNotFound) { + res.StatusCode = 404 + } + return release, res, m.tagErr +} + +func (m *mockGithubRepositoryServiceImpl) CreateRelease(ctx context.Context, owner string, repo string, release *github.RepositoryRelease) (*github.RepositoryRelease, *github.Response, error) { + releaseID := int64(1) + return &github.RepositoryRelease{ID: &releaseID}, &github.Response{}, m.createErr +} + +func (m *mockGithubRepositoryServiceImpl) UploadReleaseAsset(ctx context.Context, owner string, repo string, id int64, opt *github.UploadOptions, file *os.File) (*github.ReleaseAsset, *github.Response, error) { + assetName := "test" + assetUrl := "testurl.com.br" + return &github.ReleaseAsset{Name: &assetName, BrowserDownloadURL: &assetUrl}, &github.Response{}, m.uploadErr +} diff --git a/pkg/build/cmd/publishimages_enterprise2.go b/pkg/build/cmd/publishimages_enterprise2.go new file mode 100644 index 00000000000..b76b2688d6f --- /dev/null +++ b/pkg/build/cmd/publishimages_enterprise2.go @@ -0,0 +1,100 @@ +package main + +import ( + "fmt" + "log" + "os" + "os/exec" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/build/docker" + "github.com/grafana/grafana/pkg/build/gcloud" + "github.com/urfave/cli/v2" +) + +func Enterprise2(c *cli.Context) error { + if c.NArg() > 0 { + if err := cli.ShowSubcommandHelp(c); err != nil { + return cli.Exit(err.Error(), 1) + } + return cli.Exit("", 1) + } + + if err := gcloud.ActivateServiceAccount(); err != nil { + return fmt.Errorf("couldn't activate service account, err: %w", err) + } + + metadata, err := config.GenerateMetadata(c) + if err != nil { + return err + } + + buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode) + if err != nil { + return err + } + + cfg := docker.Config{ + Archs: buildConfig.Docker.Architectures, + Distribution: buildConfig.Docker.Distribution, + DockerHubRepo: c.String("dockerhub-repo"), + Tag: metadata.GrafanaVersion, + } + + err = dockerLoginEnterprise2() + if err != nil { + return err + } + + var distributionStr []string + for _, distribution := range cfg.Distribution { + switch distribution { + case alpine: + distributionStr = append(distributionStr, "") + case ubuntu: + distributionStr = append(distributionStr, "-ubuntu") + default: + return fmt.Errorf("unrecognized distribution %q", distribution) + } + } + + for _, distribution := range distributionStr { + var imageFileNames []string + for _, arch := range cfg.Archs { + imageFilename := fmt.Sprintf("%s:%s%s-%s", cfg.DockerHubRepo, cfg.Tag, distribution, arch) + err := docker.PushImage(imageFilename) + if err != nil { + return err + } + imageFileNames = append(imageFileNames, imageFilename) + } + manifest := fmt.Sprintf("%s:%s%s", cfg.DockerHubRepo, cfg.Tag, distribution) + args := []string{"manifest", "create", manifest} + args = append(args, imageFileNames...) + + //nolint:gosec + cmd := exec.Command("docker", args...) + cmd.Env = append(os.Environ(), "DOCKER_CLI_EXPERIMENTAL=enabled") + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create Docker manifest: %w\n%s", err, output) + } + + err = docker.PushManifest(manifest) + if err != nil { + return err + } + } + + return nil +} + +func dockerLoginEnterprise2() error { + log.Println("Docker login...") + cmd := exec.Command("gcloud", "auth", "configure-docker") + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("error logging in to DockerHub: %s %q", out, err) + } + + log.Println("Successful login!") + return nil +} diff --git a/pkg/build/cmd/publishmetrics.go b/pkg/build/cmd/publishmetrics.go index d88911fc328..9d0576891fd 100644 --- a/pkg/build/cmd/publishmetrics.go +++ b/pkg/build/cmd/publishmetrics.go @@ -8,8 +8,9 @@ import ( "os" "regexp" - "github.com/grafana/grafana/pkg/build/metrics" "github.com/urfave/cli/v2" + + "github.com/grafana/grafana/pkg/build/metrics" ) func PublishMetrics(c *cli.Context) error { @@ -17,24 +18,24 @@ func PublishMetrics(c *cli.Context) error { input, err := io.ReadAll(os.Stdin) if err != nil { - return cli.NewExitError(fmt.Sprintf("Reading from stdin failed: %s", err), 1) + return cli.Exit(fmt.Sprintf("Reading from stdin failed: %s", err), 1) } reMetrics := regexp.MustCompile(`(?ms)^Metrics: (\{.+\})`) ms := reMetrics.FindSubmatch(input) if len(ms) == 0 { - return cli.NewExitError(fmt.Sprintf("Input on wrong format: %q", string(input)), 1) + return cli.Exit(fmt.Sprintf("Input on wrong format: %q", string(input)), 1) } m := map[string]string{} if err := json.Unmarshal(ms[1], &m); err != nil { - return cli.NewExitError(fmt.Sprintf("decoding metrics failed: %s", err), 1) + return cli.Exit(fmt.Sprintf("decoding metrics failed: %s", err), 1) } log.Printf("Received metrics %+v", m) if err := metrics.Publish(m, apiKey); err != nil { - return cli.NewExitError(fmt.Sprintf("publishing metrics failed: %s", err), 1) + return cli.Exit(fmt.Sprintf("publishing metrics failed: %s", err), 1) } return nil diff --git a/pkg/build/cmd/publishpackages.go b/pkg/build/cmd/publishpackages.go deleted file mode 100644 index a44e30503aa..00000000000 --- a/pkg/build/cmd/publishpackages.go +++ /dev/null @@ -1,117 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "strings" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/fsutil" - "github.com/grafana/grafana/pkg/build/gcloud" - "github.com/grafana/grafana/pkg/build/gpg" - "github.com/grafana/grafana/pkg/build/packaging" - "github.com/urfave/cli/v2" -) - -// PublishPackages implements the sub-command "publish-packages". -func PublishPackages(c *cli.Context) error { - if err := gcloud.ActivateServiceAccount(); err != nil { - return fmt.Errorf("couldn't activate service account, err: %w", err) - } - - metadata, err := GenerateMetadata(c) - if err != nil { - return err - } - - releaseMode, err := metadata.GetReleaseMode() - if err != nil { - return err - } - - dryRun := c.Bool("dry-run") - simulateRelease := c.Bool("simulate-release") - // Test release mode and dryRun imply simulateRelease - if releaseMode.IsTest || dryRun { - simulateRelease = true - } - - grafanaAPIKey := strings.TrimSpace(os.Getenv("GRAFANA_COM_API_KEY")) - if grafanaAPIKey == "" { - return cli.NewExitError("the environment variable GRAFANA_COM_API_KEY must be set", 1) - } - - edition := config.Edition(c.String("edition")) - - // TODO: Verify config values - cfg := packaging.PublishConfig{ - Config: config.Config{ - Version: metadata.GrafanaVersion, - Bucket: c.String("packages-bucket"), - DebDBBucket: c.String("deb-db-bucket"), - DebRepoBucket: c.String("deb-repo-bucket"), - RPMRepoBucket: c.String("rpm-repo-bucket"), - }, - Edition: edition, - ReleaseMode: releaseMode, - GrafanaAPIKey: grafanaAPIKey, - DryRun: dryRun, - TTL: c.String("ttl"), - SimulateRelease: simulateRelease, - } - if err := gpg.LoadGPGKeys(&cfg.Config); err != nil { - return err - } - defer gpg.RemoveGPGFiles(cfg.Config) - - // Only update package manager repos for releases. - // In test release mode, the operator should configure different GCS buckets for the package repos, - // so should be safe. - if cfg.ReleaseMode.Mode == config.TagMode { - workDir, err := fsutil.CreateTempDir("") - if err != nil { - return err - } - defer func() { - if err := os.RemoveAll(workDir); err != nil { - log.Printf("Failed to remove temporary directory %q: %s\n", workDir, err.Error()) - } - }() - if err := updatePkgRepos(cfg, workDir); err != nil { - return err - } - } - - log.Println("Successfully published packages!") - return nil -} - -// updatePkgRepos updates package manager repositories. -func updatePkgRepos(cfg packaging.PublishConfig, workDir string) error { - if err := gpg.Import(cfg.Config); err != nil { - return err - } - - // If updating the Deb repo fails, still continue with the RPM repo, so we don't have to retry - // both by hand - debErr := packaging.UpdateDebRepo(cfg, workDir) - if debErr != nil { - log.Printf("Updating Deb repo failed: %s\n", debErr) - } - rpmErr := packaging.UpdateRPMRepo(cfg, workDir) - if rpmErr != nil { - log.Printf("Updating RPM repo failed: %s\n", rpmErr) - } - - if debErr != nil { - return debErr - } - if rpmErr != nil { - return rpmErr - } - - log.Println("Updated Deb and RPM repos successfully!") - - return nil -} diff --git a/pkg/build/cmd/storestorybook.go b/pkg/build/cmd/storestorybook.go index 0daeb7080f6..65adfa1fbb7 100644 --- a/pkg/build/cmd/storestorybook.go +++ b/pkg/build/cmd/storestorybook.go @@ -13,7 +13,7 @@ import ( func StoreStorybook(c *cli.Context) error { deployment := c.String("deployment") - metadata, err := GenerateMetadata(c) + metadata, err := config.GenerateMetadata(c) if err != nil { return err } diff --git a/pkg/build/cmd/uploadcdn.go b/pkg/build/cmd/uploadcdn.go new file mode 100644 index 00000000000..8b8ddb0cd81 --- /dev/null +++ b/pkg/build/cmd/uploadcdn.go @@ -0,0 +1,75 @@ +package main + +import ( + "fmt" + "log" + "os" + "path/filepath" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/build/gcloud/storage" + "github.com/urfave/cli/v2" +) + +// UploadCDN implements the sub-command "upload-cdn". +func UploadCDN(c *cli.Context) error { + if c.NArg() > 0 { + if err := cli.ShowSubcommandHelp(c); err != nil { + return cli.Exit(err.Error(), 1) + } + return cli.Exit("", 1) + } + + metadata, err := config.GenerateMetadata(c) + if err != nil { + return err + } + + version := metadata.GrafanaVersion + if err != nil { + return cli.Exit(err.Error(), 1) + } + + buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode) + if err != nil { + return err + } + + edition := os.Getenv("EDITION") + log.Printf("Uploading Grafana CDN Assets, version %s, %s edition...", version, edition) + + editionPath := "" + + switch config.Edition(edition) { + case config.EditionOSS: + editionPath = "grafana-oss" + case config.EditionEnterprise: + editionPath = "grafana" + case config.EditionEnterprise2: + editionPath = os.Getenv("ENTERPRISE2_CDN_PATH") + default: + panic(fmt.Sprintf("unrecognized edition %q", edition)) + } + + gcs, err := storage.New() + if err != nil { + return err + } + + bucket := gcs.Bucket(buildConfig.Buckets.CDNAssets) + srcPath := buildConfig.Buckets.CDNAssetsDir + srcPath = filepath.Join(srcPath, editionPath, version) + + if err := gcs.DeleteDir(c.Context, bucket, srcPath); err != nil { + return err + } + log.Printf("Successfully cleaned source: %s/%s\n", buildConfig.Buckets.CDNAssets, srcPath) + + if err := gcs.CopyLocalDir(c.Context, "./public", bucket, srcPath, false); err != nil { + return err + } + + log.Printf("Successfully uploaded cdn static assets to: %s/%s!\n", buildConfig.Buckets.CDNAssets, srcPath) + + return nil +} diff --git a/pkg/build/cmd/uploadpackages.go b/pkg/build/cmd/uploadpackages.go new file mode 100644 index 00000000000..3270948ae66 --- /dev/null +++ b/pkg/build/cmd/uploadpackages.go @@ -0,0 +1,213 @@ +package main + +import ( + "encoding/base64" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/build/droneutil" + "github.com/grafana/grafana/pkg/build/gcloud" + "github.com/grafana/grafana/pkg/build/packaging" + "github.com/urfave/cli/v2" +) + +const releaseFolder = "release" +const mainFolder = "main" +const releaseBranchFolder = "prerelease" + +type uploadConfig struct { + config.Config + + edition config.Edition + versionMode config.VersionMode + gcpKey string + distDir string + versionFolder string +} + +// UploadPackages implements the sub-command "upload-packages". +func UploadPackages(c *cli.Context) error { + if c.NArg() > 0 { + if err := cli.ShowSubcommandHelp(c); err != nil { + return cli.Exit(err.Error(), 1) + } + return cli.Exit("", 1) + } + + gcpKeyB64 := strings.TrimSpace(os.Getenv("GCP_KEY")) + if gcpKeyB64 == "" { + return cli.Exit("the environment variable GCP_KEY must be set", 1) + } + gcpKeyB, err := base64.StdEncoding.DecodeString(gcpKeyB64) + if err != nil { + return cli.Exit("failed to base64 decode $GCP_KEY", 1) + } + gcpKey := string(gcpKeyB) + + distDir, err := filepath.Abs("dist") + if err != nil { + return cli.Exit(err.Error(), 1) + } + + metadata, err := config.GenerateMetadata(c) + if err != nil { + return err + } + + version := metadata.GrafanaVersion + + releaseMode, err := metadata.GetReleaseMode() + if err != nil { + return cli.Exit(err.Error(), 1) + } + + releaseModeConfig, err := config.GetBuildConfig(releaseMode.Mode) + if err != nil { + return cli.Exit(err.Error(), 1) + } + + var edition config.Edition + if e, ok := os.LookupEnv("EDITION"); ok { + edition = config.Edition(e) + } + + if c.Bool("enterprise2") { + edition = config.EditionEnterprise2 + } + + if edition == "" { + return fmt.Errorf("both EDITION envvar and '--enterprise2' flag are missing. At least one of those is required") + } + + // TODO: Verify config values + cfg := uploadConfig{ + Config: config.Config{ + Version: version, + Bucket: releaseModeConfig.Buckets.Artifacts, + }, + edition: edition, + versionMode: releaseMode.Mode, + gcpKey: gcpKey, + distDir: distDir, + } + + event, err := droneutil.GetDroneEventFromEnv() + if err != nil { + return err + } + + if cfg.edition == config.EditionEnterprise2 { + cfg.Bucket, err = bucketForEnterprise2(releaseModeConfig, event) + if err != nil { + return err + } + } + + cfg.versionFolder, err = getVersionFolder(cfg, event) + if err != nil { + return err + } + + if err := uploadPackages(cfg); err != nil { + return cli.Exit(err.Error(), 1) + } + + log.Println("Successfully uploaded packages!") + return nil +} + +// Corner case for custom enterprise2 mode +func bucketForEnterprise2(releaseModeConfig *config.BuildConfig, event string) (string, error) { + if event == config.Custom { + buildConfig, err := config.GetBuildConfig(config.ReleaseBranchMode) + if err != nil { + return "", err + } + return buildConfig.Buckets.ArtifactsEnterprise2, nil + } + + if releaseModeConfig.Buckets.ArtifactsEnterprise2 != "" { + return releaseModeConfig.Buckets.ArtifactsEnterprise2, nil + } + + return "", fmt.Errorf("enterprise2 bucket var doesn't exist") +} + +func getVersionFolder(cfg uploadConfig, event string) (string, error) { + switch cfg.versionMode { + case config.TagMode: + return releaseFolder, nil + case config.MainMode, config.DownstreamMode: + return mainFolder, nil + case config.ReleaseBranchMode: + return releaseBranchFolder, nil + default: + // Corner case for custom enterprise2 mode + if event == config.Custom && cfg.versionMode == config.Enterprise2Mode { + return releaseFolder, nil + } + return "", fmt.Errorf("unrecognized version mode: %s", cfg.versionMode) + } +} + +func uploadPackages(cfg uploadConfig) error { + log.Printf("Uploading Grafana packages, version %s, %s edition, %s mode...\n", cfg.Version, cfg.edition, + cfg.versionMode) + + if err := gcloud.ActivateServiceAccount(); err != nil { + return fmt.Errorf("couldn't activate service account, err: %w", err) + } + + edition := strings.ToLower(string(cfg.edition)) + + var sfx string + switch cfg.edition { + case config.EditionOSS: + case config.EditionEnterprise: + sfx = "-enterprise" + case config.EditionEnterprise2: + sfx = "-enterprise2" + default: + panic(fmt.Sprintf("unrecognized edition %q", cfg.edition)) + } + matches, err := filepath.Glob(filepath.Join(cfg.distDir, fmt.Sprintf("grafana%s*", sfx))) + if err != nil { + return fmt.Errorf("failed to list packages: %w", err) + } + fpaths := []string{} + rePkg := packaging.PackageRegexp(cfg.edition) + for _, fpath := range matches { + fname := filepath.Base(fpath) + if strings.Contains(fname, "latest") || !rePkg.MatchString(fname) { + log.Printf("Ignoring file %q\n", fpath) + continue + } + + fpaths = append(fpaths, fpath) + } + + var tag, gcsPath string + droneTag := strings.TrimSpace(os.Getenv("DRONE_TAG")) + if droneTag != "" { + tag = droneTag + gcsPath = fmt.Sprintf("gs://%s/%s/%s/%s", cfg.Bucket, tag, edition, cfg.versionFolder) + } else { + gcsPath = fmt.Sprintf("gs://%s/%s/%s/", cfg.Bucket, edition, cfg.versionFolder) + } + log.Printf("Uploading %d file(s) to GCS (%s)...\n", len(fpaths), gcsPath) + + args := []string{"-m", "cp"} + args = append(args, fpaths...) + args = append(args, gcsPath) + cmd := exec.Command("gsutil", args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to upload files to GCS: %s", output) + } + + return nil +} diff --git a/pkg/build/cmd/uploadpackages_test.go b/pkg/build/cmd/uploadpackages_test.go new file mode 100644 index 00000000000..cdb7bdab945 --- /dev/null +++ b/pkg/build/cmd/uploadpackages_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "errors" + "fmt" + "testing" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_getVersionFolder(t *testing.T) { + type args struct { + cfg uploadConfig + event string + versionFolder string + } + tests := []struct { + name string + args args + err error + }{ + {"tag mode", args{uploadConfig{versionMode: config.TagMode}, "", releaseFolder}, nil}, + {"main mode", args{uploadConfig{versionMode: config.MainMode}, "", mainFolder}, nil}, + {"downstream mode", args{uploadConfig{versionMode: config.DownstreamMode}, "", mainFolder}, nil}, + {"release branch mode", args{uploadConfig{versionMode: config.ReleaseBranchMode}, "", releaseBranchFolder}, nil}, + {"enterprise pro mode", args{uploadConfig{versionMode: config.Enterprise2Mode}, config.Custom, releaseFolder}, nil}, + {"unrecognised version mode", args{uploadConfig{versionMode: "foo"}, config.Custom, ""}, errors.New("")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + versionMode, err := getVersionFolder(tt.args.cfg, tt.args.event) + if tt.err != nil { + require.Error(t, err) + } + require.Equal(t, versionMode, tt.args.versionFolder) + }) + } +} + +func Test_checkForEnterprise2Edition(t *testing.T) { + type args struct { + releaseModeConfig *config.BuildConfig + event string + } + tests := []struct { + name string + args args + want string + err error + }{ + {"event is not custom", args{releaseModeConfig: &config.BuildConfig{Buckets: config.Buckets{ArtifactsEnterprise2: "dummy"}}}, "dummy", nil}, + {"event is not custom and string is empty", args{releaseModeConfig: &config.BuildConfig{Buckets: config.Buckets{ArtifactsEnterprise2: ""}}}, "", fmt.Errorf("enterprise2 bucket var doesn't exist")}, + {"event is custom", args{releaseModeConfig: nil, event: "custom"}, "grafana-downloads-enterprise2", nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := bucketForEnterprise2(tt.args.releaseModeConfig, tt.args.event) + if tt.err != nil { + require.Error(t, err) + } + assert.Equalf(t, tt.want, got, "bucketForEnterprise2(%v, %v)", tt.args.releaseModeConfig, tt.args.event) + }) + } +} diff --git a/pkg/build/cmd/verifydrone.go b/pkg/build/cmd/verifydrone.go index ed34e0f77df..19da6fb5a35 100644 --- a/pkg/build/cmd/verifydrone.go +++ b/pkg/build/cmd/verifydrone.go @@ -13,10 +13,11 @@ import ( "github.com/google/go-cmp/cmp" - "github.com/grafana/grafana/pkg/build/fsutil" cliv1 "github.com/urfave/cli" "github.com/urfave/cli/v2" "gopkg.in/yaml.v3" + + "github.com/grafana/grafana/pkg/build/fsutil" ) func VerifyDrone(c *cli.Context) error { @@ -24,7 +25,7 @@ func VerifyDrone(c *cli.Context) error { const backup = ".drone.yml.bak" if err := fsutil.CopyFile(yml, backup); err != nil { - return cli.NewExitError(fmt.Sprintf("failed to copy %s to %s: %s", yml, backup, err), 1) + return cli.Exit(fmt.Sprintf("failed to copy %s to %s: %s", yml, backup, err), 1) } defer func() { if err := os.Remove(yml); err != nil { @@ -73,7 +74,7 @@ func readConfig(fpath string) ([]map[string]interface{}, error) { //nolint:gosec f, err := os.Open(fpath) if err != nil { - return nil, cli.NewExitError(fmt.Sprintf("failed to read %s: %s", fpath, err), 1) + return nil, cli.Exit(fmt.Sprintf("failed to read %s: %s", fpath, err), 1) } defer func() { if err := f.Close(); err != nil { @@ -90,7 +91,7 @@ func readConfig(fpath string) ([]map[string]interface{}, error) { if errors.Is(err, io.EOF) { break } - return nil, cli.NewExitError(fmt.Sprintf("Failed to decode %s: %s", fpath, err), 1) + return nil, cli.Exit(fmt.Sprintf("Failed to decode %s: %s", fpath, err), 1) } if m["kind"] == "signature" { @@ -118,7 +119,7 @@ func verifyYAML(yml, backup string) error { } if !cmp.Equal(c1, c2) { - return cli.NewExitError(fmt.Sprintf("%s is out of sync with .drone.star - regenerate it with drone starlark convert", + return cli.Exit(fmt.Sprintf("%s is out of sync with .drone.star - regenerate it with drone starlark convert", yml), 1) } diff --git a/pkg/build/cmd/verifystarlark.go b/pkg/build/cmd/verifystarlark.go new file mode 100644 index 00000000000..ff33a77a3af --- /dev/null +++ b/pkg/build/cmd/verifystarlark.go @@ -0,0 +1,142 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os/exec" + "path/filepath" + "strings" + + "github.com/urfave/cli/v2" +) + +func mapSlice[I any, O any](a []I, f func(I) O) []O { + o := make([]O, len(a)) + for i, e := range a { + o[i] = f(e) + } + return o +} + +// VerifyStarlark is the CLI Action for verifying Starlark files in a workspace. +// It expects a single context argument which is the path to the workspace. +// The actual verification procedure can return multiple errors which are +// joined together to be one holistic error for the action. +func VerifyStarlark(c *cli.Context) error { + if c.NArg() != 1 { + var message string + if c.NArg() == 0 { + message = "ERROR: missing required argument " + } + if c.NArg() > 1 { + message = "ERROR: too many arguments" + } + + if err := cli.ShowSubcommandHelp(c); err != nil { + return err + } + + return cli.Exit(message, 1) + } + + workspace := c.Args().Get(0) + verificationErrs, executionErr := verifyStarlark(c.Context, workspace, buildifierLintCommand) + if executionErr != nil { + return executionErr + } + + if len(verificationErrs) == 0 { + return nil + } + + noun := "file" + if len(verificationErrs) > 1 { + noun += "s" + } + + return fmt.Errorf("verification failed for %d %s:\n%s", + len(verificationErrs), + noun, + strings.Join( + mapSlice(verificationErrs, func(e error) string { return e.Error() }), + "\n", + )) +} + +type commandFunc = func(path string) (command string, args []string) + +func buildifierLintCommand(path string) (string, []string) { + return "buildifier", []string{"-lint", "warn", "-mode", "check", path} +} + +// verifyStarlark walks all directories starting at provided workspace path and +// verifies any Starlark files it finds. +// Starlark files are assumed to end with the .star extension. +// The verification relies on linting frovided by the 'buildifier' binary which +// must be in the PATH. +// A slice of verification errors are returned, one for each file that failed verification. +// If any execution of the `buildifier` command fails, this is returned separately. +// commandFn is executed on every Starlark file to determine the command and arguments to be executed. +// The caller is trusted and it is the callers responsibility to ensure that the resulting command is safe to execute. +func verifyStarlark(ctx context.Context, workspace string, commandFn commandFunc) ([]error, error) { + var verificationErrs []error + + // All errors from filepath.WalkDir are filtered by the fs.WalkDirFunc. + // Lstat or ReadDir errors are reported as verificationErrors. + // If any execution of the `buildifier` command fails or if the context is cancelled, + // it is reported as an error and any verification of subsequent files is skipped. + err := filepath.WalkDir(workspace, func(path string, d fs.DirEntry, err error) error { + // Skip verification of the file or files within the directory if there is an error + // returned by Lstat or ReadDir. + if err != nil { + verificationErrs = append(verificationErrs, err) + return nil + } + + if d.IsDir() { + return nil + } + + if filepath.Ext(path) == ".star" { + command, args := commandFn(path) + // The caller is trusted. + //nolint:gosec + cmd := exec.CommandContext(ctx, command, args...) + cmd.Dir = workspace + + _, err = cmd.Output() + if err == nil { // No error, early return. + return nil + } + + // The error returned from cmd.Output() is never wrapped. + //nolint:errorlint + if err, ok := err.(*exec.ExitError); ok { + switch err.ExitCode() { + // Case comments are informed by the output of `buildifier --help` + case 1: // syntax errors in input + verificationErrs = append(verificationErrs, errors.New(string(err.Stderr))) + return nil + case 2: // usage errors: invoked incorrectly + return fmt.Errorf("command %q: %s", cmd, err.Stderr) + case 3: // unexpected runtime errors: file I/O problems or internal bugs + return fmt.Errorf("command %q: %s", cmd, err.Stderr) + case 4: // check mode failed (reformat is needed) + verificationErrs = append(verificationErrs, errors.New(string(err.Stderr))) + return nil + default: + return fmt.Errorf("command %q: %s", cmd, err.Stderr) + } + } + + // Error was not an exit error from the command. + return fmt.Errorf("command %q: %v", cmd, err) + } + + return nil + }) + + return verificationErrs, err +} diff --git a/pkg/build/cmd/verifystarlark_test.go b/pkg/build/cmd/verifystarlark_test.go new file mode 100644 index 00000000000..1121ca5bd64 --- /dev/null +++ b/pkg/build/cmd/verifystarlark_test.go @@ -0,0 +1,137 @@ +//go:build requires_buildifier + +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestVerifyStarlark(t *testing.T) { + t.Run("execution errors", func(t *testing.T) { + t.Run("invalid usage", func(t *testing.T) { + ctx := context.Background() + workspace := t.TempDir() + err := os.WriteFile(filepath.Join(workspace, "ignored.star"), []byte{}, os.ModePerm) + if err != nil { + t.Fatalf(err.Error()) + } + + _, executionErr := verifyStarlark(ctx, workspace, func(string) (string, []string) { return "buildifier", []string{"--invalid"} }) + if executionErr == nil { + t.Fatalf("Expected execution error but got none") + } + }) + + t.Run("context cancellation", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + workspace := t.TempDir() + err := os.WriteFile(filepath.Join(workspace, "ignored.star"), []byte{}, os.ModePerm) + if err != nil { + t.Fatalf(err.Error()) + } + err = os.WriteFile(filepath.Join(workspace, "other-ignored.star"), []byte{}, os.ModePerm) + if err != nil { + t.Fatalf(err.Error()) + } + cancel() + + _, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand) + if executionErr == nil { + t.Fatalf("Expected execution error but got none") + } + }) + }) + + t.Run("verification errors", func(t *testing.T) { + t.Run("a single file with lint", func(t *testing.T) { + ctx := context.Background() + workspace := t.TempDir() + + invalidContent := []byte(`load("scripts/drone/other.star", "function") + +function()`) + err := os.WriteFile(filepath.Join(workspace, "has-lint.star"), invalidContent, os.ModePerm) + if err != nil { + t.Fatalf(err.Error()) + } + + verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand) + if executionErr != nil { + t.Fatalf("Unexpected execution error: %v", executionErr) + } + if len(verificationErrs) == 0 { + t.Fatalf(`"has-lint.star" requires linting but the verifyStarlark function provided no linting error`) + } + if len(verificationErrs) > 1 { + t.Fatalf(`verifyStarlark returned multiple errors for the "has-lint.star" file but only one was expected: %v`, verificationErrs) + } + if !strings.Contains(verificationErrs[0].Error(), "has-lint.star:1: module-docstring: The file has no module docstring.") { + t.Fatalf(`"has-lint.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0]) + } + }) + + t.Run("no files with lint", func(t *testing.T) { + ctx := context.Background() + workspace := t.TempDir() + + content := []byte(`""" +This module does nothing. +""" + +load("scripts/drone/other.star", "function") + +function() +`) + require.NoError(t, os.WriteFile(filepath.Join(workspace, "no-lint.star"), content, os.ModePerm)) + + verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand) + if executionErr != nil { + t.Fatalf("Unexpected execution error: %v", executionErr) + } + if len(verificationErrs) != 0 { + t.Log(`"no-lint.star" has no lint but the verifyStarlark function provided at least one error`) + for _, err := range verificationErrs { + t.Log(err) + } + t.FailNow() + } + }) + + t.Run("multiple files with lint", func(t *testing.T) { + ctx := context.Background() + workspace := t.TempDir() + + invalidContent := []byte(`load("scripts/drone/other.star", "function") + +function()`) + require.NoError(t, os.WriteFile(filepath.Join(workspace, "has-lint.star"), invalidContent, os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(workspace, "has-lint2.star"), invalidContent, os.ModePerm)) + + verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand) + if executionErr != nil { + t.Fatalf("Unexpected execution error: %v", executionErr) + } + if len(verificationErrs) == 0 { + t.Fatalf(`Two files require linting but the verifyStarlark function provided no linting error`) + } + if len(verificationErrs) == 1 { + t.Fatalf(`Two files require linting but the verifyStarlark function provided only one linting error: %v`, verificationErrs[0]) + } + if len(verificationErrs) > 2 { + t.Fatalf(`verifyStarlark returned more errors than expected: %v`, verificationErrs) + } + if !strings.Contains(verificationErrs[0].Error(), "has-lint.star:1: module-docstring: The file has no module docstring.") { + t.Errorf(`"has-lint.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0]) + } + if !strings.Contains(verificationErrs[1].Error(), "has-lint2.star:1: module-docstring: The file has no module docstring.") { + t.Fatalf(`"has-lint2.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0]) + } + }) + }) +} diff --git a/pkg/build/cmd/verifystorybook.go b/pkg/build/cmd/verifystorybook.go new file mode 100644 index 00000000000..c074bb76857 --- /dev/null +++ b/pkg/build/cmd/verifystorybook.go @@ -0,0 +1,33 @@ +// Package verifystorybook contains the sub-command "verify-storybook". +package main + +import ( + "fmt" + "log" + "path/filepath" + + "github.com/urfave/cli/v2" + + "github.com/grafana/grafana/pkg/infra/fs" +) + +// VerifyStorybook Action implements the sub-command "verify-storybook". +func VerifyStorybook(c *cli.Context) error { + const grafanaDir = "." + + paths := []string{ + "packages/grafana-ui/dist/storybook/index.html", + "packages/grafana-ui/dist/storybook/iframe.html"} + for _, p := range paths { + exists, err := fs.Exists(filepath.Join(grafanaDir, p)) + if err != nil { + return cli.Exit(fmt.Sprintf("failed to verify Storybook build: %s", err), 1) + } + if !exists { + return fmt.Errorf("failed to verify Storybook build, missing %q", p) + } + } + + log.Printf("Successfully verified Storybook integrity") + return nil +} diff --git a/pkg/build/config/config.go b/pkg/build/config/config.go index 7c8b4b133bf..7aa4176e13a 100644 --- a/pkg/build/config/config.go +++ b/pkg/build/config/config.go @@ -1,20 +1,18 @@ package config type Config struct { - Version string - Bucket string - DebRepoBucket string - DebDBBucket string - RPMRepoBucket string - GPGPassPath string - GPGPrivateKey string - GPGPublicKey string - GCPKeyFile string - NumWorkers int - GitHubUser string - GitHubToken string - PullEnterprise bool - NetworkConcurrency bool - PackageVersion string - SignPackages bool + Version string + Bucket string + DebRepoBucket string + DebDBBucket string + RPMRepoBucket string + GPGPassPath string + GPGPrivateKey string + GPGPublicKey string + NumWorkers int + GitHubUser string + GitHubToken string + PullEnterprise bool + PackageVersion string + SignPackages bool } diff --git a/pkg/build/config/genmetadata.go b/pkg/build/config/genmetadata.go new file mode 100644 index 00000000000..c201d77dd39 --- /dev/null +++ b/pkg/build/config/genmetadata.go @@ -0,0 +1,102 @@ +package config + +import ( + "fmt" + "os" + "strings" + + "github.com/grafana/grafana/pkg/build/droneutil" + "github.com/urfave/cli/v2" +) + +func GenerateMetadata(c *cli.Context) (Metadata, error) { + var metadata Metadata + version := "" + + event, err := droneutil.GetDroneEventFromEnv() + if err != nil { + return Metadata{}, err + } + + tag, ok := os.LookupEnv("DRONE_TAG") + if !ok { + fmt.Println("DRONE_TAG envvar not present, %w", err) + } + + var releaseMode ReleaseMode + switch event { + case string(PullRequestMode): + releaseMode = ReleaseMode{Mode: PullRequestMode} + case Push: + mode, err := CheckDroneTargetBranch() + if err != nil { + return Metadata{}, err + } + releaseMode = ReleaseMode{Mode: mode} + case Custom: + if edition, _ := os.LookupEnv("EDITION"); edition == string(EditionEnterprise2) { + releaseMode = ReleaseMode{Mode: Enterprise2Mode} + if tag != "" { + version = strings.TrimPrefix(tag, "v") + } + break + } + mode, err := CheckDroneTargetBranch() + if err != nil { + return Metadata{}, err + } + // if there is a custom event targeting the main branch, that's an enterprise downstream build + if mode == MainBranch { + releaseMode = ReleaseMode{Mode: DownstreamMode} + } else { + releaseMode = ReleaseMode{Mode: mode} + } + case Tag, Promote: + if tag == "" { + return Metadata{}, fmt.Errorf("DRONE_TAG envvar not present for a tag/promotion event, %w", err) + } + version = strings.TrimPrefix(tag, "v") + mode, err := CheckSemverSuffix() + if err != nil { + return Metadata{}, err + } + releaseMode = mode + case Cronjob: + releaseMode = ReleaseMode{Mode: CronjobMode} + } + + if version == "" { + version, err = generateVersionFromBuildID() + if err != nil { + return Metadata{}, err + } + } + + currentCommit, err := GetDroneCommit() + if err != nil { + return Metadata{}, err + } + metadata = Metadata{ + GrafanaVersion: version, + ReleaseMode: releaseMode, + GrabplVersion: c.App.Version, + CurrentCommit: currentCommit, + } + + fmt.Printf("building Grafana version: %s, release mode: %+v", metadata.GrafanaVersion, metadata.ReleaseMode) + + return metadata, nil +} + +func generateVersionFromBuildID() (string, error) { + buildID, ok := os.LookupEnv("DRONE_BUILD_NUMBER") + if !ok { + return "", fmt.Errorf("unable to get DRONE_BUILD_NUMBER environmental variable") + } + var err error + version, err := GetGrafanaVersion(buildID, ".") + if err != nil { + return "", err + } + return version, nil +} diff --git a/pkg/build/config/genmetadata_test.go b/pkg/build/config/genmetadata_test.go new file mode 100644 index 00000000000..605b68496c7 --- /dev/null +++ b/pkg/build/config/genmetadata_test.go @@ -0,0 +1,81 @@ +package config + +import ( + "flag" + "os" + "testing" + + "github.com/stretchr/testify/require" + "github.com/urfave/cli/v2" +) + +const ( + DroneBuildEvent = "DRONE_BUILD_EVENT" + DroneTargetBranch = "DRONE_TARGET_BRANCH" + DroneTag = "DRONE_TAG" + DroneSemverPrerelease = "DRONE_SEMVER_PRERELEASE" + DroneBuildNumber = "DRONE_BUILD_NUMBER" +) + +const ( + hashedGrafanaVersion = "9.2.0-12345pre" + versionedBranch = "v9.2.x" +) + +func TestGetMetadata(t *testing.T) { + tcs := []struct { + envMap map[string]string + expVersion string + mode ReleaseMode + }{ + {map[string]string{DroneBuildEvent: PullRequest, DroneTargetBranch: "", DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: PullRequestMode}}, + {map[string]string{DroneBuildEvent: Push, DroneTargetBranch: versionedBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: ReleaseBranchMode}}, + {map[string]string{DroneBuildEvent: Push, DroneTargetBranch: MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: MainMode}}, + {map[string]string{DroneBuildEvent: Custom, DroneTargetBranch: versionedBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: ReleaseBranchMode}}, + {map[string]string{DroneBuildEvent: Custom, DroneTargetBranch: MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: DownstreamMode}}, + {map[string]string{DroneBuildEvent: Custom, DroneTargetBranch: MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345", "EDITION": string(EditionEnterprise2)}, hashedGrafanaVersion, ReleaseMode{Mode: Enterprise2Mode}}, + {map[string]string{DroneBuildEvent: Tag, DroneTargetBranch: "", DroneTag: "v9.2.0", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, "9.2.0", ReleaseMode{Mode: TagMode, IsBeta: false, IsTest: false}}, + {map[string]string{DroneBuildEvent: Tag, DroneTargetBranch: "", DroneTag: "v9.2.0-beta", DroneSemverPrerelease: "beta", DroneBuildNumber: "12345"}, "9.2.0-beta", ReleaseMode{Mode: TagMode, IsBeta: true, IsTest: false}}, + {map[string]string{DroneBuildEvent: Tag, DroneTargetBranch: "", DroneTag: "v9.2.0-test", DroneSemverPrerelease: "test", DroneBuildNumber: "12345"}, "9.2.0-test", ReleaseMode{Mode: TagMode, IsBeta: false, IsTest: true}}, + {map[string]string{DroneBuildEvent: Promote, DroneTargetBranch: "", DroneTag: "v9.2.0", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, "9.2.0", ReleaseMode{Mode: TagMode, IsBeta: false, IsTest: false}}, + {map[string]string{DroneBuildEvent: Promote, DroneTargetBranch: "", DroneTag: "v9.2.0-beta", DroneSemverPrerelease: "beta", DroneBuildNumber: "12345"}, "9.2.0-beta", ReleaseMode{Mode: TagMode, IsBeta: true, IsTest: false}}, + {map[string]string{DroneBuildEvent: Promote, DroneTargetBranch: "", DroneTag: "v9.2.0-test", DroneSemverPrerelease: "test", DroneBuildNumber: "12345"}, "9.2.0-test", ReleaseMode{Mode: TagMode, IsBeta: false, IsTest: true}}, + } + + ctx := cli.NewContext(cli.NewApp(), &flag.FlagSet{}, nil) + for _, tc := range tcs { + t.Run("Should return valid metadata, ", func(t *testing.T) { + setUpEnv(t, tc.envMap) + testMetadata(t, ctx, tc.expVersion, tc.mode) + }) + } +} + +func testMetadata(t *testing.T, ctx *cli.Context, version string, releaseMode ReleaseMode) { + t.Helper() + + metadata, err := GenerateMetadata(ctx) + require.NoError(t, err) + t.Run("with a valid version", func(t *testing.T) { + expVersion := metadata.GrafanaVersion + require.Equal(t, expVersion, version) + }) + + t.Run("with a valid release mode from the built-in list", func(t *testing.T) { + expMode := metadata.ReleaseMode + require.NoError(t, err) + require.Equal(t, expMode, releaseMode) + }) +} + +func setUpEnv(t *testing.T, envMap map[string]string) { + t.Helper() + + os.Clearenv() + err := os.Setenv("DRONE_COMMIT", "abcd12345") + require.NoError(t, err) + for k, v := range envMap { + err := os.Setenv(k, v) + require.NoError(t, err) + } +} diff --git a/pkg/build/config/package.json b/pkg/build/config/package.json new file mode 100644 index 00000000000..1a5184b05c7 --- /dev/null +++ b/pkg/build/config/package.json @@ -0,0 +1,3 @@ +{ + "version": "9.2.0-pre" +} diff --git a/pkg/build/config/version.go b/pkg/build/config/version.go index fa8c8b3555b..aaeeaf56652 100644 --- a/pkg/build/config/version.go +++ b/pkg/build/config/version.go @@ -7,6 +7,8 @@ import ( "path/filepath" "regexp" "strings" + + "github.com/grafana/grafana/pkg/build/git" ) type Metadata struct { @@ -100,8 +102,8 @@ func GetGrafanaVersion(buildID, grafanaDir string) (string, error) { } func CheckDroneTargetBranch() (VersionMode, error) { + rePRCheckBranch := git.PRCheckRegexp() reRlsBranch := regexp.MustCompile(`^v\d+\.\d+\.x$`) - rePRCheckBranch := regexp.MustCompile(`^pr-check-\d+`) target := os.Getenv("DRONE_TARGET_BRANCH") if target == "" { return "", fmt.Errorf("failed to get DRONE_TARGET_BRANCH environmental variable") diff --git a/pkg/build/config/version_mode.go b/pkg/build/config/version_mode.go index f7aa6578fe4..a74b5b5f49e 100644 --- a/pkg/build/config/version_mode.go +++ b/pkg/build/config/version_mode.go @@ -8,7 +8,8 @@ const ( TagMode VersionMode = "release" ReleaseBranchMode VersionMode = "branch" PullRequestMode VersionMode = "pull_request" - CustomMode VersionMode = "custom" + DownstreamMode VersionMode = "downstream" + Enterprise2Mode VersionMode = "enterprise2" CronjobMode VersionMode = "cron" ) diff --git a/pkg/build/config/versions.go b/pkg/build/config/versions.go index baf8e6b8f46..874cedef5f5 100644 --- a/pkg/build/config/versions.go +++ b/pkg/build/config/versions.go @@ -59,7 +59,7 @@ var Versions = VersionMap{ Storybook: "grafana-storybook", }, }, - CustomMode: { + DownstreamMode: { Variants: []Variant{ VariantArmV6, VariantArmV7, @@ -165,4 +165,42 @@ var Versions = VersionMap{ StorybookSrcDir: "artifacts/storybook", }, }, + Enterprise2Mode: { + Variants: []Variant{ + VariantArmV6, + VariantArmV7, + VariantArmV7Musl, + VariantArm64, + VariantArm64Musl, + VariantDarwinAmd64, + VariantWindowsAmd64, + VariantLinuxAmd64, + VariantLinuxAmd64Musl, + }, + PluginSignature: PluginSignature{ + Sign: true, + AdminSign: true, + }, + Docker: Docker{ + ShouldSave: true, + Architectures: []Architecture{ + ArchAMD64, + ArchARM64, + ArchARMv7, + }, + Distribution: []Distribution{ + Alpine, + Ubuntu, + }, + PrereleaseBucket: "grafana-prerelease/artifacts/docker", + }, + Buckets: Buckets{ + Artifacts: "grafana-prerelease/artifacts/downloads", + ArtifactsEnterprise2: "grafana-prerelease/artifacts/downloads-enterprise2", + CDNAssets: "grafana-prerelease", + CDNAssetsDir: "artifacts/static-assets", + Storybook: "grafana-prerelease", + StorybookSrcDir: "artifacts/storybook", + }, + }, } diff --git a/pkg/build/docker/build.go b/pkg/build/docker/build.go index f77d54cd43e..7267d4a094b 100644 --- a/pkg/build/docker/build.go +++ b/pkg/build/docker/build.go @@ -85,6 +85,11 @@ func BuildImage(version string, arch config.Architecture, grafanaDir string, use var additionalDockerRepo string var tags []string var imageFileBase string + var dockerEnterprise2Repo string + if repo, ok := os.LookupEnv("DOCKER_ENTERPRISE2_REPO"); ok { + dockerEnterprise2Repo = repo + } + switch edition { case config.EditionOSS: dockerRepo = "grafana/grafana-image-tags" @@ -94,6 +99,10 @@ func BuildImage(version string, arch config.Architecture, grafanaDir string, use dockerRepo = "grafana/grafana-enterprise-image-tags" imageFileBase = "grafana-enterprise" editionStr = "-enterprise" + case config.EditionEnterprise2: + dockerRepo = dockerEnterprise2Repo + imageFileBase = "grafana-enterprise2" + editionStr = "-enterprise2" default: return []string{}, fmt.Errorf("unrecognized edition %s", edition) } diff --git a/pkg/build/docker/push.go b/pkg/build/docker/push.go new file mode 100644 index 00000000000..da87ef2cb16 --- /dev/null +++ b/pkg/build/docker/push.go @@ -0,0 +1,62 @@ +package docker + +import ( + "fmt" + "log" + "os" + "os/exec" + "time" +) + +const ( + tries = 3 + sleepTime = 30 +) + +func PushImage(newImage string) error { + var err error + for i := 0; i < tries; i++ { + log.Printf("push attempt #%d...", i+1) + var out []byte + cmd := exec.Command("docker", "push", newImage) + cmd.Dir = "." + out, err = cmd.CombinedOutput() + if err != nil { + log.Printf("output: %s", out) + log.Printf("sleep for %d, before retrying...", sleepTime) + time.Sleep(sleepTime * time.Second) + } else { + log.Printf("Successfully pushed %s!", newImage) + break + } + } + if err != nil { + return fmt.Errorf("error pushing images to DockerHub: %q", err) + } + return nil +} + +func PushManifest(manifest string) error { + log.Printf("Pushing Docker manifest %s...", manifest) + + var err error + for i := 0; i < tries; i++ { + log.Printf("push attempt #%d...", i+1) + var out []byte + cmd := exec.Command("docker", "manifest", "push", manifest) + cmd.Env = append(os.Environ(), "DOCKER_CLI_EXPERIMENTAL=enabled") + out, err = cmd.CombinedOutput() + if err != nil { + log.Printf("output: %s", out) + log.Printf("sleep for %d, before retrying...", sleepTime) + time.Sleep(sleepTime * time.Second) + } else { + log.Printf("Successful manifest push! %s", string(out)) + break + } + } + if err != nil { + return fmt.Errorf("failed to push manifest, err: %w", err) + } + return nil +} diff --git a/pkg/build/env/lookup.go b/pkg/build/env/lookup.go new file mode 100644 index 00000000000..993b7259e14 --- /dev/null +++ b/pkg/build/env/lookup.go @@ -0,0 +1,18 @@ +package env + +import ( + "strings" +) + +// Lookup is the equivalent of os.LookupEnv, only you are able to provide the list of environment variables. +// To use this as os.LookupEnv would be used, simply call +// `env.Lookup("ENVIRONMENT_VARIABLE", os.Environ())` +func Lookup(name string, vars []string) (string, bool) { + for _, v := range vars { + if strings.HasPrefix(v, name) { + return strings.TrimPrefix(v, name+"="), true + } + } + + return "", false +} diff --git a/pkg/build/env/lookup_test.go b/pkg/build/env/lookup_test.go new file mode 100644 index 00000000000..cebfb4fac29 --- /dev/null +++ b/pkg/build/env/lookup_test.go @@ -0,0 +1,43 @@ +package env_test + +import ( + "testing" + + "github.com/grafana/grafana/pkg/build/env" + + "github.com/stretchr/testify/require" +) + +func TestLookup(t *testing.T) { + values := []string{"ENV_1=a", "ENV_2=b", "ENV_3=c", "ENV_4_TEST="} + + { + v, ok := env.Lookup("ENV_1", values) + require.Equal(t, v, "a") + require.True(t, ok) + } + + { + v, ok := env.Lookup("ENV_2", values) + require.Equal(t, v, "b") + require.True(t, ok) + } + + { + v, ok := env.Lookup("ENV_3", values) + require.Equal(t, v, "c") + require.True(t, ok) + } + + { + v, ok := env.Lookup("ENV_4_TEST", values) + require.Equal(t, v, "") + require.True(t, ok) + } + + { + v, ok := env.Lookup("NOT_THERE", values) + require.Equal(t, v, "") + require.False(t, ok) + } +} diff --git a/pkg/build/git/git.go b/pkg/build/git/git.go new file mode 100644 index 00000000000..7539a815de6 --- /dev/null +++ b/pkg/build/git/git.go @@ -0,0 +1,142 @@ +package git + +import ( + "context" + "errors" + "fmt" + "net/http" + "regexp" + + "github.com/google/go-github/v45/github" + "github.com/grafana/grafana/pkg/build/stringutil" + "golang.org/x/oauth2" +) + +const ( + MainBranch = "main" + HomeDir = "." + RepoOwner = "grafana" + OSSRepo = "grafana" + EnterpriseRepo = "grafana-enterprise" + EnterpriseCheckName = "Grafana Enterprise" + EnterpriseCheckDescription = "Downstream tests to ensure that your changes are compatible with Grafana Enterprise" +) + +var EnterpriseCheckLabels = []string{"enterprise-ok", "enterprise-failed", "enterprise-override"} + +var ( + ErrorNoDroneBuildLink = errors.New("no drone build link") +) + +type GitService interface { + DeleteRef(ctx context.Context, owner string, repo string, ref string) (*github.Response, error) +} + +type LabelsService interface { + ListLabelsByIssue(ctx context.Context, owner string, repo string, number int, opts *github.ListOptions) ([]*github.Label, *github.Response, error) + RemoveLabelForIssue(ctx context.Context, owner string, repo string, number int, label string) (*github.Response, error) + AddLabelsToIssue(ctx context.Context, owner string, repo string, number int, labels []string) ([]*github.Label, *github.Response, error) +} + +type CommentService interface { + CreateComment(ctx context.Context, owner string, repo string, number int, comment *github.IssueComment) (*github.IssueComment, *github.Response, error) +} + +type StatusesService interface { + CreateStatus(ctx context.Context, owner, repo, ref string, status *github.RepoStatus) (*github.RepoStatus, *github.Response, error) +} + +// NewGitHubClient creates a new Client using the provided GitHub token if not empty. +func NewGitHubClient(ctx context.Context, token string) *github.Client { + var tc *http.Client + if token != "" { + ts := oauth2.StaticTokenSource(&oauth2.Token{ + AccessToken: token, + }) + tc = oauth2.NewClient(ctx, ts) + } + + return github.NewClient(tc) +} + +func PRCheckRegexp() *regexp.Regexp { + reBranch, err := regexp.Compile(`^prc-([0-9]+)-([A-Za-z0-9]+)\/(.+)$`) + if err != nil { + panic(fmt.Sprintf("Failed to compile regexp: %s", err)) + } + + return reBranch +} + +func AddLabelToPR(ctx context.Context, client LabelsService, prID int, newLabel string) error { + // Check existing labels + labels, _, err := client.ListLabelsByIssue(ctx, RepoOwner, OSSRepo, prID, nil) + if err != nil { + return err + } + + duplicate := false + for _, label := range labels { + if *label.Name == newLabel { + duplicate = true + continue + } + + // Delete existing "enterprise-xx" labels + if stringutil.Contains(EnterpriseCheckLabels, *label.Name) { + _, err := client.RemoveLabelForIssue(ctx, RepoOwner, OSSRepo, prID, *label.Name) + if err != nil { + return err + } + } + } + + if duplicate { + return nil + } + + _, _, err = client.AddLabelsToIssue(ctx, RepoOwner, OSSRepo, prID, []string{newLabel}) + if err != nil { + return err + } + + return nil +} + +func DeleteEnterpriseBranch(ctx context.Context, client GitService, branchName string) error { + ref := "heads/" + branchName + if _, err := client.DeleteRef(ctx, RepoOwner, EnterpriseRepo, ref); err != nil { + return err + } + + return nil +} + +// CreateEnterpriseStatus sets the status on a commit for the enterprise build check. +func CreateEnterpriseStatus(ctx context.Context, client StatusesService, sha, link, status string) (*github.RepoStatus, error) { + check, _, err := client.CreateStatus(ctx, RepoOwner, OSSRepo, sha, &github.RepoStatus{ + Context: github.String(EnterpriseCheckName), + Description: github.String(EnterpriseCheckDescription), + TargetURL: github.String(link), + State: github.String(status), + }) + + if err != nil { + return nil, err + } + + return check, nil +} + +func CreateEnterpriseBuildFailedComment(ctx context.Context, client CommentService, link string, prID int) error { + body := fmt.Sprintf("Drone build failed: %s", link) + + _, _, err := client.CreateComment(ctx, RepoOwner, OSSRepo, prID, &github.IssueComment{ + Body: &body, + }) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/build/git/git_checks_test.go b/pkg/build/git/git_checks_test.go new file mode 100644 index 00000000000..ed3d34a8d28 --- /dev/null +++ b/pkg/build/git/git_checks_test.go @@ -0,0 +1,55 @@ +package git_test + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-github/v45/github" + "github.com/grafana/grafana/pkg/build/git" + "github.com/stretchr/testify/require" +) + +type TestChecksService struct { + CreateCheckRunError error +} + +func (s *TestChecksService) CreateStatus(ctx context.Context, owner, repo, ref string, status *github.RepoStatus) (*github.RepoStatus, *github.Response, error) { + if s.CreateCheckRunError != nil { + return nil, nil, s.CreateCheckRunError + } + + return &github.RepoStatus{ + ID: github.Int64(1), + URL: status.URL, + }, nil, nil +} + +func TestCreateEnterpriseRepoStatus(t *testing.T) { + t.Run("It should create a repo status", func(t *testing.T) { + var ( + ctx = context.Background() + client = &TestChecksService{} + link = "http://example.com" + sha = "1234" + ) + + _, err := git.CreateEnterpriseStatus(ctx, client, link, sha, "success") + + require.NoError(t, err) + }) + t.Run("It should return an error if GitHub fails to create the status", func(t *testing.T) { + var ( + ctx = context.Background() + createCheckError = errors.New("create check run error") + client = &TestChecksService{ + CreateCheckRunError: createCheckError, + } + link = "http://example.com" + sha = "1234" + ) + + _, err := git.CreateEnterpriseStatus(ctx, client, link, sha, "success") + require.ErrorIs(t, err, createCheckError) + }) +} diff --git a/pkg/build/git/git_issues_test.go b/pkg/build/git/git_issues_test.go new file mode 100644 index 00000000000..4eaa3bb6169 --- /dev/null +++ b/pkg/build/git/git_issues_test.go @@ -0,0 +1,134 @@ +package git_test + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-github/v45/github" + "github.com/grafana/grafana/pkg/build/git" + "github.com/stretchr/testify/require" +) + +type TestLabelsService struct { + Labels []*github.Label + ListLabelsError error + RemoveLabelError error + AddLabelsError error +} + +func (s *TestLabelsService) ListLabelsByIssue(ctx context.Context, owner string, repo string, number int, opts *github.ListOptions) ([]*github.Label, *github.Response, error) { + if s.ListLabelsError != nil { + return nil, nil, s.ListLabelsError + } + + labels := s.Labels + if labels == nil { + labels = []*github.Label{} + } + + return labels, nil, nil +} + +func (s *TestLabelsService) RemoveLabelForIssue(ctx context.Context, owner string, repo string, number int, label string) (*github.Response, error) { + if s.RemoveLabelError != nil { + return nil, s.RemoveLabelError + } + + return &github.Response{}, nil +} + +func (s *TestLabelsService) AddLabelsToIssue(ctx context.Context, owner string, repo string, number int, labels []string) ([]*github.Label, *github.Response, error) { + if s.AddLabelsError != nil { + return nil, nil, s.AddLabelsError + } + + l := make([]*github.Label, len(labels)) + for i, v := range labels { + l[i] = &github.Label{ + Name: github.String(v), + } + } + + return l, nil, nil +} + +func TestAddLabelToPR(t *testing.T) { + t.Run("It should add a label to a pull request", func(t *testing.T) { + var ( + ctx = context.Background() + client = &TestLabelsService{} + pr = 20 + label = "test-label" + ) + + require.NoError(t, git.AddLabelToPR(ctx, client, pr, label)) + }) + t.Run("It should not return an error if the label already exists", func(t *testing.T) { + var ( + ctx = context.Background() + client = &TestLabelsService{ + Labels: []*github.Label{ + { + Name: github.String("test-label"), + }, + }, + } + pr = 20 + label = "test-label" + ) + + require.NoError(t, git.AddLabelToPR(ctx, client, pr, label)) + }) + + t.Run("It should return an error if GitHub returns an error when listing labels", func(t *testing.T) { + var ( + ctx = context.Background() + listLabelsError = errors.New("list labels error") + client = &TestLabelsService{ + ListLabelsError: listLabelsError, + Labels: []*github.Label{}, + } + pr = 20 + label = "test-label" + ) + + require.ErrorIs(t, git.AddLabelToPR(ctx, client, pr, label), listLabelsError) + }) + + t.Run("It should not return an error if there are existing enterprise-check labels.", func(t *testing.T) { + var ( + ctx = context.Background() + client = &TestLabelsService{ + Labels: []*github.Label{ + { + Name: github.String("enterprise-failed"), + }, + }, + } + pr = 20 + label = "test-label" + ) + + require.NoError(t, git.AddLabelToPR(ctx, client, pr, label)) + }) + + t.Run("It should return an error if GitHub returns an error when removing existing enterprise-check labels", func(t *testing.T) { + var ( + ctx = context.Background() + removeLabelError = errors.New("remove label error") + client = &TestLabelsService{ + RemoveLabelError: removeLabelError, + Labels: []*github.Label{ + { + Name: github.String("enterprise-failed"), + }, + }, + } + pr = 20 + label = "test-label" + ) + + require.ErrorIs(t, git.AddLabelToPR(ctx, client, pr, label), removeLabelError) + }) +} diff --git a/pkg/build/git/git_test.go b/pkg/build/git/git_test.go new file mode 100644 index 00000000000..c0cbd4c8cd8 --- /dev/null +++ b/pkg/build/git/git_test.go @@ -0,0 +1,56 @@ +package git_test + +import ( + "testing" + + "github.com/grafana/grafana/pkg/build/git" + "github.com/stretchr/testify/assert" +) + +func TestPRCheckRegexp(t *testing.T) { + type match struct { + String string + Commit string + Branch string + PR string + } + + var ( + shouldMatch = []match{ + { + String: "prc-1-a1b2c3d4/branch-name", + Branch: "branch-name", + Commit: "a1b2c3d4", + PR: "1", + }, + { + String: "prc-111-a1b2c3d4/branch/name", + Branch: "branch/name", + Commit: "a1b2c3d4", + PR: "111", + }, + { + String: "prc-102930122-a1b2c3d4/branch-name", + Branch: "branch-name", + Commit: "a1b2c3d4", + PR: "102930122", + }, + } + + shouldNotMatch = []string{"prc-a/branch", "km/test", "test", "prc", "prc/test", "price"} + ) + + regex := git.PRCheckRegexp() + + for _, v := range shouldMatch { + assert.Truef(t, regex.MatchString(v.String), "regex '%s' should match %s", regex.String(), v) + m := regex.FindStringSubmatch(v.String) + assert.Equal(t, m[1], v.PR) + assert.Equal(t, m[2], v.Commit) + assert.Equal(t, m[3], v.Branch) + } + + for _, v := range shouldNotMatch { + assert.False(t, regex.MatchString(v), "regex '%s' should not match %s", regex.String(), v) + } +} diff --git a/pkg/build/grafana/build.go b/pkg/build/grafana/build.go index 4d661e8d851..0596d094875 100644 --- a/pkg/build/grafana/build.go +++ b/pkg/build/grafana/build.go @@ -99,7 +99,7 @@ func BuildGrafanaBinary(ctx context.Context, name, version string, args BuildArg descriptor := GrafanaDescriptor(opts) - log.Printf("Building %q for %s\nwith env: %v", binary, descriptor, opts.Env()) + log.Printf("Building %q for %s", binary, descriptor) opts.LdFlags = append(args.LdFlags, GrafanaLDFlags(version, revision)...) diff --git a/pkg/build/lerna/lerna.go b/pkg/build/lerna/lerna.go index de0cbbfe461..de04f5f0e88 100644 --- a/pkg/build/lerna/lerna.go +++ b/pkg/build/lerna/lerna.go @@ -1,6 +1,7 @@ package lerna import ( + "context" "encoding/json" "fmt" "os" @@ -9,6 +10,7 @@ import ( "strings" "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/build/fsutil" ) // BuildFrontendPackages will bump the version for the package to the latest canary build @@ -56,3 +58,29 @@ func GetLernaVersion(grafanaDir string) (string, error) { } return strings.TrimSpace(version), nil } + +func PackFrontendPackages(ctx context.Context, tag, grafanaDir, artifactsDir string) error { + exists, err := fsutil.Exists(artifactsDir) + if err != nil { + return err + } + if exists { + err = os.RemoveAll(artifactsDir) + if err != nil { + return err + } + } + // nolint:gosec + if err = os.MkdirAll(artifactsDir, 0755); err != nil { + return err + } + + // nolint:gosec + cmd := exec.CommandContext(ctx, "yarn", "lerna", "exec", "--no-private", "--", "yarn", "pack", "--out", fmt.Sprintf("../../npm-artifacts/%%s-%v.tgz", tag)) + cmd.Dir = grafanaDir + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), output, err) + } + + return nil +} diff --git a/pkg/build/npm/npm.go b/pkg/build/npm/npm.go new file mode 100644 index 00000000000..c2b93aef46b --- /dev/null +++ b/pkg/build/npm/npm.go @@ -0,0 +1,240 @@ +package npm + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/grafana/grafana/pkg/build/gcloud/storage" + "github.com/grafana/grafana/pkg/build/lerna" + "github.com/grafana/grafana/pkg/build/versions" +) + +const GrafanaDir = "." +const NpmArtifactDir = "./npm-artifacts" + +// TODO: could this be replaced by `yarn lerna list -p` ? +var packages = []string{ + "@grafana/ui", + "@grafana/data", + "@grafana/toolkit", + "@grafana/runtime", + "@grafana/e2e", + "@grafana/e2e-selectors", + "@grafana/schema", +} + +// PublishNpmPackages will publish local NPM packages to NPM registry. +func PublishNpmPackages(ctx context.Context, tag string) error { + version, err := versions.GetVersion(tag) + if err != nil { + return err + } + + log.Printf("Grafana version: %s", version.Version) + + if err := setNpmCredentials(); err != nil { + return err + } + + npmArtifacts, err := storage.ListLocalFiles(NpmArtifactDir) + if err != nil { + return err + } + for _, packedFile := range npmArtifacts { + // nolint:gosec + cmd := exec.CommandContext(ctx, "npm", "publish", packedFile.FullPath, "--tag", version.Channel) + cmd.Dir = GrafanaDir + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), out, err) + } + } + + return updateTag(ctx, version, tag) +} + +// StoreNpmPackages will store local NPM packages in GCS bucket `bucketName`. +func StoreNpmPackages(ctx context.Context, tag, bucketName string) error { + err := lerna.PackFrontendPackages(ctx, tag, GrafanaDir, NpmArtifactDir) + if err != nil { + return err + } + + gcs, err := storage.New() + if err != nil { + return err + } + + bucket := gcs.Bucket(bucketName) + bucketPath := fmt.Sprintf("artifacts/npm/%s/", tag) + if err = gcs.CopyLocalDir(ctx, NpmArtifactDir, bucket, bucketPath, true); err != nil { + return err + } + + log.Print("Successfully stored npm packages!") + return nil +} + +// FetchNpmPackages will store NPM packages stored in GCS bucket `bucketName` on local disk in `frontend.NpmArtifactDir`. +func FetchNpmPackages(ctx context.Context, tag, bucketName string) error { + gcs, err := storage.New() + if err != nil { + return err + } + + bucketPath := fmt.Sprintf("artifacts/npm/%s/", tag) + bucket := gcs.Bucket(bucketName) + err = gcs.DownloadDirectory(ctx, bucket, NpmArtifactDir, storage.FilesFilter{ + Prefix: bucketPath, + FileExts: []string{".tgz"}, + }) + if err != nil { + return err + } + return nil +} + +// updateTag will move next or latest npm dist-tags, if needed. +// +// Note: This function makes the assumption that npm dist-tags has already +// been updated and hence why move of dist-tags not always happens: +// +// If stable the dist-tag latest was used. +// If beta the dist-tag next was used. +// +// Scenarios: +// +// 1. Releasing a newer stable than the current stable +// Latest and next is 9.1.5. +// 9.1.6 is released, latest and next should point to 9.1.6. +// The next dist-tag is moved to point to 9.1.6. +// +// 2. Releasing during an active beta period: +// Latest and next is 9.1.6. +// 9.2.0-beta1 is released, the latest should stay on 9.1.6, next should point to 9.2.0-beta1 +// No move of dist-tags +// 9.1.7 is relased, the latest should point to 9.1.7, next should stay to 9.2.0-beta1 +// No move of dist-tags +// Next week 9.2.0-beta2 is released, the latest should point to 9.1.7, next should point to 9.2.0-beta2 +// No move of dist-tags +// In two weeks 9.2.0 stable is relased, the latest and next should point to 9.2.0. +// The next dist-tag is moved to point to 9.2.0. +// +// 3. Releasing an older stable than the current stable +// Latest and next is 9.2.0. +// Next 9.1.8 is released, latest should point to 9.2.0, next should point to 9.2.0 +// The latest dist-tag is moved to point to 9.2.0. +func updateTag(ctx context.Context, version *versions.Version, releaseVersion string) error { + if version.Channel != versions.Latest { + return nil + } + + latestStableVersion, err := getLatestStableVersion() + if err != nil { + return err + } + + betaVersion, err := getLatestBetaVersion() + if err != nil { + return err + } + + isLatest, err := versions.IsGreaterThanOrEqual(releaseVersion, latestStableVersion) + if err != nil { + return err + } + + isNewerThanLatestBeta, err := versions.IsGreaterThanOrEqual(releaseVersion, betaVersion) + if err != nil { + return err + } + + for _, pkg := range packages { + if !isLatest { + err = runMoveLatestNPMTagCommand(ctx, pkg, latestStableVersion) + if err != nil { + return err + } + } + + if isLatest && isNewerThanLatestBeta { + err = runMoveNextNPMTagCommand(ctx, pkg, version.Version) + if err != nil { + return err + } + } + } + + return nil +} + +func getLatestStableVersion() (string, error) { + return versions.GetLatestVersion(versions.LatestStableVersionURL) +} + +func getLatestBetaVersion() (string, error) { + return versions.GetLatestVersion(versions.LatestBetaVersionURL) +} + +func runMoveNextNPMTagCommand(ctx context.Context, pkg string, packageVersion string) error { + // nolint:gosec + cmd := exec.CommandContext(ctx, "npm", "dist-tag", "add", fmt.Sprintf("%s@%s", pkg, packageVersion), "next") + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), out, err) + } + + return nil +} + +func runMoveLatestNPMTagCommand(ctx context.Context, pkg string, latestStableVersion string) error { + // nolint:gosec + cmd := exec.CommandContext(ctx, "npm", "dist-tag", "add", fmt.Sprintf("%s@%s", pkg, latestStableVersion), "latest") + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), out, err) + } + + return nil +} + +// setNpmCredentials Creates a .npmrc file in the users home folder and writes the +// necessary credentials to it for publishing packages to the NPM registry. +func setNpmCredentials() error { + npmToken := strings.TrimSpace(os.Getenv("NPM_TOKEN")) + if npmToken == "" { + return fmt.Errorf("npm token is not set") + } + + homeDir, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("failed to obtain home directory, err: %q", err) + } + + npmPath := filepath.Join(homeDir, ".npmrc") + registry := []byte(fmt.Sprintf("//registry.npmjs.org/:_authToken=%s", npmToken)) + if _, err = os.Stat(npmPath); os.IsNotExist(err) { + // nolint:gosec + f, err := os.Create(npmPath) + if err != nil { + return fmt.Errorf("couldn't create npmrc file, err: %q", err) + } + _, err = f.Write(registry) + if err != nil { + return fmt.Errorf("failed to write to file, err: %q", err) + } + defer func() { + if err := f.Close(); err != nil { + log.Printf("Failed to close file: %s", err.Error()) + } + }() + } else { + err = os.WriteFile(npmPath, registry, 0644) + if err != nil { + return fmt.Errorf("error writing to file, err: %q", err) + } + } + return nil +} diff --git a/pkg/build/packaging/deb.go b/pkg/build/packaging/deb.go index a5c01bd2ea6..094207b0270 100644 --- a/pkg/build/packaging/deb.go +++ b/pkg/build/packaging/deb.go @@ -8,10 +8,11 @@ import ( "path/filepath" "strings" + "github.com/urfave/cli/v2" + "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/fsutil" "github.com/grafana/grafana/pkg/infra/fs" - "github.com/urfave/cli/v2" ) func writeAptlyConf(dbDir, repoDir string) error { @@ -166,7 +167,7 @@ func UpdateDebRepo(cfg PublishConfig, workDir string) error { cmd := exec.Command("aptly", "publish", "update", "-batch", passArg, "-force-overwrite", tp, "filesystem:repo:grafana") if output, err := cmd.CombinedOutput(); err != nil { - return cli.NewExitError(fmt.Sprintf("failed to update Debian %q repository: %s", tp, output), 1) + return cli.Exit(fmt.Sprintf("failed to update Debian %q repository: %s", tp, output), 1) } } @@ -179,7 +180,7 @@ func UpdateDebRepo(cfg PublishConfig, workDir string) error { //nolint:gosec cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", dbDir, u) if output, err := cmd.CombinedOutput(); err != nil { - return cli.NewExitError(fmt.Sprintf("failed to upload Debian repo database to GCS: %s", output), 1) + return cli.Exit(fmt.Sprintf("failed to upload Debian repo database to GCS: %s", output), 1) } } @@ -193,14 +194,14 @@ func UpdateDebRepo(cfg PublishConfig, workDir string) error { //nolint:gosec cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", grafDir, u) if output, err := cmd.CombinedOutput(); err != nil { - return cli.NewExitError(fmt.Sprintf("failed to upload Debian repo resources to GCS: %s", output), 1) + return cli.Exit(fmt.Sprintf("failed to upload Debian repo resources to GCS: %s", output), 1) } allRepoResources := fmt.Sprintf("%s/**/*", u) log.Printf("Setting cache ttl for Debian repo resources on GCS (%s)...\n", allRepoResources) //nolint:gosec cmd = exec.Command("gsutil", "-m", "setmeta", "-h", CacheSettings+cfg.TTL, allRepoResources) if output, err := cmd.CombinedOutput(); err != nil { - return cli.NewExitError(fmt.Sprintf("failed to set cache ttl for Debian repo resources on GCS: %s", output), 1) + return cli.Exit(fmt.Sprintf("failed to set cache ttl for Debian repo resources on GCS: %s", output), 1) } } @@ -238,7 +239,7 @@ func addPkgsToRepo(cfg PublishConfig, workDir, tmpDir, repoName string) error { //nolint:gosec cmd := exec.Command("aptly", "repo", "add", "-force-replace", repoName, tmpDir) if output, err := cmd.CombinedOutput(); err != nil { - return cli.NewExitError(fmt.Sprintf("failed to add packages to local Debian repository: %s", output), 1) + return cli.Exit(fmt.Sprintf("failed to add packages to local Debian repository: %s", output), 1) } return nil diff --git a/pkg/build/packaging/grafana_test.go b/pkg/build/packaging/grafana_test.go new file mode 100644 index 00000000000..8f143893085 --- /dev/null +++ b/pkg/build/packaging/grafana_test.go @@ -0,0 +1,22 @@ +package packaging_test + +import ( + "testing" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/build/packaging" + "github.com/stretchr/testify/assert" +) + +func TestPackageRegexp(t *testing.T) { + t.Run("It should match enterprise2 packages", func(t *testing.T) { + rgx := packaging.PackageRegexp(config.EditionEnterprise2) + matches := []string{ + "grafana-enterprise2-1.2.3-4567pre.linux-amd64.tar.gz", + "grafana-enterprise2-1.2.3-4567pre.linux-amd64.tar.gz.sha256", + } + for _, v := range matches { + assert.Truef(t, rgx.MatchString(v), "'%s' should match regex '%s'", v, rgx.String()) + } + }) +} diff --git a/pkg/build/packaging/rpm.go b/pkg/build/packaging/rpm.go index 24da71a185a..a4e8e557221 100644 --- a/pkg/build/packaging/rpm.go +++ b/pkg/build/packaging/rpm.go @@ -10,12 +10,15 @@ import ( "path/filepath" "strings" + // Consider switching this over to a community fork unless there is + // an option to move us away from OpenPGP. + "golang.org/x/crypto/openpgp" //nolint:staticcheck + "golang.org/x/crypto/openpgp/armor" //nolint:staticcheck + "golang.org/x/crypto/openpgp/packet" //nolint:staticcheck + "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/fsutil" "github.com/grafana/grafana/pkg/infra/fs" - "golang.org/x/crypto/openpgp" - "golang.org/x/crypto/openpgp/armor" - "golang.org/x/crypto/openpgp/packet" ) // UpdateRPMRepo updates the RPM repository with the new release. diff --git a/pkg/build/stringutil/contains.go b/pkg/build/stringutil/contains.go new file mode 100644 index 00000000000..b53efe70759 --- /dev/null +++ b/pkg/build/stringutil/contains.go @@ -0,0 +1,10 @@ +package stringutil + +func Contains(arr []string, s string) bool { + for _, e := range arr { + if e == s { + return true + } + } + return false +} diff --git a/pkg/build/versions/version.go b/pkg/build/versions/version.go new file mode 100644 index 00000000000..c1ee0b33d67 --- /dev/null +++ b/pkg/build/versions/version.go @@ -0,0 +1,160 @@ +package versions + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "regexp" + "strconv" + + "github.com/Masterminds/semver/v3" +) + +var ( + reGrafanaTag = regexp.MustCompile(`^v(\d+\.\d+\.\d+$)`) + reGrafanaTagBeta = regexp.MustCompile(`^v(\d+\.\d+\.\d+-beta)`) + reGrafanaTagCustom = regexp.MustCompile(`^v(\d+\.\d+\.\d+-\w+)`) +) + +const ( + Latest = "latest" + Next = "next" + Test = "test" +) + +type Version struct { + Version string + Channel string +} + +type VersionFromAPI struct { + Version string `json:"version"` +} + +type LatestGcomAPI = string + +const ( + LatestStableVersionURL LatestGcomAPI = "https://grafana.com/api/grafana/versions/stable" + LatestBetaVersionURL LatestGcomAPI = "https://grafana.com/api/grafana/versions/beta" +) + +func GetLatestVersion(url LatestGcomAPI) (string, error) { + // nolint:gosec + resp, err := http.Get(url) + if err != nil { + return "", err + } + defer func() { + if err := resp.Body.Close(); err != nil { + log.Printf("Failed to close body: %s", err.Error()) + } + }() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("server returned non 200 status code: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + var apiResponse VersionFromAPI + err = json.Unmarshal(body, &apiResponse) + if err != nil { + return "", err + } + + return apiResponse.Version, nil +} + +// IsGreaterThanOrEqual semantically checks whether newVersion is greater than or equal to stableVersion. +func IsGreaterThanOrEqual(newVersion, stableVersion string) (bool, error) { + v1SemVer, err := semver.NewVersion(newVersion) + if err != nil { + return isGreaterThanOrEqualFourDigit(newVersion, stableVersion) + } + + v2SemVer, err := semver.NewVersion(stableVersion) + if err != nil { + return isGreaterThanOrEqualFourDigit(newVersion, stableVersion) + } + + comp := v1SemVer.Compare(v2SemVer) + switch comp { + case -1: + return false, nil + case 1, 0: + return true, nil + default: + return true, fmt.Errorf("unknown comparison value between scemantic versions, err: %q", err) + } +} + +var fourDigitRe = regexp.MustCompile(`(\d+\.\d+\.\d+)\.(\d+)`) + +func parseFourDigit(version string) (*semver.Version, int, error) { + matches := fourDigitRe.FindStringSubmatch(version) + if len(matches) < 2 { + semVer, err := semver.NewVersion(version) + if err != nil { + return nil, 0, err + } + return semVer, 0, nil + } + semVer, err := semver.NewVersion(matches[1]) + if err != nil { + return nil, 0, err + } + i, err := strconv.Atoi(matches[2]) + if err != nil { + return nil, 0, err + } + + return semVer, i, nil +} + +func isGreaterThanOrEqualFourDigit(newVersion, stableVersion string) (bool, error) { + newVersionSemVer, newVersionSemVerNo, err := parseFourDigit(newVersion) + if err != nil { + return false, err + } + + stableVersionSemVer, stableVersionSemVerNo, err := parseFourDigit(stableVersion) + if err != nil { + return false, err + } + + if stableVersionSemVer.Original() != newVersionSemVer.Original() { + return IsGreaterThanOrEqual(newVersionSemVer.Original(), stableVersionSemVer.Original()) + } + + return newVersionSemVerNo >= stableVersionSemVerNo, nil +} + +func GetVersion(tag string) (*Version, error) { + var version Version + switch { + case reGrafanaTag.MatchString(tag): + version = Version{ + Version: reGrafanaTag.FindStringSubmatch(tag)[1], + Channel: Latest, + } + case reGrafanaTagBeta.MatchString(tag): + version = Version{ + Version: reGrafanaTagBeta.FindStringSubmatch(tag)[1], + Channel: Next, + } + case reGrafanaTagCustom.MatchString(tag): + version = Version{ + Version: reGrafanaTagCustom.FindStringSubmatch(tag)[1], + Channel: Test, + } + default: + return nil, fmt.Errorf("%s not a supported Grafana version, exitting", tag) + } + + return &version, nil +} diff --git a/pkg/build/versions/version_test.go b/pkg/build/versions/version_test.go new file mode 100644 index 00000000000..8d80ace1429 --- /dev/null +++ b/pkg/build/versions/version_test.go @@ -0,0 +1,69 @@ +package versions + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIsGreaterThanOrEqual(t *testing.T) { + testCases := []struct { + newVersion string + stableVersion string + expected bool + }{ + {newVersion: "9.0.0", stableVersion: "8.0.0", expected: true}, + {newVersion: "6.0.0", stableVersion: "6.0.0", expected: true}, + {newVersion: "7.0.0", stableVersion: "8.0.0", expected: false}, + {newVersion: "8.5.0-beta1", stableVersion: "8.0.0", expected: true}, + {newVersion: "8.5.0", stableVersion: "8.5.0-beta1", expected: true}, + {newVersion: "9.0.0.1", stableVersion: "9.0.0", expected: true}, + {newVersion: "9.0.0.2", stableVersion: "9.0.0.1", expected: true}, + {newVersion: "9.1.0", stableVersion: "9.0.0.1", expected: true}, + {newVersion: "9.1-0-beta1", stableVersion: "9.0.0.1", expected: true}, + {newVersion: "9.0.0.1", stableVersion: "9.0.1.1", expected: false}, + {newVersion: "9.0.1.1", stableVersion: "9.0.0.1", expected: true}, + {newVersion: "9.0.0.1", stableVersion: "9.0.0.1", expected: true}, + {newVersion: "7.0.0.1", stableVersion: "8.0.0", expected: false}, + {newVersion: "9.1-0-beta1", stableVersion: "9.1-0-beta2", expected: false}, + {newVersion: "9.1-0-beta3", stableVersion: "9.1-0-beta2", expected: true}, + } + + for _, tc := range testCases { + name := fmt.Sprintf("newVersion %s greater than or equal stableVersion %s = %v", tc.newVersion, tc.stableVersion, tc.expected) + t.Run(name, func(t *testing.T) { + result, err := IsGreaterThanOrEqual(tc.newVersion, tc.stableVersion) + require.NoError(t, err) + require.Equal(t, tc.expected, result) + }) + } +} + +func TestGetLatestVersion(t *testing.T) { + t.Run("it returns a version", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + response := VersionFromAPI{ + Version: "8.4.0", + } + jsonRes, err := json.Marshal(&response) + require.NoError(t, err) + _, err = w.Write(jsonRes) + require.NoError(t, err) + })) + version, err := GetLatestVersion(server.URL) + require.NoError(t, err) + require.Equal(t, "8.4.0", version) + }) + + t.Run("it handles non 200 responses", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + _, err := GetLatestVersion(server.URL) + require.Error(t, err) + }) +} diff --git a/public/app/features/dashboard/components/ShareModal/SharePublicDashboard/SharePublicDashboard.test.tsx b/public/app/features/dashboard/components/ShareModal/SharePublicDashboard/SharePublicDashboard.test.tsx index 690fc241f97..e6ea9772cea 100644 --- a/public/app/features/dashboard/components/ShareModal/SharePublicDashboard/SharePublicDashboard.test.tsx +++ b/public/app/features/dashboard/components/ShareModal/SharePublicDashboard/SharePublicDashboard.test.tsx @@ -87,6 +87,7 @@ beforeEach(() => { mockPanel = new PanelModel({ id: 'mockPanelId', + timezone: 'utc', }); jest.spyOn(contextSrv, 'hasAccess').mockReturnValue(true); @@ -143,7 +144,7 @@ describe('SharePublic', () => { await renderSharePublicDashboard({ panel: mockPanel, dashboard: mockDashboard, onDismiss: () => {} }); await screen.findByText('Welcome to Grafana public dashboards alpha!'); - expect(screen.getByText('2022-08-30 00:00:00 to 2022-09-04 01:59:59')).toBeInTheDocument(); + expect(screen.getByText('2022-08-30 00:00:00 to 2022-09-04 00:59:59')).toBeInTheDocument(); }); it('when modal is opened, then loader spinner appears and inputs are disabled', async () => { await renderSharePublicDashboard({ panel: mockPanel, dashboard: mockDashboard, onDismiss: () => {} }); diff --git a/scripts/drone/TAGS b/scripts/drone/TAGS new file mode 100644 index 00000000000..1c9f9da89a5 --- /dev/null +++ b/scripts/drone/TAGS @@ -0,0 +1,628 @@ + +events/release.star,6652 +ver_mode =ver_mode64,1602 +release_trigger =release_trigger65,1623 +def store_npm_packages_step():store_npm_packages_step74,1752 +def retrieve_npm_packages_step():retrieve_npm_packages_step90,2193 +def release_npm_packages_step():release_npm_packages_step107,2663 +def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):oss_pipelines123,3076 + environment =environment135,3492 + edition =edition136,3529 + services =services137,3549 + volumes =volumes138,3609 + package_steps =package_steps139,3659 + publish_steps =publish_steps140,3682 + should_publish =should_publish141,3705 + should_upload =should_upload142,3748 + init_steps =init_steps143,3818 + build_steps =build_steps152,4033 + integration_test_steps =integration_test_steps159,4342 + build_storybook =build_storybook182,5254 + publish_step =publish_step190,5674 + store_npm_step =store_npm_step191,5758 + windows_package_steps =windows_package_steps196,5957 + windows_pipeline =windows_pipeline198,6044 + name =name199,6077 + edition =edition200,6127 + trigger =trigger201,6154 + steps =steps202,6181 + platform =platform203,6256 + depends_on =depends_on204,6286 + environment =environment207,6393 + pipelines =pipelines209,6434 + name =name211,6470 + edition =edition212,6550 + trigger =trigger213,6581 + services =services214,6612 + steps =steps215,6639 + environment =environment216,6717 + volumes =volumes217,6756 + name =name225,6970 + edition =edition226,7038 + trigger =trigger227,7073 + services =services228,7108 + steps =steps229,7145 + environment =environment230,7329 + volumes =volumes231,7372 + deps =deps234,7433 +def enterprise_pipelines(ver_mode = ver_mode, trigger = release_trigger):enterprise_pipelines247,7856 + environment =environment259,8284 + edition =edition260,8328 + services =services261,8355 + volumes =volumes262,8415 + package_steps =package_steps263,8465 + publish_steps =publish_steps264,8488 + should_publish =should_publish265,8511 + should_upload =should_upload266,8554 + include_enterprise =include_enterprise267,8624 + edition2 =edition2268,8673 + init_steps =init_steps269,8702 + build_steps =build_steps277,8909 + integration_test_steps =integration_test_steps284,9218 + build_storybook =build_storybook312,10299 + publish_step =publish_step324,10892 + store_npm_step =store_npm_step325,10976 + windows_package_steps =windows_package_steps330,11175 + step =step333,11284 + deps_on_clone_enterprise_step =deps_on_clone_enterprise_step337,11418 + windows_pipeline =windows_pipeline347,11746 + name =name348,11779 + edition =edition349,11836 + trigger =trigger350,11863 + steps =steps351,11890 + platform =platform352,11965 + depends_on =depends_on353,11995 + environment =environment356,12109 + pipelines =pipelines358,12150 + name =name360,12186 + edition =edition361,12273 + trigger =trigger362,12304 + services =services363,12335 + steps =steps364,12362 + environment =environment365,12440 + volumes =volumes366,12479 + name =name374,12711 + edition =edition375,12786 + trigger =trigger376,12821 + services =services377,12856 + steps =steps378,12893 + environment =environment379,13213 + volumes =volumes380,13256 + deps =deps383,13317 +def enterprise2_pipelines(prefix = "", ver_mode = ver_mode, trigger = release_trigger):enterprise2_pipelines397,13769 + environment =environment412,14364 + edition =edition415,14424 + volumes =volumes416,14451 + package_steps =package_steps417,14501 + publish_steps =publish_steps418,14524 + should_publish =should_publish419,14547 + should_upload =should_upload420,14590 + include_enterprise =include_enterprise421,14660 + edition2 =edition2422,14709 + init_steps =init_steps423,14738 + build_steps =build_steps431,14945 + fetch_images =fetch_images442,15355 + upload_cdn =upload_cdn444,15497 + step =step458,16187 + deps_on_clone_enterprise_step =deps_on_clone_enterprise_step462,16321 + pipelines =pipelines472,16608 + name =name474,16644 + edition =edition475,16742 + trigger =trigger476,16773 + services =services477,16804 + steps =steps478,16831 + volumes =volumes479,16909 + environment =environment480,16940 +def publish_artifacts_step(mode):publish_artifacts_step486,17019 + security =security487,17053 + security =security489,17098 +def publish_artifacts_pipelines(mode):publish_artifacts_pipelines501,17538 + trigger =trigger502,17577 + steps =steps506,17655 + name =name512,17768 + trigger =trigger513,17820 + steps =steps514,17847 + edition =edition515,17870 + environment =environment516,17895 +def publish_packages_pipeline():publish_packages_pipeline519,17945 + trigger =trigger526,18162 + oss_steps =oss_steps530,18244 + enterprise_steps =enterprise_steps538,18560 + deps =deps545,18903 + name =name552,19062 + trigger =trigger553,19101 + steps =steps554,19128 + edition =edition555,19155 + depends_on =depends_on556,19180 + environment =environment557,19207 + name =name559,19266 + trigger =trigger560,19312 + steps =steps561,19339 + edition =edition562,19373 + depends_on =depends_on563,19398 + environment =environment564,19425 +def publish_npm_pipelines(mode):publish_npm_pipelines567,19482 + trigger =trigger568,19515 + steps =steps572,19593 + name =name580,19772 + trigger =trigger581,19827 + steps =steps582,19854 + edition =edition583,19877 + environment =environment584,19902 +def artifacts_page_pipeline():artifacts_page_pipeline587,19952 + trigger =trigger588,19983 + name =name593,20087 + trigger =trigger594,20128 + steps =steps595,20155 + edition =edition596,20220 + environment =environment597,20245 +def get_e2e_suffix():get_e2e_suffix600,20295 + +events/cron.star,1016 +aquasec_trivy_image =aquasec_trivy_image8,209 +def cronjobs(edition):cronjobs10,255 + grafana_com_nightly_pipeline =grafana_com_nightly_pipeline11,278 + cronName =cronName12,332 + name =name13,374 + steps =steps14,412 +def cron_job_pipeline(cronName, name, steps):cron_job_pipeline24,773 +def scan_docker_image_pipeline(edition, tag):scan_docker_image_pipeline43,1175 + edition =edition55,1530 + edition =edition57,1579 + docker_image =docker_image59,1608 + cronName =cronName62,1695 + name =name63,1725 + steps =steps64,1775 +def scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image):scan_docker_image_unkown_low_medium_vulnerabilities_step71,2047 +def scan_docker_image_high_critical_vulnerabilities_step(docker_image):scan_docker_image_high_critical_vulnerabilities_step80,2353 +def slack_job_failed_step(channel, image):slack_job_failed_step89,2646 +def post_to_grafana_com_step():post_to_grafana_com_step103,3069 + +events/main.star,633 +ver_mode =ver_mode49,966 +trigger =trigger50,984 +def main_pipelines(edition):main_pipelines62,1168 + drone_change_trigger =drone_change_trigger63,1197 + pipelines =pipelines79,1513 + name =name89,1951 + slack_channel =slack_channel90,1994 + trigger =trigger91,2045 + template =template92,2089 + secret =secret93,2135 + name =name97,2276 + slack_channel =slack_channel98,2310 + trigger =trigger99,2366 + depends_on =depends_on100,2425 + template =template101,2563 + secret =secret102,2604 + +events/pr.star,252 +ver_mode =ver_mode48,997 +trigger =trigger49,1013 +def pr_pipelines(edition):pr_pipelines62,1198 +def get_pr_trigger(include_paths = None, exclude_paths = None):get_pr_trigger76,2396 + paths_ex =paths_ex91,3080 + paths_in =paths_in92,3115 + +services/services.star,225 +def integration_test_services_volumes():integration_test_services_volumes5,79 +def integration_test_services(edition):integration_test_services14,292 + services =services15,332 +def ldap_service():ldap_service59,1616 + +utils/utils.star,561 +failure_template =failure_template11,191 +drone_change_template =drone_change_template12,509 + services =services19,932 + platform =platform20,955 + depends_on =depends_on21,983 + environment =environment22,1008 + volumes =volumes23,1036 + platform_conf =platform_conf50,2166 + platform_conf =platform_conf62,2534 + pipeline =pipeline70,2713 +def notify_pipeline(name, slack_channel, trigger, depends_on = [], template = None, secret = None):notify_pipeline105,3545 + trigger =trigger106,3645 + +pipelines/trigger_downstream.star,440 +trigger =trigger14,249 +def enterprise_downstream_pipeline(edition, ver_mode):enterprise_downstream_pipeline26,433 + environment =environment27,488 + steps =steps28,527 + deps =deps29,587 + name =name31,672 + edition =edition32,714 + trigger =trigger33,741 + services =services34,768 + steps =steps35,791 + depends_on =depends_on36,814 + environment =environment37,841 + +pipelines/verify_starlark.star,323 +def verify_starlark(trigger, ver_mode):verify_starlark17,305 + environment =environment18,345 + steps =steps19,382 + name =name26,546 + edition =edition27,600 + trigger =trigger28,625 + services =services29,652 + steps =steps30,675 + environment =environment31,698 + +pipelines/build.star,508 +def build_e2e(trigger, ver_mode, edition):build_e2e39,936 + environment =environment50,1096 + variants =variants51,1135 + init_steps =init_steps52,1219 + build_steps =build_steps61,1491 + publish_suffix =publish_suffix107,4049 + publish_suffix =publish_suffix109,4100 + name =name112,4158 + edition =edition113,4224 + environment =environment114,4249 + services =services115,4284 + steps =steps116,4307 + trigger =trigger117,4349 + +pipelines/shellcheck.star,386 +trigger =trigger15,235 +def shellcheck_step():shellcheck_step31,483 +def shellcheck_pipeline():shellcheck_pipeline43,725 + environment =environment44,752 + steps =steps45,789 + name =name50,886 + edition =edition51,918 + trigger =trigger52,943 + services =services53,970 + steps =steps54,993 + environment =environment55,1016 + +pipelines/verify_drone.star,317 +def verify_drone(trigger, ver_mode):verify_drone17,293 + environment =environment18,330 + steps =steps19,367 + name =name26,528 + edition =edition27,579 + trigger =trigger28,604 + services =services29,631 + steps =steps30,654 + environment =environment31,677 + +pipelines/test_backend.star,474 +def test_backend(trigger, ver_mode, edition = "oss"):test_backend23,463 + environment =environment35,882 + init_steps =init_steps36,921 + test_steps =test_steps46,1291 + pipeline_name =pipeline_name51,1387 + pipeline_name =pipeline_name53,1492 + name =name55,1584 + edition =edition56,1614 + trigger =trigger57,1641 + services =services58,1668 + steps =steps59,1691 + environment =environment60,1732 + +pipelines/lint_frontend.star,415 +def lint_frontend_pipeline(trigger, ver_mode):lint_frontend_pipeline16,260 + environment =environment26,546 + yarn_step =yarn_step27,583 + init_steps =init_steps29,660 + test_steps =test_steps33,736 + name =name37,812 + edition =edition38,864 + trigger =trigger39,889 + services =services40,916 + steps =steps41,939 + environment =environment42,980 + +pipelines/docs.star,494 +docs_paths =docs_paths19,383 +def docs_pipelines(edition, ver_mode, trigger):docs_pipelines28,511 + environment =environment29,559 + steps =steps30,598 + name =name40,815 + edition =edition41,858 + trigger =trigger42,885 + services =services43,912 + steps =steps44,935 + environment =environment45,958 +def lint_docs():lint_docs48,1000 +def trigger_docs_main():trigger_docs_main63,1328 +def trigger_docs_pr():trigger_docs_pr72,1478 + +pipelines/test_frontend.star,476 +def test_frontend(trigger, ver_mode, edition = "oss"):test_frontend20,374 + environment =environment32,794 + init_steps =init_steps33,833 + test_steps =test_steps41,1102 + pipeline_name =pipeline_name45,1205 + pipeline_name =pipeline_name47,1311 + name =name49,1404 + edition =edition50,1434 + trigger =trigger51,1461 + services =services52,1488 + steps =steps53,1511 + environment =environment54,1552 + +pipelines/integration_tests.star,483 +def integration_tests(trigger, ver_mode, edition):integration_tests26,542 + environment =environment37,900 + services =services38,939 + volumes =volumes39,989 + init_steps =init_steps40,1039 + test_steps =test_steps48,1282 + name =name54,1412 + edition =edition55,1468 + trigger =trigger56,1493 + services =services57,1520 + steps =steps58,1549 + environment =environment59,1590 + volumes =volumes60,1625 + +pipelines/windows.star,954 +def windows(trigger, edition, ver_mode):windows17,339 + environment =environment29,798 + init_cmds =init_cmds30,837 + steps =steps38,1205 + bucket =bucket49,1497 + ver_part =ver_part51,1590 + dir =dir52,1628 + dir =dir54,1670 + bucket =bucket55,1695 + build_no =build_no56,1736 + ver_part =ver_part57,1780 + installer_commands =installer_commands58,1842 + committish =committish100,3763 + committish =committish102,3846 + committish =committish104,3906 + download_grabpl_step_cmds =download_grabpl_step_cmds107,4057 + clone_cmds =clone_cmds113,4363 + name =name146,5711 + edition =edition147,5742 + trigger =trigger148,5769 + steps =steps149,5830 + depends_on =depends_on150,5889 + platform =platform151,6007 + environment =environment152,6037 + +pipelines/lint_backend.star,418 +def lint_backend_pipeline(trigger, ver_mode):lint_backend_pipeline18,306 + environment =environment28,590 + wire_step =wire_step29,627 + init_steps =init_steps31,704 + test_steps =test_steps36,809 + name =name43,959 + edition =edition44,1010 + trigger =trigger45,1035 + services =services46,1062 + steps =steps47,1085 + environment =environment48,1126 + +pipelines/publish_images.star,998 +def publish_image_steps(edition, mode, docker_repo):publish_image_steps17,303 + additional_docker_repo =additional_docker_repo31,922 + additional_docker_repo =additional_docker_repo33,979 + steps =steps34,1034 +def publish_image_pipelines_public():publish_image_pipelines_public45,1369 + mode =mode51,1521 + trigger =trigger52,1541 + name =name57,1641 + trigger =trigger58,1694 + steps =steps59,1721 + edition =edition60,1813 + environment =environment61,1835 + name =name63,1894 + trigger =trigger64,1954 + steps =steps65,1981 + edition =edition66,2091 + environment =environment67,2113 +def publish_image_pipelines_security():publish_image_pipelines_security70,2170 + mode =mode71,2210 + trigger =trigger72,2232 + name =name77,2332 + trigger =trigger78,2392 + steps =steps79,2419 + edition =edition80,2529 + environment =environment81,2551 + +steps/lib.star,8579 +grabpl_version =grabpl_version7,181 +build_image =build_image8,208 +publish_image =publish_image9,254 +deploy_docker_image =deploy_docker_image10,304 +alpine_image =alpine_image11,380 +curl_image =curl_image12,411 +windows_image =windows_image13,452 +wix_image =wix_image14,501 +go_image =go_image15,536 +disable_tests =disable_tests17,564 +trigger_oss =trigger_oss18,586 +def slack_step(channel, template, secret):slack_step24,653 +def yarn_install_step(edition = "oss"):yarn_install_step35,918 + deps =deps36,958 + deps =deps38,1004 +def wire_install_step():wire_install_step48,1222 +def identify_runner_step(platform = "linux"):identify_runner_step60,1454 +def clone_enterprise_step(ver_mode):clone_enterprise_step78,1916 + committish =committish87,2193 + committish =committish89,2268 + committish =committish91,2317 +def init_enterprise_step(ver_mode):init_enterprise_step105,2747 + source_commit =source_commit115,3098 + source_commit =source_commit117,3151 + environment =environment118,3191 + token =token121,3280 + environment =environment123,3369 + token =token126,3458 + environment =environment128,3518 + token =token129,3543 +def download_grabpl_step(platform = "linux"):download_grabpl_step148,4147 +def lint_drone_step():lint_drone_step173,4973 +def lint_starlark_step():lint_starlark_step185,5216 +def enterprise_downstream_step(edition, ver_mode):enterprise_downstream_step206,6000 + repo =repo219,6482 + step =step225,6623 +def lint_backend_step():lint_backend_step247,7248 +def benchmark_ldap_step():benchmark_ldap_step265,7713 +def build_storybook_step(edition, ver_mode):build_storybook_step278,8087 +def store_storybook_step(edition, ver_mode, trigger = None):store_storybook_step300,8743 + commands =commands314,9202 + commands =commands323,9521 + step =step325,9593 + when_cond =when_cond338,10125 + step =step346,10330 +def e2e_tests_artifacts(edition):e2e_tests_artifacts349,10391 +def upload_cdn_step(edition, ver_mode, trigger = None):upload_cdn_step386,12378 + deps =deps397,12763 + step =step407,12970 + step =step420,13423 +def build_backend_step(edition, ver_mode, variants = None):build_backend_step423,13482 + variants_str =variants_str437,14070 + variants_str =variants_str439,14109 + cmds =cmds443,14256 + build_no =build_no449,14418 + cmds =cmds450,14461 +def build_frontend_step(edition, ver_mode):build_frontend_step468,14906 + build_no =build_no478,15246 + cmds =cmds482,15356 + cmds =cmds487,15505 +def build_frontend_package_step(edition, ver_mode):build_frontend_package_step505,15960 + build_no =build_no515,16312 + cmds =cmds519,16422 + cmds =cmds524,16580 +def build_plugins_step(edition, ver_mode):build_plugins_step542,17053 + env =env544,17121 + env =env548,17220 +def test_backend_step():test_backend_step563,17607 +def test_backend_integration_step():test_backend_integration_step575,17880 +def betterer_frontend_step(edition = "oss"):betterer_frontend_step587,18187 + deps =deps596,18427 +def test_frontend_step(edition = "oss"):test_frontend_step609,18728 + deps =deps618,18962 +def lint_frontend_step():lint_frontend_step634,19343 +def test_a11y_frontend_step(ver_mode, edition, port = 3001):test_a11y_frontend_step652,19793 + commands =commands664,20279 + failure =failure667,20345 + failure =failure672,20483 +def frontend_metrics_step(edition, trigger = None):frontend_metrics_step693,21146 + step =step706,21507 + step =step721,22007 +def codespell_step():codespell_step724,22066 +def package_step(edition, ver_mode, variants = None):package_step736,22468 + deps =deps750,23006 + variants_str =variants_str757,23167 + variants_str =variants_str759,23206 + sign_args =sign_args762,23332 + env =env763,23362 + test_args =test_args769,23628 + sign_args =sign_args771,23661 + env =env772,23684 + test_args =test_args773,23703 + cmds =cmds777,23829 + build_no =build_no784,24036 + cmds =cmds785,24079 +def grafana_server_step(edition, port = 3001):grafana_server_step798,24459 + package_file_pfx =package_file_pfx808,24729 + package_file_pfx =package_file_pfx810,24788 + package_file_pfx =package_file_pfx812,24889 + environment =environment814,24938 +def e2e_tests_step(suite, edition, port = 3001, tries = None):e2e_tests_step837,25554 + cmd =cmd838,25617 +def cloud_plugins_e2e_tests_step(suite, edition, cloud, trigger = None):cloud_plugins_e2e_tests_step856,26186 + environment =environment869,26649 + when =when870,26670 + when =when872,26700 + environment =environment874,26748 + when =when882,27129 + branch =branch888,27345 + step =step889,27401 + step =step901,27822 +def build_docs_website_step():build_docs_website_step904,27874 +def copy_packages_for_docker_step(edition = None):copy_packages_for_docker_step916,28272 +def build_docker_images_step(edition, archs = None, ubuntu = False, publish = False):build_docker_images_step929,28622 + cmd =cmd943,29193 + ubuntu_sfx =ubuntu_sfx947,29307 + ubuntu_sfx =ubuntu_sfx949,29342 + environment =environment955,29468 +def fetch_images_step(edition):fetch_images_step979,30079 +def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None):publish_images_step997,30745 + name =name1013,31562 + docker_repo =docker_repo1014,31585 + mode =mode1016,31663 + mode =mode1018,31709 + environment =environment1020,31728 + cmd =cmd1026,31912 + deps =deps1029,32041 + deps =deps1032,32147 + name =name1035,32250 + docker_repo =docker_repo1036,32273 + cmd =cmd1038,32459 + step =step1040,32565 + step =step1052,32929 +def postgres_integration_tests_step():postgres_integration_tests_step1056,32989 + cmds =cmds1057,33028 +def mysql_integration_tests_step():mysql_integration_tests_step1079,33850 + cmds =cmds1080,33886 +def redis_integration_tests_step():redis_integration_tests_step1100,34629 +def memcached_integration_tests_step():memcached_integration_tests_step1114,35026 +def release_canary_npm_packages_step(edition, trigger = None):release_canary_npm_packages_step1128,35435 + step =step1141,35805 + step =step1153,36143 +def enterprise2_suffix(edition):enterprise2_suffix1156,36202 +def upload_packages_step(edition, ver_mode, trigger = None):upload_packages_step1161,36320 + deps =deps1176,36816 + step =step1184,37036 + step =step1195,37471 +def publish_grafanacom_step(edition, ver_mode):publish_grafanacom_step1198,37530 + cmd =cmd1211,38044 + build_no =build_no1215,38188 + cmd =cmd1216,38231 +def publish_linux_packages_step(edition, package_manager = "deb"):publish_linux_packages_step1239,38866 +def get_windows_steps(edition, ver_mode):get_windows_steps1261,39989 + init_cmds =init_cmds1270,40281 + steps =steps1278,40649 + bucket =bucket1289,40941 + ver_part =ver_part1291,41034 + dir =dir1292,41072 + dir =dir1294,41114 + bucket =bucket1295,41139 + build_no =build_no1296,41180 + ver_part =ver_part1297,41224 + installer_commands =installer_commands1298,41286 + committish =committish1340,43207 + committish =committish1342,43290 + committish =committish1344,43350 + download_grabpl_step_cmds =download_grabpl_step_cmds1347,43501 + clone_cmds =clone_cmds1353,43807 +def verify_gen_cue_step(edition):verify_gen_cue_step1387,45152 + deps =deps1388,45186 +def verify_gen_jsonnet_step(edition):verify_gen_jsonnet_step1402,45694 + deps =deps1403,45732 +def trigger_test_release():trigger_test_release1417,46236 +def artifacts_page_step():artifacts_page_step1451,47731 +def end_to_end_tests_deps():end_to_end_tests_deps1466,48058 +def compile_build_cmd(edition = "oss"):compile_build_cmd1476,48321 + dependencies =dependencies1477,48361 + dependencies =dependencies1479,48432 +def get_trigger_storybook(ver_mode):get_trigger_storybook1492,48780 + trigger_storybook =trigger_storybook1500,49031 + trigger_storybook =trigger_storybook1502,49088 + trigger_storybook =trigger_storybook1506,49168 + +vault.star,444 +pull_secret =pull_secret4,87 +github_token =github_token5,120 +drone_token =drone_token6,150 +prerelease_bucket =prerelease_bucket7,178 +gcp_upload_artifacts_key =gcp_upload_artifacts_key8,218 +azure_sp_app_id =azure_sp_app_id9,272 +azure_sp_app_pw =azure_sp_app_pw10,308 +azure_tenant =azure_tenant11,344 +def from_secret(secret):from_secret13,375 +def vault_secret(name, path, key):vault_secret18,451 +def secrets():secrets28,633 + +version.star,116 +ver_mode =ver_mode12,197 +trigger =trigger13,225 +def version_branch_pipelines():version_branch_pipelines15,268 diff --git a/scripts/drone/events/cron.star b/scripts/drone/events/cron.star index e36814f5dc3..9b718f79572 100644 --- a/scripts/drone/events/cron.star +++ b/scripts/drone/events/cron.star @@ -1,79 +1,115 @@ -load('scripts/drone/vault.star', 'from_secret') +""" +This module provides functions for cronjob pipelines and steps used within. +""" -aquasec_trivy_image = 'aquasec/trivy:0.21.0' +load("scripts/drone/vault.star", "from_secret") +load( + "scripts/drone/steps/lib.star", + "compile_build_cmd", + "publish_image", +) -def cronjobs(edition): +aquasec_trivy_image = "aquasec/trivy:0.21.0" + +def cronjobs(): return [ - scan_docker_image_pipeline(edition, 'latest'), - scan_docker_image_pipeline(edition, 'main'), - scan_docker_image_pipeline(edition, 'latest-ubuntu'), - scan_docker_image_pipeline(edition, 'main-ubuntu'), + scan_docker_image_pipeline("latest"), + scan_docker_image_pipeline("main"), + scan_docker_image_pipeline("latest-ubuntu"), + scan_docker_image_pipeline("main-ubuntu"), + grafana_com_nightly_pipeline(), ] def cron_job_pipeline(cronName, name, steps): return { - 'kind': 'pipeline', - 'type': 'docker', - 'platform': { - 'os': 'linux', - 'arch': 'amd64', + "kind": "pipeline", + "type": "docker", + "platform": { + "os": "linux", + "arch": "amd64", }, - 'name': name, - 'trigger': { - 'event': 'cron', - 'cron': cronName, + "name": name, + "trigger": { + "event": "cron", + "cron": cronName, }, - 'clone': { - 'retries': 3, + "clone": { + "retries": 3, }, - 'steps': steps, + "steps": steps, } -def scan_docker_image_pipeline(edition, tag): - if edition != 'oss': - edition='grafana-enterprise' - else: - edition='grafana' +def scan_docker_image_pipeline(tag): + """Generates a cronjob pipeline for nightly scans of grafana Docker images. - dockerImage='grafana/{}:{}'.format(edition, tag) + Args: + tag: determines which image tag is scanned. + + Returns: + Drone cronjob pipeline. + """ + docker_image = "grafana/grafana:{}".format(tag) return cron_job_pipeline( - cronName='nightly', - name='scan-' + dockerImage + '-image', - steps=[ - scan_docker_image_unkown_low_medium_vulnerabilities_step(dockerImage), - scan_docker_image_high_critical_vulnerabilities_step(dockerImage), - slack_job_failed_step('grafana-backend-ops', dockerImage), - ]) + cronName = "nightly", + name = "scan-" + docker_image + "-image", + steps = [ + scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image), + scan_docker_image_high_critical_vulnerabilities_step(docker_image), + slack_job_failed_step("grafana-backend-ops", docker_image), + ], + ) -def scan_docker_image_unkown_low_medium_vulnerabilities_step(dockerImage): +def scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image): return { - 'name': 'scan-unkown-low-medium-vulnerabilities', - 'image': aquasec_trivy_image, - 'commands': [ - 'trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM ' + dockerImage, + "name": "scan-unkown-low-medium-vulnerabilities", + "image": aquasec_trivy_image, + "commands": [ + "trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image, ], } -def scan_docker_image_high_critical_vulnerabilities_step(dockerImage): +def scan_docker_image_high_critical_vulnerabilities_step(docker_image): return { - 'name': 'scan-high-critical-vulnerabilities', - 'image': aquasec_trivy_image, - 'commands': [ - 'trivy --exit-code 1 --severity HIGH,CRITICAL ' + dockerImage, + "name": "scan-high-critical-vulnerabilities", + "image": aquasec_trivy_image, + "commands": [ + "trivy --exit-code 1 --severity HIGH,CRITICAL " + docker_image, ], } def slack_job_failed_step(channel, image): return { - 'name': 'slack-notify-failure', - 'image': 'plugins/slack', - 'settings': { - 'webhook': from_secret('slack_webhook_backend'), - 'channel': channel, - 'template': 'Nightly docker image scan job for ' + image + ' failed: {{build.link}}', + "name": "slack-notify-failure", + "image": "plugins/slack", + "settings": { + "webhook": from_secret("slack_webhook_backend"), + "channel": channel, + "template": "Nightly docker image scan job for " + + image + + " failed: {{build.link}}", }, - 'when': { - 'status': 'failure' - } + "when": {"status": "failure"}, } + +def post_to_grafana_com_step(): + return { + "name": "post-to-grafana-com", + "image": publish_image, + "environment": { + "GRAFANA_COM_API_KEY": from_secret("grafana_api_key"), + "GCP_KEY": from_secret("gcp_key"), + }, + "depends_on": ["compile-build-cmd"], + "commands": ["./bin/build publish grafana-com --edition oss"], + } + +def grafana_com_nightly_pipeline(): + return cron_job_pipeline( + cronName = "grafana-com-nightly", + name = "grafana-com-nightly", + steps = [ + compile_build_cmd(), + post_to_grafana_com_step(), + ], + ) diff --git a/scripts/drone/events/main.star b/scripts/drone/events/main.star index a718362e958..f3228431503 100644 --- a/scripts/drone/events/main.star +++ b/scripts/drone/events/main.star @@ -1,114 +1,116 @@ -load( - 'scripts/drone/utils/utils.star', - 'pipeline', - 'notify_pipeline', - 'failure_template', - 'drone_change_template', -) +""" +This module returns all the pipelines used in the event of pushes to the main branch. +""" load( - 'scripts/drone/pipelines/docs.star', - 'docs_pipelines', - 'trigger_docs_main', + "scripts/drone/utils/utils.star", + "drone_change_template", + "failure_template", + "notify_pipeline", ) - load( - 'scripts/drone/pipelines/test_frontend.star', - 'test_frontend', + "scripts/drone/pipelines/docs.star", + "docs_pipelines", + "trigger_docs_main", ) - load( - 'scripts/drone/pipelines/test_backend.star', - 'test_backend', + "scripts/drone/pipelines/test_frontend.star", + "test_frontend", ) - load( - 'scripts/drone/pipelines/integration_tests.star', - 'integration_tests', + "scripts/drone/pipelines/test_backend.star", + "test_backend", ) - load( - 'scripts/drone/pipelines/build.star', - 'build_e2e', + "scripts/drone/pipelines/integration_tests.star", + "integration_tests", ) - load( - 'scripts/drone/pipelines/windows.star', - 'windows', + "scripts/drone/pipelines/build.star", + "build_e2e", ) - load( - 'scripts/drone/pipelines/publish.star', - 'publish', + "scripts/drone/pipelines/windows.star", + "windows", ) - load( - 'scripts/drone/pipelines/trigger_downstream.star', - 'enterprise_downstream_pipeline', + "scripts/drone/pipelines/trigger_downstream.star", + "enterprise_downstream_pipeline", ) - load( - 'scripts/drone/pipelines/lint_backend.star', - 'lint_backend_pipeline', + "scripts/drone/pipelines/lint_backend.star", + "lint_backend_pipeline", ) - load( - 'scripts/drone/pipelines/lint_frontend.star', - 'lint_frontend_pipeline', + "scripts/drone/pipelines/lint_frontend.star", + "lint_frontend_pipeline", ) -load('scripts/drone/vault.star', 'from_secret') - - -ver_mode = 'main' +ver_mode = "main" trigger = { - 'event': ['push',], - 'branch': 'main', - 'paths': { - 'exclude': [ - '*.md', - 'docs/**', - 'latest.json', + "event": [ + "push", + ], + "branch": "main", + "paths": { + "exclude": [ + "*.md", + "docs/**", + "latest.json", ], }, } -def main_pipelines(edition): +def main_pipelines(): drone_change_trigger = { - 'event': ['push',], - 'branch': 'main', - 'repo': [ - 'grafana/grafana', + "event": [ + "push", ], - 'paths': { - 'include': [ - '.drone.yml', + "branch": "main", + "repo": [ + "grafana/grafana", + ], + "paths": { + "include": [ + ".drone.yml", ], - 'exclude': [ - 'exclude', + "exclude": [ + "exclude", ], }, } pipelines = [ - docs_pipelines(edition, ver_mode, trigger_docs_main()), + docs_pipelines(ver_mode, trigger_docs_main()), test_frontend(trigger, ver_mode), lint_frontend_pipeline(trigger, ver_mode), test_backend(trigger, ver_mode), lint_backend_pipeline(trigger, ver_mode), - build_e2e(trigger, ver_mode, edition), - integration_tests(trigger, ver_mode, edition), - windows(trigger, edition, ver_mode), - notify_pipeline( - name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger, - template=drone_change_template, secret='drone-changes-webhook', - ), - publish(trigger, ver_mode, edition), - enterprise_downstream_pipeline(edition, ver_mode), - notify_pipeline( - name='main-notify', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status=['failure']), - depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', 'main-publish'], - template=failure_template, secret='slack_webhook' - )] + build_e2e(trigger, ver_mode), + integration_tests(trigger, prefix = ver_mode, ver_mode = ver_mode), + windows(trigger, edition = "oss", ver_mode = ver_mode), + notify_pipeline( + name = "notify-drone-changes", + slack_channel = "slack-webhooks-test", + trigger = drone_change_trigger, + template = drone_change_template, + secret = "drone-changes-webhook", + ), + enterprise_downstream_pipeline(), + notify_pipeline( + name = "main-notify", + slack_channel = "grafana-ci-notifications", + trigger = dict(trigger, status = ["failure"]), + depends_on = [ + "main-test-frontend", + "main-test-backend", + "main-build-e2e-publish", + "main-integration-tests", + "main-windows", + ], + template = failure_template, + secret = "slack_webhook", + ), + ] return pipelines diff --git a/scripts/drone/events/pr.star b/scripts/drone/events/pr.star index c193cba8a33..73ab62d2680 100644 --- a/scripts/drone/events/pr.star +++ b/scripts/drone/events/pr.star @@ -1,85 +1,155 @@ -load( - 'scripts/drone/utils/utils.star', - 'pipeline', -) +""" +This module returns all pipelines used in the event of a pull request. +It also includes a function generating a PR trigger from a list of included and excluded paths. +""" load( - 'scripts/drone/pipelines/test_frontend.star', - 'test_frontend', + "scripts/drone/pipelines/test_frontend.star", + "test_frontend", ) - load( - 'scripts/drone/pipelines/test_backend.star', - 'test_backend', + "scripts/drone/pipelines/test_backend.star", + "test_backend", ) - load( - 'scripts/drone/pipelines/integration_tests.star', - 'integration_tests', + "scripts/drone/pipelines/integration_tests.star", + "integration_tests", ) - load( - 'scripts/drone/pipelines/build.star', - 'build_e2e', + "scripts/drone/pipelines/build.star", + "build_e2e", ) - load( - 'scripts/drone/pipelines/verify_drone.star', - 'verify_drone', + "scripts/drone/pipelines/verify_drone.star", + "verify_drone", ) - load( - 'scripts/drone/pipelines/docs.star', - 'docs_pipelines', - 'trigger_docs_pr', + "scripts/drone/pipelines/verify_starlark.star", + "verify_starlark", ) - load( - 'scripts/drone/pipelines/shellcheck.star', - 'shellcheck_pipeline', + "scripts/drone/pipelines/docs.star", + "docs_pipelines", + "trigger_docs_pr", ) - load( - 'scripts/drone/pipelines/lint_backend.star', - 'lint_backend_pipeline', + "scripts/drone/pipelines/shellcheck.star", + "shellcheck_pipeline", ) - load( - 'scripts/drone/pipelines/lint_frontend.star', - 'lint_frontend_pipeline', + "scripts/drone/pipelines/lint_backend.star", + "lint_backend_pipeline", +) +load( + "scripts/drone/pipelines/lint_frontend.star", + "lint_frontend_pipeline", ) -ver_mode = 'pr' +ver_mode = "pr" trigger = { - 'event': [ - 'pull_request', + "event": [ + "pull_request", ], - 'paths': { - 'exclude': [ - '*.md', - 'docs/**', - 'latest.json', + "paths": { + "exclude": [ + "*.md", + "docs/**", + "latest.json", ], }, } - -def pr_pipelines(edition): +def pr_pipelines(): return [ - verify_drone(get_pr_trigger(include_paths=['scripts/drone/**', '.drone.yml', '.drone.star']), ver_mode), - test_frontend(get_pr_trigger(exclude_paths=['pkg/**', 'packaging/**', 'go.sum', 'go.mod']), ver_mode), - lint_frontend_pipeline(get_pr_trigger(exclude_paths=['pkg/**', 'packaging/**', 'go.sum', 'go.mod']), ver_mode), - test_backend(get_pr_trigger(include_paths=['pkg/**', 'packaging/**', '.drone.yml', 'conf/**', 'go.sum', 'go.mod', 'public/app/plugins/**/plugin.json', 'devenv/**']), ver_mode), - lint_backend_pipeline(get_pr_trigger(include_paths=['pkg/**', 'packaging/**', 'conf/**', 'go.sum', 'go.mod', 'public/app/plugins/**/plugin.json', 'devenv/**']), ver_mode), - build_e2e(trigger, ver_mode, edition), - integration_tests(get_pr_trigger(include_paths=['pkg/**', 'packaging/**', '.drone.yml', 'conf/**', 'go.sum', 'go.mod', 'public/app/plugins/**/plugin.json']), ver_mode, edition), - docs_pipelines(edition, ver_mode, trigger_docs_pr()), + verify_drone( + get_pr_trigger( + include_paths = ["scripts/drone/**", ".drone.yml", ".drone.star"], + ), + ver_mode, + ), + verify_starlark( + get_pr_trigger( + include_paths = ["scripts/drone/**", ".drone.star"], + ), + ver_mode, + ), + test_frontend( + get_pr_trigger( + exclude_paths = ["pkg/**", "packaging/**", "go.sum", "go.mod"], + ), + ver_mode, + ), + lint_frontend_pipeline( + get_pr_trigger( + exclude_paths = ["pkg/**", "packaging/**", "go.sum", "go.mod"], + ), + ver_mode, + ), + test_backend( + get_pr_trigger( + include_paths = [ + "pkg/**", + "packaging/**", + ".drone.yml", + "conf/**", + "go.sum", + "go.mod", + "public/app/plugins/**/plugin.json", + "devenv/**", + ], + ), + ver_mode, + ), + lint_backend_pipeline( + get_pr_trigger( + include_paths = [ + "pkg/**", + "packaging/**", + "conf/**", + "go.sum", + "go.mod", + "public/app/plugins/**/plugin.json", + "devenv/**", + ".bingo/**", + ], + ), + ver_mode, + ), + build_e2e(trigger, ver_mode), + integration_tests( + get_pr_trigger( + include_paths = [ + "pkg/**", + "packaging/**", + ".drone.yml", + "conf/**", + "go.sum", + "go.mod", + "public/app/plugins/**/plugin.json", + ], + ), + prefix = ver_mode, + ), + docs_pipelines(ver_mode, trigger_docs_pr()), shellcheck_pipeline(), ] +def get_pr_trigger(include_paths = None, exclude_paths = None): + """Generates a trigger filter from the lists of included and excluded path patterns. -def get_pr_trigger(include_paths=None, exclude_paths=None): - paths_ex = ['docs/**', '*.md'] + This function is primarily intended to generate a trigger for code changes + as the patterns 'docs/**' and '*.md' are always excluded. + + Args: + include_paths: a list of path patterns using the same syntax as gitignore. + Changes affecting files matching these path patterns trigger the pipeline. + exclude_paths: a list of path patterns using the same syntax as gitignore. + Changes affecting files matching these path patterns do not trigger the pipeline. + + Returns: + Drone trigger. + """ + paths_ex = ["docs/**", "*.md"] paths_in = [] if include_paths: for path in include_paths: @@ -88,12 +158,11 @@ def get_pr_trigger(include_paths=None, exclude_paths=None): for path in exclude_paths: paths_ex.extend([path]) return { - 'event': [ - 'pull_request', + "event": [ + "pull_request", ], - 'paths': { - 'exclude': paths_ex, - 'include': paths_in, + "paths": { + "exclude": paths_ex, + "include": paths_in, }, } - diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index fe34c118723..269df3a996d 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -1,468 +1,657 @@ -load( - 'scripts/drone/steps/lib.star', - 'disable_tests', - 'clone_enterprise_step', - 'download_grabpl_step', - 'yarn_install_step', - 'wire_install_step', - 'init_enterprise_step', - 'lint_drone_step', - 'build_image', - 'identify_runner_step', - 'publish_image', - 'lint_backend_step', - 'lint_frontend_step', - 'test_backend_step', - 'test_backend_integration_step', - 'test_frontend_step', - 'build_backend_step', - 'build_frontend_step', - 'build_frontend_package_step', - 'build_plugins_step', - 'package_step', - 'grafana_server_step', - 'e2e_tests_step', - 'e2e_tests_artifacts', - 'build_storybook_step', - 'copy_packages_for_docker_step', - 'build_docker_images_step', - 'postgres_integration_tests_step', - 'mysql_integration_tests_step', - 'redis_integration_tests_step', - 'memcached_integration_tests_step', - 'get_windows_steps', - 'benchmark_ldap_step', - 'store_storybook_step', - 'upload_packages_step', - 'publish_packages_step', - 'publish_grafanacom_step', - 'upload_cdn_step', - 'verify_gen_cue_step', - 'publish_images_step', - 'publish_linux_packages_step', - 'trigger_oss', - 'artifacts_page_step', - 'fetch_images_step', - 'compile_build_cmd', -) +""" +This module returns all the pipelines used in the event of a release along with supporting functions. +""" load( - 'scripts/drone/services/services.star', - 'integration_test_services', - 'integration_test_services_volumes', - 'ldap_service', + "scripts/drone/steps/lib.star", + "artifacts_page_step", + "build_backend_step", + "build_docker_images_step", + "build_frontend_package_step", + "build_frontend_step", + "build_image", + "build_plugins_step", + "build_storybook_step", + "clone_enterprise_step", + "compile_build_cmd", + "copy_packages_for_docker_step", + "download_grabpl_step", + "e2e_tests_artifacts", + "e2e_tests_step", + "fetch_images_step", + "get_windows_steps", + "grafana_server_step", + "identify_runner_step", + "init_enterprise_step", + "memcached_integration_tests_step", + "mysql_integration_tests_step", + "package_step", + "postgres_integration_tests_step", + "publish_grafanacom_step", + "publish_image", + "publish_images_step", + "publish_linux_packages_step", + "redis_integration_tests_step", + "store_storybook_step", + "trigger_oss", + "upload_cdn_step", + "upload_packages_step", + "verify_gen_cue_step", + "wire_install_step", + "yarn_install_step", ) - load( - 'scripts/drone/utils/utils.star', - 'pipeline', - 'notify_pipeline', - 'failure_template', - 'drone_change_template', + "scripts/drone/services/services.star", + "integration_test_services", + "integration_test_services_volumes", ) - load( - 'scripts/drone/pipelines/test_frontend.star', - 'test_frontend', + "scripts/drone/utils/utils.star", + "pipeline", + "with_deps", ) - load( - 'scripts/drone/pipelines/test_backend.star', - 'test_backend', + "scripts/drone/pipelines/test_frontend.star", + "test_frontend", + "test_frontend_enterprise", ) +load( + "scripts/drone/pipelines/test_backend.star", + "test_backend", + "test_backend_enterprise", +) +load("scripts/drone/vault.star", "from_secret", "prerelease_bucket") -load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token', 'prerelease_bucket') +ver_mode = "release" +release_trigger = { + "event": {"exclude": ["promote"]}, + "ref": [ + "refs/tags/v*", + ], +} def store_npm_packages_step(): return { - 'name': 'store-npm-packages', - 'image': build_image, - 'depends_on': [ - 'build-frontend-packages', + "name": "store-npm-packages", + "image": build_image, + "depends_on": [ + "compile-build-cmd", + "build-frontend-packages", ], - 'environment': { - 'GCP_KEY': from_secret('gcp_key'), - 'PRERELEASE_BUCKET': from_secret(prerelease_bucket) + "environment": { + "GCP_KEY": from_secret("gcp_key"), + "PRERELEASE_BUCKET": from_secret(prerelease_bucket), }, - 'commands': [ - './bin/grabpl artifacts npm store --tag ${DRONE_TAG}' - ], + "commands": ["./bin/build artifacts npm store --tag ${DRONE_TAG}"], } def retrieve_npm_packages_step(): return { - 'name': 'retrieve-npm-packages', - 'image': publish_image, - 'depends_on': [ - 'yarn-install', + "name": "retrieve-npm-packages", + "image": publish_image, + "depends_on": [ + "compile-build-cmd", + "yarn-install", ], - 'failure': 'ignore', - 'environment': { - 'GCP_KEY': from_secret('gcp_key'), - 'PRERELEASE_BUCKET': from_secret(prerelease_bucket) + "failure": "ignore", + "environment": { + "GCP_KEY": from_secret("gcp_key"), + "PRERELEASE_BUCKET": from_secret(prerelease_bucket), }, - 'commands': [ - './bin/grabpl artifacts npm retrieve --tag v${TAG}' - ], + "commands": ["./bin/build artifacts npm retrieve --tag ${DRONE_TAG}"], } def release_npm_packages_step(): return { - 'name': 'release-npm-packages', - 'image': build_image, - 'depends_on': [ - 'retrieve-npm-packages', + "name": "release-npm-packages", + "image": build_image, + "depends_on": [ + "compile-build-cmd", + "retrieve-npm-packages", ], - 'failure': 'ignore', - 'environment': { - 'NPM_TOKEN': from_secret('npm_token'), + "failure": "ignore", + "environment": { + "NPM_TOKEN": from_secret("npm_token"), }, - 'commands': [ - './bin/grabpl artifacts npm release --tag v${TAG}' - ], + "commands": ["./bin/build artifacts npm release --tag ${DRONE_TAG}"], } -def get_oss_pipelines(trigger, ver_mode): - environment = {'EDITION': 'oss'} - edition = 'oss' - services = integration_test_services(edition=edition) +def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger): + """Generates all pipelines used for Grafana OSS. + + Args: + ver_mode: controls which steps are included in the pipeline. + Defaults to 'release'. + trigger: controls which events can trigger the pipeline execution. + Defaults to tag events for tags with a 'v' prefix. + + Returns: + List of Drone pipelines. + """ + + environment = {"EDITION": "oss"} + + services = integration_test_services(edition = "oss") volumes = integration_test_services_volumes() - package_steps = [] - publish_steps = [] - should_publish = ver_mode == 'release' - should_upload = should_publish or ver_mode in ('release-branch',) + init_steps = [ identify_runner_step(), download_grabpl_step(), - verify_gen_cue_step(edition), + verify_gen_cue_step(), wire_install_step(), yarn_install_step(), compile_build_cmd(), ] build_steps = [ - build_backend_step(edition=edition, ver_mode=ver_mode), - build_frontend_step(edition=edition, ver_mode=ver_mode), - build_frontend_package_step(edition=edition, ver_mode=ver_mode), - build_plugins_step(edition=edition, ver_mode=ver_mode), + build_backend_step(edition = "oss", ver_mode = ver_mode), + build_frontend_step(edition = "oss", ver_mode = ver_mode), + build_frontend_package_step(edition = "oss", ver_mode = ver_mode), + build_plugins_step(edition = "oss", ver_mode = ver_mode), + package_step(edition = "oss", ver_mode = ver_mode), + copy_packages_for_docker_step(), + build_docker_images_step(edition = "oss", publish = True), + build_docker_images_step( + edition = "oss", + publish = True, + ubuntu = True, + ), + grafana_server_step(edition = "oss"), + e2e_tests_step("dashboards-suite", tries = 3), + e2e_tests_step("smoke-tests-suite", tries = 3), + e2e_tests_step("panels-suite", tries = 3), + e2e_tests_step("various-suite", tries = 3), + e2e_tests_artifacts(), + build_storybook_step(ver_mode = ver_mode), ] + publish_steps = [] + + if ver_mode in ( + "release", + "release-branch", + ): + publish_steps.extend( + [ + upload_cdn_step(edition = "oss", ver_mode = ver_mode, trigger = trigger_oss), + upload_packages_step( + edition = "oss", + ver_mode = ver_mode, + trigger = trigger_oss, + ), + ], + ) + + if ver_mode in ("release",): + publish_steps.extend( + [ + store_storybook_step(ver_mode = ver_mode), + store_npm_packages_step(), + ], + ) + integration_test_steps = [ - postgres_integration_tests_step(edition=edition, ver_mode=ver_mode), - mysql_integration_tests_step(edition=edition, ver_mode=ver_mode), + postgres_integration_tests_step(), + mysql_integration_tests_step(), ] - # Insert remaining steps - build_steps.extend([ - package_step(edition=edition, ver_mode=ver_mode), - copy_packages_for_docker_step(), - build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=True), - build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=True), - grafana_server_step(edition=edition), - ]) - - if not disable_tests: - build_steps.extend([ - e2e_tests_step('dashboards-suite', edition=edition, tries=3), - e2e_tests_step('smoke-tests-suite', edition=edition, tries=3), - e2e_tests_step('panels-suite', edition=edition, tries=3), - e2e_tests_step('various-suite', edition=edition, tries=3), - e2e_tests_artifacts(edition=edition), - ]) - - build_storybook = build_storybook_step(edition=edition, ver_mode=ver_mode) - if build_storybook: - build_steps.append(build_storybook) - - if should_upload: - publish_steps.append(upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss)) - publish_steps.append(upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss)) - if should_publish: - publish_step = store_storybook_step(edition=edition, ver_mode=ver_mode) - store_npm_step = store_npm_packages_step() - if publish_step: - publish_steps.append(publish_step) - if store_npm_step: - publish_steps.append(store_npm_step) - windows_package_steps = get_windows_steps(edition=edition, ver_mode=ver_mode) - windows_pipeline = pipeline( - name='{}-oss-windows'.format(ver_mode), edition=edition, trigger=trigger, - steps=[identify_runner_step('windows')] + windows_package_steps, - platform='windows', depends_on=[ - 'oss-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode), - ], environment=environment, + name = "{}-oss-windows".format(ver_mode), + edition = "oss", + trigger = trigger, + steps = get_windows_steps(edition = "oss", ver_mode = ver_mode), + platform = "windows", + depends_on = [ + # 'oss-build-e2e-publish-{}'.format(ver_mode), + "{}-oss-build-e2e-publish".format(ver_mode), + "{}-oss-test-frontend".format(ver_mode), + "{}-oss-test-backend".format(ver_mode), + "{}-oss-integration-tests".format(ver_mode), + ], + environment = environment, ) + pipelines = [ pipeline( - name='{}-oss-build{}-publish'.format(ver_mode, get_e2e_suffix()), edition=edition, trigger=trigger, services=[], - steps=init_steps + build_steps + package_steps + publish_steps, - environment=environment, volumes=volumes, + name = "{}-oss-build-e2e-publish".format(ver_mode), + edition = "oss", + trigger = trigger, + services = [], + steps = init_steps + build_steps + publish_steps, + environment = environment, + volumes = volumes, ), + test_frontend(trigger, ver_mode), + test_backend(trigger, ver_mode), + pipeline( + name = "{}-oss-integration-tests".format(ver_mode), + edition = "oss", + trigger = trigger, + services = services, + steps = [ + download_grabpl_step(), + identify_runner_step(), + verify_gen_cue_step(), + wire_install_step(), + ] + + integration_test_steps, + environment = environment, + volumes = volumes, + ), + windows_pipeline, ] - if not disable_tests: - pipelines.extend([ - test_frontend(trigger, ver_mode), - test_backend(trigger, ver_mode), - pipeline( - name='{}-oss-integration-tests'.format(ver_mode), edition=edition, trigger=trigger, services=services, - steps=[download_grabpl_step(), identify_runner_step(), verify_gen_cue_step(edition), wire_install_step(), ] + integration_test_steps, - environment=environment, volumes=volumes, - ) - ]) - deps = { - 'depends_on': [ - '{}-oss-build{}-publish'.format(ver_mode, get_e2e_suffix()), - '{}-oss-test-frontend'.format(ver_mode), - '{}-oss-test-backend'.format(ver_mode), - '{}-oss-integration-tests'.format(ver_mode) - ] - } - windows_pipeline.update(deps) - pipelines.extend([windows_pipeline]) return pipelines -def get_enterprise_pipelines(trigger, ver_mode): - environment = {'EDITION': 'enterprise'} - edition = 'enterprise' - services = integration_test_services(edition=edition) +def enterprise_pipelines(ver_mode = ver_mode, trigger = release_trigger): + """Generates all pipelines used for Grafana Enterprise. + + Args: + ver_mode: controls which steps are included in the pipeline. + Defaults to 'release'. + trigger: controls which events can trigger the pipeline execution. + Defaults to tag events for tags with a 'v' prefix. + + Returns: + List of Drone pipelines. + """ + if ver_mode == "release": + source = "${DRONE_TAG}" + elif ver_mode == "release-branch": + source = "${DRONE_BRANCH}" + else: + source = "${DRONE_COMMIT}" + + environment = {"EDITION": "enterprise"} + + services = integration_test_services(edition = "enterprise") volumes = integration_test_services_volumes() - package_steps = [] - publish_steps = [] - should_publish = ver_mode == 'release' - should_upload = should_publish or ver_mode in ('release-branch',) - include_enterprise = edition == 'enterprise' - edition2 = 'enterprise2' + init_steps = [ download_grabpl_step(), identify_runner_step(), - clone_enterprise_step(ver_mode), + clone_enterprise_step(source = source), init_enterprise_step(ver_mode), - compile_build_cmd(edition), - ] + compile_build_cmd("enterprise"), + ] + with_deps( + [ + wire_install_step(), + yarn_install_step(), + verify_gen_cue_step(), + ], + [ + "init-enterprise", + ], + ) build_steps = [ - build_backend_step(edition=edition, ver_mode=ver_mode), - build_frontend_step(edition=edition, ver_mode=ver_mode), - build_frontend_package_step(edition=edition, ver_mode=ver_mode), - build_plugins_step(edition=edition, ver_mode=ver_mode), + build_backend_step(edition = "enterprise", ver_mode = ver_mode), + build_frontend_step(edition = "enterprise", ver_mode = ver_mode), + build_frontend_package_step(edition = "enterprise", ver_mode = ver_mode), + build_plugins_step(edition = "enterprise", ver_mode = ver_mode), + package_step( + edition = "enterprise", + ver_mode = ver_mode, + ), + copy_packages_for_docker_step(), + build_docker_images_step(edition = "enterprise", publish = True), + build_docker_images_step( + edition = "enterprise", + publish = True, + ubuntu = True, + ), + grafana_server_step(edition = "enterprise"), + e2e_tests_step("dashboards-suite", tries = 3), + e2e_tests_step("smoke-tests-suite", tries = 3), + e2e_tests_step("panels-suite", tries = 3), + e2e_tests_step("various-suite", tries = 3), + e2e_tests_artifacts(), ] + publish_steps = [] + + if ver_mode in ( + "release", + "release-branch", + ): + upload_packages_enterprise = upload_packages_step( + edition = "enterprise", + ver_mode = ver_mode, + trigger = trigger_oss, + ) + upload_packages_enterprise["depends_on"] = ["package"] + + publish_steps.extend( + [ + upload_cdn_step( + edition = "enterprise", + ver_mode = ver_mode, + trigger = trigger_oss, + ), + upload_packages_enterprise, + ], + ) + integration_test_steps = [ - postgres_integration_tests_step(edition=edition, ver_mode=ver_mode), - mysql_integration_tests_step(edition=edition, ver_mode=ver_mode), + postgres_integration_tests_step(), + mysql_integration_tests_step(), ] - if include_enterprise: - build_steps.extend([ - build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-amd64']), - ]) - - # Insert remaining steps - build_steps.extend([ - package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise), - copy_packages_for_docker_step(), - build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=True), - build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=True), - grafana_server_step(edition=edition), - ]) - - if not disable_tests: - build_steps.extend([ - e2e_tests_step('dashboards-suite', edition=edition, tries=3), - e2e_tests_step('smoke-tests-suite', edition=edition, tries=3), - e2e_tests_step('panels-suite', edition=edition, tries=3), - e2e_tests_step('various-suite', edition=edition, tries=3), - e2e_tests_artifacts(edition=edition), - ]) - - build_storybook = build_storybook_step(edition=edition, ver_mode=ver_mode) - if build_storybook: - build_steps.append(build_storybook) - - if should_upload: - publish_steps.extend([ - upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss), - upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss), - package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise, variants=['linux-amd64']), - upload_cdn_step(edition=edition2, ver_mode=ver_mode), - ]) - if should_publish: - publish_step = store_storybook_step(edition=edition, ver_mode=ver_mode) - store_npm_step = store_npm_packages_step() - if publish_step: - publish_steps.append(publish_step) - if store_npm_step: - publish_steps.append(store_npm_step) - windows_package_steps = get_windows_steps(edition=edition, ver_mode=ver_mode) - - if should_upload: - step = upload_packages_step(edition=edition2, ver_mode=ver_mode) - if step: - publish_steps.append(step) - - deps_on_clone_enterprise_step = { - 'depends_on': [ - 'init-enterprise', - ] - } - - for step in [wire_install_step(), yarn_install_step(edition), verify_gen_cue_step(edition)]: - step.update(deps_on_clone_enterprise_step) - init_steps.extend([step]) - windows_pipeline = pipeline( - name='{}-enterprise-windows'.format(ver_mode), edition=edition, trigger=trigger, - steps=[identify_runner_step('windows')] + windows_package_steps, - platform='windows', depends_on=[ - 'enterprise-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode), - ], environment=environment, + name = "{}-enterprise-windows".format(ver_mode), + edition = "enterprise", + trigger = trigger, + steps = get_windows_steps(edition = "enterprise", ver_mode = ver_mode), + platform = "windows", + depends_on = [ + # 'enterprise-build-e2e-publish-{}'.format(ver_mode), + "{}-enterprise-build-e2e-publish".format(ver_mode), + "{}-enterprise-test-frontend".format(ver_mode), + "{}-enterprise-test-backend".format(ver_mode), + "{}-enterprise-integration-tests".format(ver_mode), + ], + environment = environment, ) + pipelines = [ pipeline( - name='{}-enterprise-build{}-publish'.format(ver_mode, get_e2e_suffix()), edition=edition, trigger=trigger, services=[], - steps=init_steps + build_steps + package_steps + publish_steps, environment=environment, - volumes=volumes, + name = "{}-enterprise-build-e2e-publish".format(ver_mode), + edition = "enterprise", + trigger = trigger, + services = [], + steps = init_steps + build_steps + publish_steps, + environment = environment, + volumes = volumes, + ), + test_frontend_enterprise(trigger, ver_mode, source = source), + test_backend_enterprise(trigger, ver_mode, source = source), + pipeline( + name = "{}-enterprise-integration-tests".format(ver_mode), + edition = "enterprise", + trigger = trigger, + services = services, + steps = [ + download_grabpl_step(), + identify_runner_step(), + clone_enterprise_step(source = source), + init_enterprise_step(ver_mode), + ] + + with_deps( + [ + verify_gen_cue_step(), + ], + [ + "init-enterprise", + ], + ) + + [ + wire_install_step(), + ] + + integration_test_steps + + [ + redis_integration_tests_step(), + memcached_integration_tests_step(), + ], + environment = environment, + volumes = volumes, + ), + windows_pipeline, + ] + + return pipelines + +def enterprise2_pipelines(prefix = "", ver_mode = ver_mode, trigger = release_trigger): + """Generate the next generation of pipelines for Grafana Enterprise. + + Args: + prefix: a prefix for the pipeline name used to differentiate multiple instances of + the same pipeline. + Defaults to ''. + ver_mode: controls which steps are included in the pipeline. + Defaults to 'release'. + trigger: controls which events can trigger the pipeline execution. + Defaults to tag events for tags with a 'v' prefix. + + Returns: + List of Drone pipelines. + """ + if ver_mode == "release": + source = "${DRONE_TAG}" + if prefix == "custom-": + source = "${DRONE_TARGET_BRANCH}" + elif ver_mode == "release-branch": + source = "${DRONE_BRANCH}" + else: + source = "${DRONE_COMMIT}" + + environment = { + "EDITION": "enterprise2", + } + + volumes = integration_test_services_volumes() + + init_steps = [ + download_grabpl_step(), + identify_runner_step(), + clone_enterprise_step(source = source), + init_enterprise_step(ver_mode), + compile_build_cmd("enterprise"), + ] + with_deps( + [ + wire_install_step(), + yarn_install_step(), + verify_gen_cue_step(), + ], + [ + "init-enterprise", + ], + ) + + build_steps = [ + build_frontend_step(edition = "enterprise", ver_mode = ver_mode), + build_frontend_package_step(edition = "enterprise", ver_mode = ver_mode), + build_plugins_step(edition = "enterprise", ver_mode = ver_mode), + build_backend_step( + edition = "enterprise2", + ver_mode = ver_mode, + variants = ["linux-amd64"], ), ] - if not disable_tests: - pipelines.extend([ - test_frontend(trigger, ver_mode, edition), - test_backend(trigger, ver_mode, edition), - test_backend(trigger, ver_mode, edition2), - pipeline( - name='{}-enterprise-integration-tests'.format(ver_mode), edition=edition, trigger=trigger, services=services, - steps=[download_grabpl_step(), identify_runner_step(), clone_enterprise_step(ver_mode), init_enterprise_step(ver_mode), verify_gen_cue_step(edition), wire_install_step()] + integration_test_steps + [redis_integration_tests_step(), memcached_integration_tests_step()], - environment=environment, volumes=volumes, - ), - ]) - deps = { - 'depends_on': [ - '{}-enterprise-build{}-publish'.format(ver_mode, get_e2e_suffix()), - '{}-enterprise-test-frontend'.format(ver_mode), - '{}-enterprise-test-backend'.format(ver_mode), - '{}-enterprise-integration-tests'.format(ver_mode) - ] - } - windows_pipeline.update(deps) - pipelines.extend([windows_pipeline]) + fetch_images = fetch_images_step("enterprise2") + fetch_images.update( + {"depends_on": ["build-docker-images", "build-docker-images-ubuntu"]}, + ) + + upload_cdn = upload_cdn_step(edition = "enterprise2", ver_mode = ver_mode) + upload_cdn["environment"].update( + {"ENTERPRISE2_CDN_PATH": from_secret("enterprise2-cdn-path")}, + ) + + build_steps.extend( + [ + package_step( + edition = "enterprise2", + ver_mode = ver_mode, + ), + upload_cdn, + copy_packages_for_docker_step(edition = "enterprise2"), + build_docker_images_step( + edition = "enterprise2", + publish = True, + ), + build_docker_images_step( + edition = "enterprise2", + publish = True, + ubuntu = True, + ), + fetch_images, + publish_images_step( + "enterprise2", + "release", + mode = "enterprise2", + docker_repo = "${{DOCKER_ENTERPRISE2_REPO}}", + ), + ], + ) + + publish_steps = [] + + if ver_mode in ( + "release", + "release-branch", + ): + step = upload_packages_step(edition = "enterprise2", ver_mode = ver_mode) + step["depends_on"] = ["package-enterprise2"] + + publish_steps.append(step) + + pipelines = [ + pipeline( + name = "{}{}-enterprise2-build-e2e-publish".format(prefix, ver_mode), + edition = "enterprise", + trigger = trigger, + services = [], + steps = init_steps + build_steps + publish_steps, + volumes = volumes, + environment = environment, + ), + ] return pipelines def publish_artifacts_step(mode): - security = '' - if mode == 'security': - security = '--security ' + security = "" + if mode == "security": + security = "--security " return { - 'name': 'publish-artifacts', - 'image': publish_image, - 'environment': { - 'GCP_KEY': from_secret('gcp_key'), - 'PRERELEASE_BUCKET': from_secret('prerelease_bucket'), + "name": "publish-artifacts", + "image": publish_image, + "environment": { + "GCP_KEY": from_secret("gcp_key"), + "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), + "ENTERPRISE2_SECURITY_PREFIX": from_secret("enterprise2_security_prefix"), + "SECURITY_DEST_BUCKET": from_secret("security_dest_bucket"), + "STATIC_ASSET_EDITIONS": from_secret("static_asset_editions"), }, - 'commands': ['./bin/grabpl artifacts publish {}--tag ${{TAG}} --src-bucket $${{PRERELEASE_BUCKET}}'.format(security)], - 'depends_on': ['grabpl'], + "commands": [ + "./bin/build artifacts publish {}--tag $${{DRONE_TAG}} --src-bucket $${{PRERELEASE_BUCKET}}".format( + security, + ), + ], + "depends_on": ["compile-build-cmd"], } def publish_artifacts_pipelines(mode): trigger = { - 'event': ['promote'], - 'target': [mode], + "event": ["promote"], + "target": [mode], } steps = [ - download_grabpl_step(), + compile_build_cmd(), publish_artifacts_step(mode), ] - return [pipeline( - name='publish-artifacts-{}'.format(mode), trigger=trigger, steps=steps, edition="all" - )] + return [ + pipeline( + name = "publish-artifacts-{}".format(mode), + trigger = trigger, + steps = steps, + edition = "all", + environment = {"EDITION": "all"}, + ), + ] def publish_packages_pipeline(): + """Generates pipelines used for publishing packages for both OSS and enterprise. + + Returns: + List of Drone pipelines. One for each of OSS and enterprise packages. + """ + trigger = { - 'event': ['promote'], - 'target': ['public'], + "event": ["promote"], + "target": ["public"], } oss_steps = [ download_grabpl_step(), compile_build_cmd(), - publish_packages_step(edition='oss', ver_mode='release'), - publish_grafanacom_step(edition='oss', ver_mode='release'), - publish_linux_packages_step(edition='oss', package_manager='deb'), - publish_linux_packages_step(edition='oss', package_manager='rpm'), + publish_linux_packages_step(edition = "oss", package_manager = "deb"), + publish_linux_packages_step(edition = "oss", package_manager = "rpm"), + publish_grafanacom_step(edition = "oss", ver_mode = "release"), ] enterprise_steps = [ download_grabpl_step(), compile_build_cmd(), - publish_packages_step(edition='enterprise', ver_mode='release'), - publish_grafanacom_step(edition='enterprise', ver_mode='release'), - publish_linux_packages_step(edition='enterprise', package_manager='deb'), - publish_linux_packages_step(edition='enterprise', package_manager='rpm'), + publish_linux_packages_step(edition = "enterprise", package_manager = "deb"), + publish_linux_packages_step(edition = "enterprise", package_manager = "rpm"), + publish_grafanacom_step(edition = "enterprise", ver_mode = "release"), ] deps = [ - 'publish-artifacts-public', - 'publish-docker-oss-public', - 'publish-docker-enterprise-public' + "publish-artifacts-public", + "publish-docker-oss-public", + "publish-docker-enterprise-public", ] - return [pipeline( - name='publish-packages-oss', trigger=trigger, steps=oss_steps, edition="all", depends_on=deps - ), pipeline( - name='publish-packages-enterprise', trigger=trigger, steps=enterprise_steps, edition="all", depends_on=deps - )] + return [ + pipeline( + name = "publish-packages-oss", + trigger = trigger, + steps = oss_steps, + edition = "all", + depends_on = deps, + environment = {"EDITION": "oss"}, + ), + pipeline( + name = "publish-packages-enterprise", + trigger = trigger, + steps = enterprise_steps, + edition = "all", + depends_on = deps, + environment = {"EDITION": "enterprise"}, + ), + ] -def publish_npm_pipelines(mode): +def publish_npm_pipelines(): trigger = { - 'event': ['promote'], - 'target': [mode], + "event": ["promote"], + "target": ["public"], } steps = [ - download_grabpl_step(), + compile_build_cmd(), yarn_install_step(), retrieve_npm_packages_step(), - release_npm_packages_step() + release_npm_packages_step(), ] - return [pipeline( - name='publish-npm-packages-{}'.format(mode), trigger=trigger, steps = steps, edition="all" - )] + return [ + pipeline( + name = "publish-npm-packages-public", + trigger = trigger, + steps = steps, + edition = "all", + environment = {"EDITION": "all"}, + ), + ] def artifacts_page_pipeline(): trigger = { - 'event': ['promote'], - 'target': 'security', + "event": ["promote"], + "target": "security", } - return [pipeline(name='publish-artifacts-page', trigger=trigger, steps = [download_grabpl_step(), artifacts_page_step()], edition="all")] - -def release_pipelines(ver_mode='release', trigger=None): - # 'enterprise' edition services contain both OSS and enterprise services - if not trigger: - trigger = { - 'event': { - 'exclude': [ - 'promote' - ] - }, - 'ref': ['refs/tags/v*',], - 'repo': { - 'exclude': ['grafana/grafana'], - }, - } - - # The release pipelines include also enterprise ones, so both editions are built for a release. - # We could also solve this by triggering a downstream build for the enterprise repo, but by including enterprise - # in OSS release builds, we simplify the UX for the release engineer. - oss_pipelines = get_oss_pipelines(ver_mode=ver_mode, trigger=trigger) - enterprise_pipelines = get_enterprise_pipelines(ver_mode=ver_mode, trigger=trigger) - - pipelines = oss_pipelines + enterprise_pipelines - - return pipelines - -def get_e2e_suffix(): - if not disable_tests: - return '-e2e' - return '' + return [ + pipeline( + name = "publish-artifacts-page", + trigger = trigger, + steps = [ + download_grabpl_step(), + clone_enterprise_step(source = "${DRONE_TAG}"), + init_enterprise_step("release"), + compile_build_cmd("enterprise"), + artifacts_page_step(), + ], + edition = "all", + environment = {"EDITION": "enterprise"}, + ), + ] diff --git a/scripts/drone/pipelines/aws_marketplace.star b/scripts/drone/pipelines/aws_marketplace.star new file mode 100644 index 00000000000..bbcc5b16bd0 --- /dev/null +++ b/scripts/drone/pipelines/aws_marketplace.star @@ -0,0 +1,43 @@ +""" +This module contains steps and pipelines publishing to AWS Marketplace. +""" + +load( + "scripts/drone/steps/lib.star", + "compile_build_cmd", + "fetch_images_step", + "publish_image", +) +load("scripts/drone/vault.star", "from_secret") +load( + "scripts/drone/utils/utils.star", + "pipeline", +) + +def publish_aws_marketplace_step(): + return { + "name": "publish-aws-marketplace", + "image": publish_image, + "commands": ["./bin/build publish aws --image grafana/grafana-enterprise --repo grafana-labs/grafanaenterprise --product 422b46fb-bea6-4f27-8bcc-832117bd627e"], + "depends_on": ["fetch-images-enterprise"], + "environment": { + "AWS_REGION": from_secret("aws_region"), + "AWS_ACCESS_KEY_ID": from_secret("aws_access_key_id"), + "AWS_SECRET_ACCESS_KEY": from_secret("aws_secret_access_key"), + }, + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + } + +def publish_aws_marketplace_pipeline(mode): + trigger = { + "event": ["promote"], + "target": [mode], + } + return [pipeline( + name = "publish-aws-marketplace-{}".format(mode), + trigger = trigger, + steps = [compile_build_cmd(), fetch_images_step("enterprise"), publish_aws_marketplace_step()], + edition = "", + depends_on = ["publish-docker-enterprise-public"], + environment = {"EDITION": "enterprise2"}, + )] diff --git a/scripts/drone/pipelines/build.star b/scripts/drone/pipelines/build.star index 3d7b46a5015..8b37a176a2e 100644 --- a/scripts/drone/pipelines/build.star +++ b/scripts/drone/pipelines/build.star @@ -1,107 +1,160 @@ -load( - 'scripts/drone/steps/lib.star', - 'download_grabpl_step', - 'build_image', - 'identify_runner_step', - 'wire_install_step', - 'yarn_install_step', - 'build_backend_step', - 'build_frontend_step', - 'build_frontend_package_step', - 'build_plugins_step', - 'package_step', - 'grafana_server_step', - 'e2e_tests_step', - 'e2e_tests_artifacts', - 'build_storybook_step', - 'copy_packages_for_docker_step', - 'build_docker_images_step', - 'publish_images_step', - 'postgres_integration_tests_step', - 'mysql_integration_tests_step', - 'redis_integration_tests_step', - 'memcached_integration_tests_step', - 'get_windows_steps', - 'benchmark_ldap_step', - 'enterprise_downstream_step', - 'frontend_metrics_step', - 'store_storybook_step', - 'release_canary_npm_packages_step', - 'upload_packages_step', - 'upload_cdn_step', - 'verify_gen_cue_step', - 'test_a11y_frontend_step', - 'trigger_oss', - 'betterer_frontend_step', - 'trigger_test_release', - 'compile_build_cmd', -) +"""This module contains the comprehensive build pipeline.""" load( - 'scripts/drone/utils/utils.star', - 'pipeline', + "scripts/drone/steps/lib.star", + "build_backend_step", + "build_docker_images_step", + "build_frontend_package_step", + "build_frontend_step", + "build_plugins_step", + "build_storybook_step", + "cloud_plugins_e2e_tests_step", + "compile_build_cmd", + "copy_packages_for_docker_step", + "download_grabpl_step", + "e2e_tests_artifacts", + "e2e_tests_step", + "enterprise_downstream_step", + "frontend_metrics_step", + "grafana_server_step", + "identify_runner_step", + "package_step", + "publish_images_step", + "release_canary_npm_packages_step", + "store_storybook_step", + "test_a11y_frontend_step", + "trigger_oss", + "trigger_test_release", + "upload_cdn_step", + "upload_packages_step", + "verify_gen_cue_step", + "wire_install_step", + "yarn_install_step", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", ) -def build_e2e(trigger, ver_mode, edition): - variants = ['linux-amd64', 'linux-amd64-musl', 'darwin-amd64', 'windows-amd64',] +# @unused +def build_e2e(trigger, ver_mode): + """Perform e2e building, testing, and publishing. + + Args: + trigger: controls which events can trigger the pipeline execution. + ver_mode: used in the naming of the pipeline. + + Returns: + Drone pipeline. + """ + + edition = "oss" + environment = {"EDITION": edition} init_steps = [ identify_runner_step(), download_grabpl_step(), compile_build_cmd(), - verify_gen_cue_step(edition="oss"), + verify_gen_cue_step(), wire_install_step(), yarn_install_step(), ] + build_steps = [] - if ver_mode == 'main': - build_steps.extend([trigger_test_release()]) - if ver_mode == 'pr': - build_steps.extend([enterprise_downstream_step(edition=edition, ver_mode=ver_mode)]) - build_steps.extend([ - build_backend_step(edition=edition, ver_mode=ver_mode), - build_frontend_step(edition=edition, ver_mode=ver_mode), - build_frontend_package_step(edition=edition, ver_mode=ver_mode), - build_plugins_step(edition=edition, ver_mode=ver_mode), - ]) - if ver_mode == 'main': - build_steps.extend([package_step(edition=edition, ver_mode=ver_mode)]) - elif ver_mode == 'pr': - build_steps.extend([package_step(edition=edition, ver_mode=ver_mode, variants=variants)]) - build_steps.extend([ - grafana_server_step(edition=edition), - e2e_tests_step('dashboards-suite', edition=edition), - e2e_tests_step('smoke-tests-suite', edition=edition), - e2e_tests_step('panels-suite', edition=edition), - e2e_tests_step('various-suite', edition=edition), - e2e_tests_artifacts(edition=edition), - build_storybook_step(edition=edition, ver_mode=ver_mode), - copy_packages_for_docker_step(), - test_a11y_frontend_step(ver_mode=ver_mode, edition=edition), - ]) - if ver_mode == 'main': - build_steps.extend([ - store_storybook_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss), - frontend_metrics_step(edition=edition, trigger=trigger_oss) - ]) + if ver_mode == "pr": + build_steps.extend( + [ + trigger_test_release(), + enterprise_downstream_step(ver_mode = ver_mode), + ], + ) - if ver_mode == 'main': - build_steps.extend([ - build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=False), - build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=False), - publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana/grafana', trigger=trigger_oss), - publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana/grafana-oss', trigger=trigger_oss), - release_canary_npm_packages_step(edition, trigger=trigger_oss), - upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss), - upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss) - ]) - elif ver_mode == 'pr': - build_steps.extend([build_docker_images_step(edition=edition, ver_mode=ver_mode, archs=['amd64', ])]) + build_steps.extend( + [ + build_backend_step(edition = edition, ver_mode = ver_mode), + build_frontend_step(edition = edition, ver_mode = ver_mode), + build_frontend_package_step(edition = edition, ver_mode = ver_mode), + build_plugins_step(edition = edition, ver_mode = ver_mode), + package_step(edition = edition, ver_mode = ver_mode), + grafana_server_step(edition = edition), + e2e_tests_step("dashboards-suite"), + e2e_tests_step("smoke-tests-suite"), + e2e_tests_step("panels-suite"), + e2e_tests_step("various-suite"), + cloud_plugins_e2e_tests_step( + "cloud-plugins-suite", + cloud = "azure", + trigger = trigger_oss, + ), + e2e_tests_artifacts(), + build_storybook_step(ver_mode = ver_mode), + copy_packages_for_docker_step(), + test_a11y_frontend_step(ver_mode = ver_mode), + ], + ) - publish_suffix = '' - if ver_mode == 'main': - publish_suffix = '-publish' + if ver_mode == "main": + build_steps.extend( + [ + store_storybook_step(trigger = trigger_oss, ver_mode = ver_mode), + frontend_metrics_step(trigger = trigger_oss), + build_docker_images_step( + edition = edition, + publish = False, + ), + build_docker_images_step( + edition = edition, + publish = False, + ubuntu = True, + ), + publish_images_step( + docker_repo = "grafana", + edition = edition, + mode = "", + trigger = trigger_oss, + ver_mode = ver_mode, + ), + publish_images_step( + docker_repo = "grafana-oss", + edition = edition, + mode = "", + trigger = trigger_oss, + ver_mode = ver_mode, + ), + release_canary_npm_packages_step(trigger = trigger_oss), + upload_packages_step( + edition = edition, + trigger = trigger_oss, + ver_mode = ver_mode, + ), + upload_cdn_step( + edition = edition, + trigger = trigger_oss, + ver_mode = ver_mode, + ), + ], + ) + elif ver_mode == "pr": + build_steps.extend( + [ + build_docker_images_step( + archs = [ + "amd64", + ], + edition = edition, + ), + ], + ) + + publish_suffix = "" + if ver_mode == "main": + publish_suffix = "-publish" return pipeline( - name='{}-build-e2e{}'.format(ver_mode, publish_suffix), edition="oss", trigger=trigger, services=[], steps=init_steps + build_steps, + name = "{}-build-e2e{}".format(ver_mode, publish_suffix), + edition = "oss", + environment = environment, + services = [], + steps = init_steps + build_steps, + trigger = trigger, ) diff --git a/scripts/drone/pipelines/docs.star b/scripts/drone/pipelines/docs.star index eefecd599e7..cba222fd2ff 100644 --- a/scripts/drone/pipelines/docs.star +++ b/scripts/drone/pipelines/docs.star @@ -1,37 +1,32 @@ -load( - 'scripts/drone/steps/lib.star', - 'build_image', - 'yarn_install_step', - 'identify_runner_step', - 'download_grabpl_step', - 'lint_frontend_step', - 'codespell_step', - 'test_frontend_step', - 'build_storybook_step', - 'build_docs_website_step', -) +""" +This module returns all the pipelines used in the event of documentation changes along with supporting functions. +""" load( - 'scripts/drone/services/services.star', - 'integration_test_services', - 'ldap_service', + "scripts/drone/steps/lib.star", + "build_docs_website_step", + "build_image", + "codespell_step", + "download_grabpl_step", + "identify_runner_step", + "yarn_install_step", ) - load( - 'scripts/drone/utils/utils.star', - 'pipeline', + "scripts/drone/utils/utils.star", + "pipeline", ) docs_paths = { - 'include': [ - '*.md', - 'docs/**', - 'packages/**/*.md', - 'latest.json', + "include": [ + "*.md", + "docs/**", + "packages/**/*.md", + "latest.json", ], } -def docs_pipelines(edition, ver_mode, trigger): +def docs_pipelines(ver_mode, trigger): + environment = {"EDITION": "oss"} steps = [ download_grabpl_step(), identify_runner_step(), @@ -42,38 +37,42 @@ def docs_pipelines(edition, ver_mode, trigger): ] return pipeline( - name='{}-docs'.format(ver_mode), edition=edition, trigger=trigger, services=[], steps=steps, + name = "{}-docs".format(ver_mode), + edition = "oss", + trigger = trigger, + services = [], + steps = steps, + environment = environment, ) def lint_docs(): return { - 'name': 'lint-docs', - 'image': build_image, - 'depends_on': [ - 'yarn-install', + "name": "lint-docs", + "image": build_image, + "depends_on": [ + "yarn-install", ], - 'environment': { - 'NODE_OPTIONS': '--max_old_space_size=8192', + "environment": { + "NODE_OPTIONS": "--max_old_space_size=8192", }, - 'commands': [ - 'yarn run prettier:checkDocs', + "commands": [ + "yarn run prettier:checkDocs", ], } - def trigger_docs_main(): return { - 'branch': 'main', - 'event': [ - 'push', + "branch": "main", + "event": [ + "push", ], - 'paths': docs_paths, + "paths": docs_paths, } def trigger_docs_pr(): return { - 'event': [ - 'pull_request', + "event": [ + "pull_request", ], - 'paths': docs_paths, + "paths": docs_paths, } diff --git a/scripts/drone/pipelines/github.star b/scripts/drone/pipelines/github.star new file mode 100644 index 00000000000..d3ccb6b30c9 --- /dev/null +++ b/scripts/drone/pipelines/github.star @@ -0,0 +1,40 @@ +""" +This module contains steps and pipelines relating to GitHub. +""" + +load( + "scripts/drone/steps/lib.star", + "compile_build_cmd", + "fetch_images_step", + "publish_image", +) +load("scripts/drone/vault.star", "from_secret") +load( + "scripts/drone/utils/utils.star", + "pipeline", +) + +def publish_github_step(): + return { + "name": "publish-github", + "image": publish_image, + "commands": ["./bin/build publish github --repo $${GH_REGISTRY} --create"], + "depends_on": ["fetch-images-enterprise2"], + "environment": { + "GH_TOKEN": from_secret("github_token"), + "GH_REGISTRY": from_secret("gh_registry"), + }, + } + +def publish_github_pipeline(mode): + trigger = { + "event": ["promote"], + "target": [mode], + } + return [pipeline( + name = "publish-github-{}".format(mode), + trigger = trigger, + steps = [compile_build_cmd(), fetch_images_step("enterprise2"), publish_github_step()], + edition = "", + environment = {"EDITION": "enterprise2"}, + )] diff --git a/scripts/drone/pipelines/integration_tests.star b/scripts/drone/pipelines/integration_tests.star index ee4d77ef9f1..01e2c45da9a 100644 --- a/scripts/drone/pipelines/integration_tests.star +++ b/scripts/drone/pipelines/integration_tests.star @@ -1,42 +1,75 @@ -load( - 'scripts/drone/steps/lib.star', - 'identify_runner_step', - 'download_grabpl_step', - 'verify_gen_cue_step', - 'wire_install_step', - 'postgres_integration_tests_step', - 'mysql_integration_tests_step', - 'compile_build_cmd', -) +""" +This module returns the pipeline used for integration tests. +""" load( - 'scripts/drone/services/services.star', - 'integration_test_services', - 'integration_test_services_volumes', - 'ldap_service', + "scripts/drone/steps/lib.star", + "compile_build_cmd", + "download_grabpl_step", + "enterprise_setup_step", + "identify_runner_step", + "mysql_integration_tests_step", + "postgres_integration_tests_step", + "verify_gen_cue_step", + "wire_install_step", ) - load( - 'scripts/drone/utils/utils.star', - 'pipeline', + "scripts/drone/services/services.star", + "integration_test_services", + "integration_test_services_volumes", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", ) -def integration_tests(trigger, ver_mode, edition): - services = integration_test_services(edition) +def integration_tests(trigger, prefix, ver_mode = "pr"): + """Generate a pipeline for integration tests. + + Args: + trigger: controls which events can trigger the pipeline execution. + prefix: used in the naming of the pipeline. + ver_mode: defines the event / origin of this build. In this function, if it is set to pr, then it will attempt to clone grafana-enterprise. Otherwise it has no effect. + + Returns: + Drone pipeline. + """ + environment = {"EDITION": "oss"} + + services = integration_test_services(edition = "oss") volumes = integration_test_services_volumes() - init_steps = [ + + init_steps = [] + + verify_step = verify_gen_cue_step() + + if ver_mode == "pr": + # In pull requests, attempt to clone grafana enterprise. + init_steps.append(enterprise_setup_step()) + + # Ensure that verif_gen_cue happens after we clone enterprise + # At the time of writing this, very_gen_cue is depended on by the wire step which is what everything else depends on. + verify_step["depends_on"].append("clone-enterprise") + + init_steps += [ download_grabpl_step(), compile_build_cmd(), identify_runner_step(), - verify_gen_cue_step(edition="oss"), + verify_step, wire_install_step(), ] + test_steps = [ - postgres_integration_tests_step(edition=edition, ver_mode=ver_mode), - mysql_integration_tests_step(edition=edition, ver_mode=ver_mode), + postgres_integration_tests_step(), + mysql_integration_tests_step(), ] return pipeline( - name='{}-integration-tests'.format(ver_mode), edition="oss", trigger=trigger, services=services, steps=init_steps + test_steps, - volumes=volumes + name = "{}-integration-tests".format(prefix), + edition = "oss", + trigger = trigger, + environment = environment, + services = services, + volumes = volumes, + steps = init_steps + test_steps, ) diff --git a/scripts/drone/pipelines/lint_backend.star b/scripts/drone/pipelines/lint_backend.star index f127f655db4..f2605c44598 100644 --- a/scripts/drone/pipelines/lint_backend.star +++ b/scripts/drone/pipelines/lint_backend.star @@ -1,31 +1,60 @@ -load( - 'scripts/drone/steps/lib.star', - 'identify_runner_step', - 'wire_install_step', - 'lint_backend_step', - 'lint_drone_step', - 'compile_build_cmd', -) +""" +This module returns the pipeline used for linting backend code. +""" load( - 'scripts/drone/utils/utils.star', - 'pipeline', + "scripts/drone/steps/lib.star", + "compile_build_cmd", + "enterprise_setup_step", + "identify_runner_step", + "lint_backend_step", + "lint_drone_step", + "wire_install_step", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", ) def lint_backend_pipeline(trigger, ver_mode): + """Generates the pipelines used linting backend code. + + Args: + trigger: controls which events can trigger the pipeline execution. + ver_mode: used in the naming of the pipeline. + + Returns: + Drone pipeline. + """ + environment = {"EDITION": "oss"} + wire_step = wire_install_step() - wire_step.update({ 'depends_on': [] }) + wire_step.update({"depends_on": []}) + init_steps = [ identify_runner_step(), compile_build_cmd(), - wire_step, ] + + if ver_mode == "pr": + # In pull requests, attempt to clone grafana enterprise. + init_steps.append(enterprise_setup_step()) + wire_step["depends_on"].append("clone-enterprise") + + init_steps.append(wire_step) + test_steps = [ - lint_backend_step(edition="oss"), + lint_backend_step(), ] - if ver_mode == 'main': - test_steps.extend([lint_drone_step()]) + + if ver_mode == "main": + test_steps.append(lint_drone_step()) return pipeline( - name='{}-lint-backend'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps, + name = "{}-lint-backend".format(ver_mode), + edition = "oss", + trigger = trigger, + services = [], + steps = init_steps + test_steps, + environment = environment, ) diff --git a/scripts/drone/pipelines/lint_frontend.star b/scripts/drone/pipelines/lint_frontend.star index f40621c1d9a..9ef74c2ee74 100644 --- a/scripts/drone/pipelines/lint_frontend.star +++ b/scripts/drone/pipelines/lint_frontend.star @@ -1,25 +1,54 @@ -load( - 'scripts/drone/steps/lib.star', - 'identify_runner_step', - 'yarn_install_step', - 'lint_frontend_step', -) +""" +This module returns the pipeline used for linting frontend code. +""" load( - 'scripts/drone/utils/utils.star', - 'pipeline', + "scripts/drone/steps/lib.star", + "enterprise_setup_step", + "identify_runner_step", + "lint_frontend_step", + "yarn_install_step", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", ) def lint_frontend_pipeline(trigger, ver_mode): - yarn_step = yarn_install_step() - yarn_step.update({ 'depends_on': [] }) - init_steps = [ + """Generates the pipelines used linting frontend code. + + Args: + trigger: controls which events can trigger the pipeline execution. + ver_mode: used in the naming of the pipeline. + + Returns: + Drone pipeline. + """ + environment = {"EDITION": "oss"} + + init_steps = [] + lint_step = lint_frontend_step() + + if ver_mode == "pr": + # In pull requests, attempt to clone grafana enterprise. + init_steps = [enterprise_setup_step()] + # Ensure the lint step happens after the clone-enterprise step + + lint_step["depends_on"].append("clone-enterprise") + + init_steps += [ identify_runner_step(), - yarn_step, + yarn_install_step(), ] test_steps = [ - lint_frontend_step(), + lint_step, ] + return pipeline( - name='{}-lint-frontend'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps, + name = "{}-lint-frontend".format(ver_mode), + edition = "oss", + trigger = trigger, + services = [], + steps = init_steps + test_steps, + environment = environment, ) diff --git a/scripts/drone/pipelines/publish.star b/scripts/drone/pipelines/publish.star deleted file mode 100644 index 9d80d4a55ae..00000000000 --- a/scripts/drone/pipelines/publish.star +++ /dev/null @@ -1,25 +0,0 @@ -load( - 'scripts/drone/steps/lib.star', - 'identify_runner_step', - 'download_grabpl_step', - 'publish_packages_step', - 'compile_build_cmd', -) - -load( - 'scripts/drone/utils/utils.star', - 'pipeline', -) - -def publish(trigger, ver_mode, edition): - steps = [ - download_grabpl_step(), - compile_build_cmd(), - identify_runner_step(), - publish_packages_step(edition=edition, ver_mode=ver_mode), - ] - return pipeline( - name='main-publish', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']), - steps=steps, - depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', ], - ) diff --git a/scripts/drone/pipelines/publish_images.star b/scripts/drone/pipelines/publish_images.star index 5365880d169..18c94dcbca3 100644 --- a/scripts/drone/pipelines/publish_images.star +++ b/scripts/drone/pipelines/publish_images.star @@ -1,50 +1,97 @@ -load( - 'scripts/drone/steps/lib.star', - 'download_grabpl_step', - 'publish_images_step', - 'compile_build_cmd', - 'fetch_images_step', -) +""" +This module returns the pipeline used for publishing Docker images and its steps. +""" load( - 'scripts/drone/utils/utils.star', - 'pipeline', + "scripts/drone/steps/lib.star", + "compile_build_cmd", + "download_grabpl_step", + "fetch_images_step", + "identify_runner_step", + "publish_images_step", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", ) - def publish_image_steps(edition, mode, docker_repo): - additional_docker_repo = "" - if edition == 'oss': - additional_docker_repo='grafana/grafana-oss' + """Generates the steps used for publising Docker images using grabpl. + + Args: + edition: controls which version of an image is fetched in the case of a release. + It also controls which publishing implementation is used. + If edition == 'oss', it additionally publishes the grafana/grafana-oss repository. + mode: uses to control the publishing of security images when mode == 'security'. + docker_repo: the Docker image name. + It is combined with the 'grafana/' library prefix. + + Returns: + List of Drone steps. + """ steps = [ + identify_runner_step(), download_grabpl_step(), compile_build_cmd(), fetch_images_step(edition), - publish_images_step(edition, 'release', mode, docker_repo), + publish_images_step(edition, "release", mode, docker_repo), ] - if additional_docker_repo != "": - steps.extend([publish_images_step(edition, 'release', mode, additional_docker_repo)]) + + if edition == "oss": + steps.append( + publish_images_step(edition, "release", mode, "grafana-oss"), + ) return steps def publish_image_pipelines_public(): - mode='public' + """Generates the pipeline used for publising public Docker images. + + Returns: + Drone pipeline + """ + mode = "public" trigger = { - 'event': ['promote'], - 'target': [mode], + "event": ["promote"], + "target": [mode], } - return [pipeline( - name='publish-docker-oss-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='oss', mode=mode, docker_repo='grafana/grafana'), edition="" - ), pipeline( - name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='enterprise', mode=mode, docker_repo='grafana/grafana-enterprise'), edition="" - ),] + return [ + pipeline( + name = "publish-docker-oss-{}".format(mode), + trigger = trigger, + steps = publish_image_steps(edition = "oss", mode = mode, docker_repo = "grafana"), + edition = "", + environment = {"EDITION": "oss"}, + ), + pipeline( + name = "publish-docker-enterprise-{}".format(mode), + trigger = trigger, + steps = publish_image_steps( + edition = "enterprise", + mode = mode, + docker_repo = "grafana-enterprise", + ), + edition = "", + environment = {"EDITION": "enterprise"}, + ), + ] def publish_image_pipelines_security(): - mode='security' + mode = "security" trigger = { - 'event': ['promote'], - 'target': [mode], + "event": ["promote"], + "target": [mode], } - return [pipeline( - name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='enterprise', mode=mode, docker_repo='grafana/grafana-enterprise'), edition="" - ),] + return [ + pipeline( + name = "publish-docker-enterprise-{}".format(mode), + trigger = trigger, + steps = publish_image_steps( + edition = "enterprise", + mode = mode, + docker_repo = "grafana-enterprise", + ), + edition = "", + environment = {"EDITION": "enterprise"}, + ), + ] diff --git a/scripts/drone/pipelines/shellcheck.star b/scripts/drone/pipelines/shellcheck.star index 808d7329ba9..2a2aa55149e 100644 --- a/scripts/drone/pipelines/shellcheck.star +++ b/scripts/drone/pipelines/shellcheck.star @@ -1,48 +1,50 @@ -load( - 'scripts/drone/steps/lib.star', - 'build_image', - 'compile_build_cmd' -) +""" +This module returns a Drone step and pipeline for linting with shellcheck. +""" +load("scripts/drone/steps/lib.star", "build_image", "compile_build_cmd") load( - 'scripts/drone/utils/utils.star', - 'pipeline', + "scripts/drone/utils/utils.star", + "pipeline", ) trigger = { - 'event': [ - 'pull_request', + "event": [ + "pull_request", ], - 'paths': { - 'exclude': [ - '*.md', - 'docs/**', - 'latest.json', - ], - 'include': [ - 'scripts/**/*.sh' + "paths": { + "exclude": [ + "*.md", + "docs/**", + "latest.json", ], + "include": ["scripts/**/*.sh"], }, } def shellcheck_step(): return { - 'name': 'shellcheck', - 'image': build_image, - 'depends_on': [ - 'compile-build-cmd', + "name": "shellcheck", + "image": build_image, + "depends_on": [ + "compile-build-cmd", ], - 'commands': [ - './bin/build shellcheck', + "commands": [ + "./bin/build shellcheck", ], } def shellcheck_pipeline(): + environment = {"EDITION": "oss"} steps = [ compile_build_cmd(), shellcheck_step(), ] return pipeline( - name='pr-shellcheck', edition="oss", trigger=trigger, services=[], steps=steps, + name = "pr-shellcheck", + edition = "oss", + trigger = trigger, + services = [], + steps = steps, + environment = environment, ) - diff --git a/scripts/drone/pipelines/test_backend.star b/scripts/drone/pipelines/test_backend.star index 94df1f1e59d..a3abb40e667 100644 --- a/scripts/drone/pipelines/test_backend.star +++ b/scripts/drone/pipelines/test_backend.star @@ -1,40 +1,116 @@ -load( - 'scripts/drone/steps/lib.star', - 'identify_runner_step', - 'download_grabpl_step', - 'wire_install_step', - 'test_backend_step', - 'test_backend_integration_step', - 'verify_gen_cue_step', - 'compile_build_cmd', - 'clone_enterprise_step', - 'init_enterprise_step', -) +""" +This module returns the pipeline used for testing backend code. +""" load( - 'scripts/drone/utils/utils.star', - 'pipeline', + "scripts/drone/steps/lib.star", + "clone_enterprise_step", + "compile_build_cmd", + "download_grabpl_step", + "enterprise_setup_step", + "identify_runner_step", + "init_enterprise_step", + "test_backend_integration_step", + "test_backend_step", + "verify_gen_cue_step", + "wire_install_step", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", + "with_deps", ) -def test_backend(trigger, ver_mode, edition="oss"): - environment = {'EDITION': edition} - init_steps = [] - if edition != 'oss': - init_steps.extend([clone_enterprise_step(ver_mode), download_grabpl_step(), init_enterprise_step(ver_mode),]) - init_steps.extend([ +def test_backend(trigger, ver_mode): + """Generates the pipeline used for testing OSS backend code. + + Args: + trigger: a Drone trigger for the pipeline. + ver_mode: affects the pipeline name. + + Returns: + Drone pipeline. + """ + environment = {"EDITION": "oss"} + + steps = [] + + verify_step = verify_gen_cue_step() + + if ver_mode == "pr": + # In pull requests, attempt to clone grafana enterprise. + steps.append(enterprise_setup_step()) + + # Ensure that verif_gen_cue happens after we clone enterprise + # At the time of writing this, very_gen_cue is depended on by the wire step which is what everything else depends on. + verify_step["depends_on"].append("clone-enterprise") + + steps += [ identify_runner_step(), - compile_build_cmd(edition), - verify_gen_cue_step(edition), + compile_build_cmd(edition = "oss"), + verify_step, wire_install_step(), - ]) - test_steps = [ - test_backend_step(edition), - test_backend_integration_step(edition), + test_backend_step(), + test_backend_integration_step(), ] - pipeline_name = '{}-test-backend'.format(ver_mode) + pipeline_name = "{}-test-backend".format(ver_mode) if ver_mode in ("release-branch", "release"): - pipeline_name = '{}-{}-test-backend'.format(ver_mode, edition) + pipeline_name = "{}-{}-test-backend".format(ver_mode, "oss") + return pipeline( - name=pipeline_name, edition=edition, trigger=trigger, services=[], steps=init_steps + test_steps, environment=environment + name = pipeline_name, + edition = "oss", + trigger = trigger, + steps = steps, + environment = environment, + ) + +def test_backend_enterprise(trigger, ver_mode, source, edition = "enterprise"): + """Generates the pipeline used for testing backend enterprise code. + + Args: + trigger: a Drone trigger for the pipeline. + ver_mode: affects the pipeline name. + source: controls what revision of enterprise code to test with. The source of the PR, usually. + edition: affects the clone step in the pipeline and also affects the pipeline name. + + Returns: + Drone pipeline. + """ + environment = {"EDITION": edition} + + steps = ( + [ + clone_enterprise_step(source), + download_grabpl_step(), + init_enterprise_step(ver_mode), + identify_runner_step(), + compile_build_cmd(edition), + ] + + with_deps( + [ + verify_gen_cue_step(), + ], + [ + "init-enterprise", + ], + ) + + [ + wire_install_step(), + test_backend_step(), + test_backend_integration_step(), + ] + ) + + pipeline_name = "{}-test-backend".format(ver_mode) + if ver_mode in ("release-branch", "release"): + pipeline_name = "{}-{}-test-backend".format(ver_mode, edition) + + return pipeline( + name = pipeline_name, + edition = edition, + trigger = trigger, + steps = steps, + environment = environment, ) diff --git a/scripts/drone/pipelines/test_frontend.star b/scripts/drone/pipelines/test_frontend.star index 7490c66c2e2..843b1ce8b73 100644 --- a/scripts/drone/pipelines/test_frontend.star +++ b/scripts/drone/pipelines/test_frontend.star @@ -1,36 +1,103 @@ -load( - 'scripts/drone/steps/lib.star', - 'identify_runner_step', - 'clone_enterprise_step', - 'init_enterprise_step', - 'download_grabpl_step', - 'yarn_install_step', - 'betterer_frontend_step', - 'test_frontend_step', -) +""" +This module returns the pipeline used for testing backend code. +""" load( - 'scripts/drone/utils/utils.star', - 'pipeline', + "scripts/drone/steps/lib.star", + "betterer_frontend_step", + "clone_enterprise_step", + "download_grabpl_step", + "enterprise_setup_step", + "identify_runner_step", + "init_enterprise_step", + "test_frontend_step", + "yarn_install_step", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", + "with_deps", ) -def test_frontend(trigger, ver_mode, edition="oss"): - environment = {'EDITION': edition} - init_steps = [] - if edition != 'oss': - init_steps.extend([clone_enterprise_step(ver_mode), init_enterprise_step(ver_mode),]) - init_steps.extend([ +def test_frontend(trigger, ver_mode): + """Generates the pipeline used for testing frontend code. + + Args: + trigger: a Drone trigger for the pipeline + ver_mode: indirectly controls which revision of enterprise code to use. + + Returns: + Drone pipeline. + """ + environment = {"EDITION": "oss"} + + steps = [ identify_runner_step(), download_grabpl_step(), - yarn_install_step(edition), - ]) - test_steps = [ - betterer_frontend_step(edition), - test_frontend_step(edition), + yarn_install_step(), + betterer_frontend_step(edition = "oss"), ] - pipeline_name = '{}-test-frontend'.format(ver_mode) + + pipeline_name = "{}-test-frontend".format(ver_mode) + + test_step = test_frontend_step(edition = "oss") + + if ver_mode == "pr": + # In pull requests, attempt to clone grafana enterprise. + steps.append(enterprise_setup_step()) + + # Also, make the test step depend on 'clone-enterprise + test_step["depends_on"].append("clone-enterprise") + + steps.append(test_step) + + pipeline_name = "{}-test-frontend".format(ver_mode) if ver_mode in ("release-branch", "release"): - pipeline_name = '{}-{}-test-frontend'.format(ver_mode, edition) + pipeline_name = "{}-oss-test-frontend".format(ver_mode) + return pipeline( - name=pipeline_name, edition=edition, trigger=trigger, services=[], steps=init_steps + test_steps, + name = pipeline_name, + edition = "oss", + trigger = trigger, + steps = steps, + environment = environment, + ) + +def test_frontend_enterprise(trigger, ver_mode, source, edition = "enterprise"): + """Generates the pipeline used for testing frontend enterprise code. + + Args: + trigger: a Drone trigger for the pipeline. + ver_mode: affects the pipeline name. + source: controls what revision of Grafana code to test with. + edition: affects the clone step in the pipeline and also affects the pipeline name. + + Returns: + Drone pipeline. + """ + environment = {"EDITION": edition} + steps = ( + [ + clone_enterprise_step(source), + init_enterprise_step(ver_mode), + identify_runner_step(), + download_grabpl_step(), + ] + + with_deps([yarn_install_step()], ["init-enterprise"]) + + [ + betterer_frontend_step(edition), + test_frontend_step(edition), + ] + ) + + pipeline_name = "{}-test-frontend".format(ver_mode) + if ver_mode in ("release-branch", "release"): + pipeline_name = "{}-{}-test-frontend".format(ver_mode, edition) + + return pipeline( + name = pipeline_name, + edition = edition, + trigger = trigger, + steps = steps, + environment = environment, ) diff --git a/scripts/drone/pipelines/trigger_downstream.star b/scripts/drone/pipelines/trigger_downstream.star index 2c40f06d82a..d3403de50f1 100644 --- a/scripts/drone/pipelines/trigger_downstream.star +++ b/scripts/drone/pipelines/trigger_downstream.star @@ -1,28 +1,45 @@ -load( - 'scripts/drone/steps/lib.star', - 'enterprise_downstream_step', -) +""" +This module returns the pipeline used for triggering a downstream pipeline for Grafana Enterprise. +""" load( - 'scripts/drone/utils/utils.star', - 'pipeline', + "scripts/drone/steps/lib.star", + "enterprise_downstream_step", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", ) trigger = { - 'event': ['push',], - 'branch': 'main', - 'paths': { - 'exclude': [ - '*.md', - 'docs/**', - 'latest.json', + "event": [ + "push", + ], + "branch": "main", + "paths": { + "exclude": [ + "*.md", + "docs/**", + "latest.json", ], }, } -def enterprise_downstream_pipeline(edition, ver_mode): - steps = [enterprise_downstream_step(edition, ver_mode)] - deps = ['main-publish'] +def enterprise_downstream_pipeline(): + environment = {"EDITION": "oss"} + steps = [ + enterprise_downstream_step(ver_mode = "main"), + ] + deps = [ + "main-build-e2e-publish", + "main-integration-tests", + ] return pipeline( - name='main-trigger-downstream', edition=edition, trigger=trigger, services=[], steps=steps, depends_on=deps - ) + name = "main-trigger-downstream", + edition = "oss", + trigger = trigger, + services = [], + steps = steps, + depends_on = deps, + environment = environment, + ) diff --git a/scripts/drone/pipelines/verify_drone.star b/scripts/drone/pipelines/verify_drone.star index e350c8e0a7e..83fb2458c82 100644 --- a/scripts/drone/pipelines/verify_drone.star +++ b/scripts/drone/pipelines/verify_drone.star @@ -1,17 +1,21 @@ -load( - 'scripts/drone/steps/lib.star', - 'identify_runner_step', - 'download_grabpl_step', - 'lint_drone_step', - 'compile_build_cmd', -) +""" +This module returns the pipeline used for verifying Drone configuration. +""" load( - 'scripts/drone/utils/utils.star', - 'pipeline', + "scripts/drone/steps/lib.star", + "compile_build_cmd", + "download_grabpl_step", + "identify_runner_step", + "lint_drone_step", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", ) def verify_drone(trigger, ver_mode): + environment = {"EDITION": "oss"} steps = [ identify_runner_step(), download_grabpl_step(), @@ -19,5 +23,10 @@ def verify_drone(trigger, ver_mode): lint_drone_step(), ] return pipeline( - name='{}-verify-drone'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=steps, + name = "{}-verify-drone".format(ver_mode), + edition = "oss", + trigger = trigger, + services = [], + steps = steps, + environment = environment, ) diff --git a/scripts/drone/pipelines/verify_starlark.star b/scripts/drone/pipelines/verify_starlark.star new file mode 100644 index 00000000000..2970e598f12 --- /dev/null +++ b/scripts/drone/pipelines/verify_starlark.star @@ -0,0 +1,32 @@ +""" +This module returns a Drone pipeline that verifies all Starlark files are linted. +""" + +load( + "scripts/drone/steps/lib.star", + "compile_build_cmd", + "download_grabpl_step", + "identify_runner_step", + "lint_starlark_step", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", +) + +def verify_starlark(trigger, ver_mode): + environment = {"EDITION": "oss"} + steps = [ + identify_runner_step(), + download_grabpl_step(), + compile_build_cmd(), + lint_starlark_step(), + ] + return pipeline( + name = "{}-verify-starlark".format(ver_mode), + edition = "oss", + trigger = trigger, + services = [], + steps = steps, + environment = environment, + ) diff --git a/scripts/drone/pipelines/windows.star b/scripts/drone/pipelines/windows.star index c0e0f61f7f6..b726edc2463 100644 --- a/scripts/drone/pipelines/windows.star +++ b/scripts/drone/pipelines/windows.star @@ -1,134 +1,41 @@ -load( - 'scripts/drone/steps/lib.star', - 'grabpl_version', - 'wix_image', - 'identify_runner_step', -) +""" +This module returns the pipeline used for building Grafana on Windows. +""" load( - 'scripts/drone/utils/utils.star', - 'pipeline', + "scripts/drone/utils/utils.star", + "pipeline", +) +load( + "scripts/drone/steps/lib.star", + "get_windows_steps", ) - -load('scripts/drone/vault.star', 'from_secret', 'prerelease_bucket', 'github_token') def windows(trigger, edition, ver_mode): - init_cmds = [] - sfx = '' - if edition in ('enterprise', 'enterprise2'): - sfx = '-{}'.format(edition) - else: - init_cmds.extend([ - '$$ProgressPreference = "SilentlyContinue"', - 'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format( - grabpl_version), - ]) - steps = [ - { - 'name': 'windows-init', - 'image': wix_image, - 'commands': init_cmds, - }, - ] - if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in ( - 'release', 'release-branch', - ): - bucket = '%PRERELEASE_BUCKET%/artifacts/downloads' - if ver_mode == 'release': - ver_part = '${DRONE_TAG}' - dir = 'release' - else: - dir = 'main' - bucket = 'grafana-downloads' - build_no = 'DRONE_BUILD_NUMBER' - ver_part = '--build-id $$env:{}'.format(build_no) - installer_commands = [ - '$$gcpKey = $$env:GCP_KEY', - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json', - # gcloud fails to read the file unless converted with dos2unix - 'dos2unix gcpkey.json', - 'gcloud auth activate-service-account --key-file=gcpkey.json', - 'rm gcpkey.json', - 'cp C:\\App\\nssm-2.24.zip .', - ] - if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in ( - 'release', - ): - installer_commands.extend([ - '.\\grabpl.exe windows-installer --edition {} {}'.format(edition, ver_part), - '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', - ]) - if ver_mode == 'main': - installer_commands.extend([ - 'gsutil cp $$fname gs://{}/{}/{}/'.format(bucket, edition, dir), - 'gsutil cp "$$fname.sha256" gs://{}/{}/{}/'.format(bucket, edition, dir), - ]) - else: - installer_commands.extend([ - 'gsutil cp $$fname gs://{}/{}/{}/{}/'.format(bucket, ver_part, edition, dir), - 'gsutil cp "$$fname.sha256" gs://{}/{}/{}/{}/'.format(bucket, ver_part, edition, dir), - ]) - steps.append({ - 'name': 'build-windows-installer', - 'image': wix_image, - 'depends_on': [ - 'windows-init', - ], - 'environment': { - 'GCP_KEY': from_secret('gcp_key'), - 'PRERELEASE_BUCKET': from_secret(prerelease_bucket), - 'GITHUB_TOKEN': from_secret('github_token') - }, - 'commands': installer_commands, - }) + """Generates the pipeline used for building Grafana on Windows. - if edition in ('enterprise', 'enterprise2'): - if ver_mode == 'release': - committish = '${DRONE_TAG}' - elif ver_mode == 'release-branch': - committish = '$$env:DRONE_BRANCH' - else: - committish = '$$env:DRONE_COMMIT' - # For enterprise, we have to clone both OSS and enterprise and merge the latter into the former - download_grabpl_step_cmds = [ - '$$ProgressPreference = "SilentlyContinue"', - 'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format( - grabpl_version), - ] - clone_cmds = [ - 'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"', - ] - clone_cmds.extend([ - 'cd grafana-enterprise', - 'git checkout {}'.format(committish), - ]) - steps.insert(0, { - 'name': 'clone', - 'image': wix_image, - 'environment': { - 'GITHUB_TOKEN': from_secret(github_token), - }, - 'commands': download_grabpl_step_cmds + clone_cmds, - }) - steps[1]['depends_on'] = [ - 'clone', - ] - steps[1]['commands'].extend([ - # Need to move grafana-enterprise out of the way, so directory is empty and can be cloned into - 'cp -r grafana-enterprise C:\\App\\grafana-enterprise', - 'rm -r -force grafana-enterprise', - 'cp grabpl.exe C:\\App\\grabpl.exe', - 'rm -force grabpl.exe', - 'C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise', - 'cp C:\\App\\grabpl.exe grabpl.exe', - ]) - if 'environment' in steps[1]: - steps[1]['environment'] + {'GITHUB_TOKEN': from_secret(github_token)} - else: - steps[1]['environment'] = {'GITHUB_TOKEN': from_secret(github_token)} + Args: + trigger: a Drone trigger for the pipeline. + edition: controls whether enterprise code is included in the pipeline steps. + ver_mode: controls whether a pre-release or actual release pipeline is generated. + Also indirectly controls which version of enterprise code is used. + + Returns: + Drone pipeline. + """ + environment = {"EDITION": edition} return pipeline( - name='main-windows', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']), - steps=[identify_runner_step('windows')] + steps, - depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows', + name = "main-windows", + edition = edition, + trigger = dict(trigger, repo = ["grafana/grafana"]), + steps = get_windows_steps(edition, ver_mode), + depends_on = [ + "main-test-frontend", + "main-test-backend", + "main-build-e2e-publish", + "main-integration-tests", + ], + platform = "windows", + environment = environment, ) diff --git a/scripts/drone/services/services.star b/scripts/drone/services/services.star index e81d0550077..9d7d04a2ded 100644 --- a/scripts/drone/services/services.star +++ b/scripts/drone/services/services.star @@ -1,61 +1,66 @@ +""" +This module has functions for Drone services to be used in pipelines. +""" + def integration_test_services_volumes(): return [ - { 'name': 'postgres', 'temp': { 'medium': 'memory' } }, - { 'name': 'mysql', 'temp': { 'medium': 'memory' } - }] + {"name": "postgres", "temp": {"medium": "memory"}}, + {"name": "mysql", "temp": {"medium": "memory"}}, + ] def integration_test_services(edition): services = [ { - 'name': 'postgres', - 'image': 'postgres:12.3-alpine', - 'environment': { - 'POSTGRES_USER': 'grafanatest', - 'POSTGRES_PASSWORD': 'grafanatest', - 'POSTGRES_DB': 'grafanatest', - 'PGDATA': '/var/lib/postgresql/data/pgdata', + "name": "postgres", + "image": "postgres:12.3-alpine", + "environment": { + "POSTGRES_USER": "grafanatest", + "POSTGRES_PASSWORD": "grafanatest", + "POSTGRES_DB": "grafanatest", + "PGDATA": "/var/lib/postgresql/data/pgdata", }, - 'volumes': [{ - 'name': 'postgres', - 'path': '/var/lib/postgresql/data/pgdata' - }], + "volumes": [ + {"name": "postgres", "path": "/var/lib/postgresql/data/pgdata"}, + ], }, { - 'name': 'mysql', - 'image': 'mysql:5.7.39', - 'environment': { - 'MYSQL_ROOT_PASSWORD': 'rootpass', - 'MYSQL_DATABASE': 'grafana_tests', - 'MYSQL_USER': 'grafana', - 'MYSQL_PASSWORD': 'password', + "name": "mysql", + "image": "mysql:5.7.39", + "environment": { + "MYSQL_ROOT_PASSWORD": "rootpass", + "MYSQL_DATABASE": "grafana_tests", + "MYSQL_USER": "grafana", + "MYSQL_PASSWORD": "password", }, - 'volumes': [{ - 'name': 'mysql', - 'path': '/var/lib/mysql' - }] + "volumes": [{"name": "mysql", "path": "/var/lib/mysql"}], }, ] - if edition in ('enterprise', 'enterprise2'): - services.extend([{ - 'name': 'redis', - 'image': 'redis:6.2.1-alpine', - 'environment': {}, - }, { - 'name': 'memcached', - 'image': 'memcached:1.6.9-alpine', - 'environment': {}, - }]) + if edition in ("enterprise", "enterprise2"): + services.extend( + [ + { + "name": "redis", + "image": "redis:6.2.1-alpine", + "environment": {}, + }, + { + "name": "memcached", + "image": "memcached:1.6.9-alpine", + "environment": {}, + }, + ], + ) return services def ldap_service(): return { - 'name': 'ldap', - 'image': 'osixia/openldap:1.4.0', - 'environment': { - 'LDAP_ADMIN_PASSWORD': 'grafana', - 'LDAP_DOMAIN': 'grafana.org', - 'SLAPD_ADDITIONAL_MODULES': 'memberof', + "name": "ldap", + "image": "osixia/openldap:1.4.0", + "environment": { + "LDAP_ADMIN_PASSWORD": "grafana", + "LDAP_DOMAIN": "grafana.org", + "SLAPD_ADDITIONAL_MODULES": "memberof", }, } diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 3d8cdd4e01e..52cf4443d9c 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -1,1287 +1,1561 @@ -load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token', 'prerelease_bucket') +""" +This module is a library of Drone steps and other pipeline components. +""" -grabpl_version = 'v3.0.15' -build_image = 'grafana/build-container:1.6.6' -publish_image = 'grafana/grafana-ci-deploy:1.3.3' -deploy_docker_image = 'us.gcr.io/kubernetes-dev/drone/plugins/deploy-image' -alpine_image = 'alpine:3.15.6' -curl_image = 'byrnedo/alpine-curl:0.1.8' -windows_image = 'mcr.microsoft.com/windows:1809' -wix_image = 'grafana/ci-wix:0.1.1' -go_image = 'golang:1.19.4' +load( + "scripts/drone/vault.star", + "from_secret", + "prerelease_bucket", +) + +grabpl_version = "v3.0.21" +build_image = "grafana/build-container:v1.7.1" +publish_image = "grafana/grafana-ci-deploy:1.3.3" +deploy_docker_image = "us.gcr.io/kubernetes-dev/drone/plugins/deploy-image" +alpine_image = "alpine:3.17.1" +curl_image = "byrnedo/alpine-curl:0.1.8" +windows_image = "mcr.microsoft.com/windows:1809" +wix_image = "grafana/ci-wix:0.1.1" +go_image = "golang:1.19.4" -disable_tests = False trigger_oss = { - 'repo': [ - 'grafana/grafana', - ] + "repo": [ + "grafana/grafana", + ], } - def slack_step(channel, template, secret): return { - 'name': 'slack', - 'image': 'plugins/slack', - 'settings': { - 'webhook': from_secret(secret), - 'channel': channel, - 'template': template, + "name": "slack", + "image": "plugins/slack", + "settings": { + "webhook": from_secret(secret), + "channel": channel, + "template": template, }, } -def yarn_install_step(edition="oss"): - deps = [] - if edition == 'enterprise': - deps = ['init-enterprise'] +def yarn_install_step(): return { - 'name': 'yarn-install', - 'image': build_image, - 'commands': [ - 'yarn install --immutable', + "name": "yarn-install", + "image": build_image, + "commands": [ + "yarn install --immutable", ], - 'depends_on': deps, + "depends_on": [], } - def wire_install_step(): return { - 'name': 'wire-install', - 'image': build_image, - 'commands': [ - 'make gen-go', + "name": "wire-install", + "image": build_image, + "commands": [ + "make gen-go", ], - 'depends_on': [ - 'verify-gen-cue', + "depends_on": [ + "verify-gen-cue", ], } - -def identify_runner_step(platform='linux'): - if platform == 'windows': +def identify_runner_step(platform = "linux"): + if platform == "linux": return { - 'name': 'identify-runner', - 'image': windows_image, - 'commands': [ - 'echo $env:DRONE_RUNNER_NAME', + "name": "identify-runner", + "image": alpine_image, + "commands": [ + "echo $DRONE_RUNNER_NAME", ], } else: return { - 'name': 'identify-runner', - 'image': alpine_image, - 'commands': [ - 'echo $DRONE_RUNNER_NAME', + "name": "identify-runner", + "image": windows_image, + "commands": [ + "echo $env:DRONE_RUNNER_NAME", ], } +def enterprise_setup_step(source = "${DRONE_SOURCE_BRANCH}", canFail = True): + step = clone_enterprise_step_pr(source = source, target = "${DRONE_TARGET_BRANCH}", canFail = canFail, location = "../grafana-enterprise") + step["commands"] += [ + "cd ../", + "ln -s src grafana", + "cd ./grafana-enterprise", + "./build.sh", + ] -def clone_enterprise_step(ver_mode): - if ver_mode == 'release': - committish = '${DRONE_TAG}' - elif ver_mode == 'release-branch': - committish = '${DRONE_BRANCH}' - else: - committish = '${DRONE_COMMIT}' - return { - 'name': 'clone-enterprise', - 'image': build_image, - 'environment': { - 'GITHUB_TOKEN': from_secret(github_token), + return step + +def clone_enterprise_step(source = "${DRONE_COMMIT}"): + """Clone the enterprise source into the ./grafana-enterprise directory. + + Args: + source: controls which revision of grafana-enterprise is checked out, if it exists. The name 'source' derives from the 'source branch' of a pull request. + Returns: + Drone step. + """ + step = { + "name": "clone-enterprise", + "image": build_image, + "environment": { + "GITHUB_TOKEN": from_secret("github_token"), }, - 'commands': [ + "commands": [ 'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git"', - 'cd grafana-enterprise', - 'git checkout {}'.format(committish), + "cd grafana-enterprise", + "git checkout {}".format(source), ], } + return step + +def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFail = False, location = "grafana-enterprise"): + """Clone the enterprise source into the ./grafana-enterprise directory. + + Args: + source: controls which revision of grafana-enterprise is checked out, if it exists. The name 'source' derives from the 'source branch' of a pull request. + target: controls which revision of grafana-enterprise is checked out, if it 'source' does not exist. The name 'target' derives from the 'target branch' of a pull request. If this does not exist, then 'main' will be checked out. + canFail: controls whether or not this step is allowed to fail. If it fails and this is true, then the pipeline will continue. canFail is used in pull request pipelines where enterprise may be cloned but may not clone in forks. + location: the path where grafana-enterprise is cloned. + Returns: + Drone step. + """ + step = { + "name": "clone-enterprise", + "image": build_image, + "environment": { + "GITHUB_TOKEN": from_secret("github_token"), + }, + "commands": [ + 'is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork)', + 'if [ "$is_fork" != false ]; then return 1; fi', # Only clone if we're confident that 'fork' is 'false'. Fail if it's also empty. + 'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" ' + location, + "cd {}".format(location), + 'if git checkout {0}; then echo "checked out {0}"; elif git checkout {1}; then echo "git checkout {1}"; else git checkout main; fi'.format(source, target), + ], + } + + if canFail: + step["failure"] = "ignore" + + return step + def init_enterprise_step(ver_mode): - source_commit = '' - if ver_mode == 'release': - source_commit = ' ${DRONE_TAG}' + """Adds the enterprise deployment configuration into the source directory. + + Args: + ver_mode: controls what revision of the OSS source to use. + If ver_mode is 'release', the step uses the tagged revision. + Otherwise, the DRONE_SOURCE_BRANCH is used. + + Returns: + Drone step. + """ + source_commit = "" + if ver_mode == "release": + source_commit = " ${DRONE_TAG}" environment = { - 'GITHUB_TOKEN': from_secret(github_token), + "GITHUB_TOKEN": from_secret("github_token"), } token = "--github-token $${GITHUB_TOKEN}" - elif ver_mode == 'release-branch': + elif ver_mode == "release-branch": environment = { - 'GITHUB_TOKEN': from_secret(github_token), + "GITHUB_TOKEN": from_secret("github_token"), } token = "--github-token $${GITHUB_TOKEN}" else: environment = {} token = "" return { - 'name': 'init-enterprise', - 'image': build_image, - 'depends_on': [ - 'clone-enterprise', + "name": "init-enterprise", + "image": build_image, + "depends_on": [ + "clone-enterprise", ], - 'environment': environment, - 'commands': [ - 'mv bin/grabpl /tmp/', - 'rmdir bin', - 'mv grafana-enterprise /tmp/', - '/tmp/grabpl init-enterprise {} /tmp/grafana-enterprise{}'.format(token, source_commit), - 'mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json', - 'mkdir bin', - 'mv /tmp/grabpl bin/' + "environment": environment, + "commands": [ + "mv bin/grabpl /tmp/", + "rmdir bin", + "mv grafana-enterprise /tmp/", + "/tmp/grabpl init-enterprise {} /tmp/grafana-enterprise{}".format( + token, + source_commit, + ).rstrip(), + "mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json", + "mkdir bin", + "mv /tmp/grabpl bin/", ], } - -def download_grabpl_step(platform="linux"): - if platform == 'windows': +def download_grabpl_step(platform = "linux"): + if platform == "windows": return { - 'name': 'grabpl', - 'image': wix_image, - 'commands': [ + "name": "grabpl", + "image": wix_image, + "commands": [ '$$ProgressPreference = "SilentlyContinue"', - 'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format( - grabpl_version), - ] + "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( + grabpl_version, + ), + ], } return { - 'name': 'grabpl', - 'image': curl_image, - 'commands': [ - 'mkdir -p bin', - 'curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/grabpl'.format( - grabpl_version + "name": "grabpl", + "image": curl_image, + "commands": [ + "mkdir -p bin", + "curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/grabpl".format( + grabpl_version, ), - 'chmod +x bin/grabpl', - ] + "chmod +x bin/grabpl", + ], } - def lint_drone_step(): return { - 'name': 'lint-drone', - 'image': curl_image, - 'commands': [ - './bin/build verify-drone', + "name": "lint-drone", + "image": curl_image, + "commands": [ + "./bin/build verify-drone", ], - 'depends_on': [ - 'compile-build-cmd', + "depends_on": [ + "compile-build-cmd", ], } -def enterprise_downstream_step(edition, ver_mode): - if edition in ('enterprise', 'enterprise2'): - return None +def lint_starlark_step(): + return { + "name": "lint-starlark", + "image": build_image, + "commands": [ + "./bin/build verify-starlark .", + ], + "depends_on": [ + "compile-build-cmd", + ], + } - repo = 'grafana/grafana-enterprise@' - if ver_mode == 'pr': - repo += '${DRONE_SOURCE_BRANCH}' +def enterprise_downstream_step(ver_mode): + """Triggers a downstream pipeline in the grafana-enterprise repository. + + Args: + ver_mode: indirectly controls the revision used for downstream pipelines. + It also used to allow the step to fail for pull requests without blocking merging. + + Returns: + Drone step. + """ + repo = "grafana/grafana-enterprise@" + if ver_mode == "pr": + repo += "${DRONE_SOURCE_BRANCH}" else: - repo += 'main' + repo += "main" step = { - 'name': 'trigger-enterprise-downstream', - 'image': 'grafana/drone-downstream', - 'settings': { - 'server': 'https://drone.grafana.net', - 'token': from_secret(drone_token), - 'repositories': [ + "name": "trigger-enterprise-downstream", + "image": "grafana/drone-downstream", + "settings": { + "server": "https://drone.grafana.net", + "token": from_secret("drone_token"), + "repositories": [ repo, ], - 'params': [ - 'SOURCE_BUILD_NUMBER=${DRONE_COMMIT}', - 'SOURCE_COMMIT=${DRONE_COMMIT}', + "params": [ + "SOURCE_BUILD_NUMBER=${DRONE_COMMIT}", + "SOURCE_COMMIT=${DRONE_COMMIT}", ], }, } - if ver_mode == 'pr': - step.update({ 'failure': 'ignore' }) - step['settings']['params'].append('OSS_PULL_REQUEST=${DRONE_PULL_REQUEST}') + if ver_mode == "pr": + step.update({"failure": "ignore"}) + step["settings"]["params"].append("OSS_PULL_REQUEST=${DRONE_PULL_REQUEST}") return step - -def lint_backend_step(edition): +def lint_backend_step(): return { - 'name': 'lint-backend' + enterprise2_suffix(edition), - 'image': go_image, - 'environment': { + "name": "lint-backend", + # TODO: build_image or go_image? + "image": go_image, + "environment": { # We need CGO because of go-sqlite3 - 'CGO_ENABLED': '1', + "CGO_ENABLED": "1", }, - 'depends_on': [ - 'wire-install', + "depends_on": [ + "wire-install", ], - 'commands': [ - 'apt-get update && apt-get install make', + "commands": [ + "apt-get update && apt-get install make", # Don't use Make since it will re-download the linters - 'make lint-go', + "make lint-go", ], } - def benchmark_ldap_step(): return { - 'name': 'benchmark-ldap', - 'image': build_image, - 'environment': { - 'LDAP_HOSTNAME': 'ldap', + "name": "benchmark-ldap", + "image": build_image, + "environment": { + "LDAP_HOSTNAME": "ldap", }, - 'commands': [ - 'dockerize -wait tcp://ldap:389 -timeout 120s', + "commands": [ + "dockerize -wait tcp://ldap:389 -timeout 120s", 'go test -benchmem -run=^$ ./pkg/extensions/ldapsync -bench "^(Benchmark50Users)$"', ], } - -def build_storybook_step(edition, ver_mode): - if edition in ('enterprise', 'enterprise2'): - return None - +def build_storybook_step(ver_mode): return { - 'name': 'build-storybook', - 'image': build_image, - 'depends_on': [ + "name": "build-storybook", + "image": build_image, + "depends_on": [ # Best to ensure that this step doesn't mess with what's getting built and packaged - 'build-frontend', - 'build-frontend-packages', + "build-frontend", + "build-frontend-packages", ], - 'environment': { - 'NODE_OPTIONS': '--max_old_space_size=4096', + "environment": { + "NODE_OPTIONS": "--max_old_space_size=4096", }, - 'commands': [ - 'yarn storybook:build', - './bin/grabpl verify-storybook', + "commands": [ + "yarn storybook:build", + "./bin/build verify-storybook", ], - 'when': get_trigger_storybook(ver_mode), + "when": get_trigger_storybook(ver_mode), } +def store_storybook_step(ver_mode, trigger = None): + """Publishes the Grafana UI components storybook. -def store_storybook_step(edition, ver_mode, trigger=None): - if edition in ('enterprise', 'enterprise2'): - return None + Args: + ver_mode: controls whether a release or canary version is published. + trigger: a Drone trigger for the step. + Defaults to None. + Returns: + Drone step. + """ commands = [] - if ver_mode == 'release': - commands.extend([ - './bin/build store-storybook --deployment latest', - './bin/build store-storybook --deployment ${DRONE_TAG}', - ]) + if ver_mode == "release": + commands.extend( + [ + "./bin/build store-storybook --deployment latest", + "./bin/build store-storybook --deployment ${DRONE_TAG}", + ], + ) else: # main pipelines should deploy storybook to grafana-storybook/canary public bucket - commands = ['./bin/build store-storybook --deployment canary', ] + commands = [ + "./bin/build store-storybook --deployment canary", + ] step = { - 'name': 'store-storybook', - 'image': publish_image, - 'depends_on': ['build-storybook', ] + end_to_end_tests_deps(edition), - 'environment': { - 'GCP_KEY': from_secret('gcp_key'), - 'PRERELEASE_BUCKET': from_secret(prerelease_bucket) + "name": "store-storybook", + "image": publish_image, + "depends_on": [ + "build-storybook", + ] + + end_to_end_tests_deps(), + "environment": { + "GCP_KEY": from_secret("gcp_key"), + "PRERELEASE_BUCKET": from_secret(prerelease_bucket), }, - 'commands': commands, - 'when': get_trigger_storybook(ver_mode), + "commands": commands, + "when": get_trigger_storybook(ver_mode), } if trigger and ver_mode in ("release-branch", "main"): # no dict merge operation available, https://github.com/harness/drone-cli/pull/220 when_cond = { - 'repo': ['grafana/grafana',], - 'paths': { - 'include': [ - 'packages/grafana-ui/**', + "repo": [ + "grafana/grafana", + ], + "paths": { + "include": [ + "packages/grafana-ui/**", ], - } + }, } - step = dict(step, when=when_cond) + step = dict(step, when = when_cond) return step - -def e2e_tests_artifacts(edition): +def e2e_tests_artifacts(): return { - 'name': 'e2e-tests-artifacts-upload' + enterprise2_suffix(edition), - 'image': 'google/cloud-sdk:367.0.0', - 'depends_on': [ - 'end-to-end-tests-dashboards-suite', - 'end-to-end-tests-panels-suite', - 'end-to-end-tests-smoke-tests-suite', - 'end-to-end-tests-various-suite', + "name": "e2e-tests-artifacts-upload", + "image": "google/cloud-sdk:406.0.0", + "depends_on": [ + "end-to-end-tests-dashboards-suite", + "end-to-end-tests-panels-suite", + "end-to-end-tests-smoke-tests-suite", + "end-to-end-tests-various-suite", ], - 'when': { - 'status': [ - 'success', - 'failure', - ] + "failure": "ignore", + "when": { + "status": [ + "success", + "failure", + ], }, - 'environment': { - 'GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY': from_secret('gcp_upload_artifacts_key'), - 'E2E_TEST_ARTIFACTS_BUCKET': 'releng-pipeline-artifacts-dev', - 'GITHUB_TOKEN': from_secret('github_token'), + "environment": { + "GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY": from_secret("gcp_upload_artifacts_key"), + "E2E_TEST_ARTIFACTS_BUCKET": "releng-pipeline-artifacts-dev", + "GITHUB_TOKEN": from_secret("github_token"), }, - 'commands': [ - 'apt-get update', - 'apt-get install -yq zip', - 'ls -lah ./e2e', - 'find ./e2e -type f -name "*.mp4"', - 'printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json', - 'gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json', + "commands": [ + "apt-get update", + "apt-get install -yq zip", + "printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json", + "gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json", # we want to only include files in e2e folder that end with .spec.ts.mp4 'find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@', - 'gsutil cp e2e/videos.zip gs://$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip', - 'export E2E_ARTIFACTS_VIDEO_ZIP=https://storage.googleapis.com/$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip', + "gsutil cp e2e/videos.zip gs://$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip", + "export E2E_ARTIFACTS_VIDEO_ZIP=https://storage.googleapis.com/$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip", 'echo "E2E Test artifacts uploaded to: $${E2E_ARTIFACTS_VIDEO_ZIP}"', 'curl -X POST https://api.github.com/repos/${DRONE_REPO}/statuses/${DRONE_COMMIT_SHA} -H "Authorization: token $${GITHUB_TOKEN}" -d ' + '"{\\"state\\":\\"success\\",\\"target_url\\":\\"$${E2E_ARTIFACTS_VIDEO_ZIP}\\", \\"description\\": \\"Click on the details to download e2e recording videos\\", \\"context\\": \\"e2e_artifacts\\"}"', ], } +def upload_cdn_step(edition, ver_mode, trigger = None): + """Uploads CDN assets using the Grafana build tool. -def upload_cdn_step(edition, ver_mode, trigger=None): + Args: + edition: controls the output directory for the CDN assets. + ver_mode: only uses the step trigger when ver_mode == 'release-branch' or 'main' + trigger: a Drone trigger for the step. + Defaults to None. + + Returns: + Drone step. + """ deps = [] - if edition in 'enterprise2': - deps.extend([ - 'package' + enterprise2_suffix(edition), - ]) + if edition in "enterprise2": + deps.extend( + [ + "package" + enterprise2_suffix(edition), + ], + ) else: - deps.extend([ - 'grafana-server', - ]) + deps.extend( + [ + "grafana-server", + ], + ) step = { - 'name': 'upload-cdn-assets' + enterprise2_suffix(edition), - 'image': publish_image, - 'depends_on': deps, - 'environment': { - 'GCP_KEY': from_secret('gcp_key'), - 'PRERELEASE_BUCKET': from_secret(prerelease_bucket) + "name": "upload-cdn-assets" + enterprise2_suffix(edition), + "image": publish_image, + "depends_on": deps, + "environment": { + "GCP_KEY": from_secret("gcp_key"), + "PRERELEASE_BUCKET": from_secret(prerelease_bucket), }, - 'commands': [ - './bin/grabpl upload-cdn --edition {}'.format(edition), + "commands": [ + "./bin/build upload-cdn --edition {}".format(edition), ], } if trigger and ver_mode in ("release-branch", "main"): - step = dict(step, when=trigger) + step = dict(step, when = trigger) return step +def build_backend_step(edition, ver_mode, variants = None): + """Build the backend code using the Grafana build tool. -def build_backend_step(edition, ver_mode, variants=None): - variants_str = '' + Args: + edition: controls which edition of the backend is built. + ver_mode: if ver_mode != 'release', pass the DRONE_BUILD_NUMBER environment + variable as the value for the --build-id option. + TODO: is this option actually used by the build-backend subcommand? + variants: a list of variants be passed to the build-backend subcommand + using the --variants option. + Defaults to None. + + Returns: + Drone step. + """ + variants_str = "" if variants: - variants_str = ' --variants {}'.format(','.join(variants)) + variants_str = " --variants {}".format(",".join(variants)) # TODO: Convert number of jobs to percentage - if ver_mode == 'release': + if ver_mode == "release": cmds = [ - './bin/build build-backend --jobs 8 --edition {} ${{DRONE_TAG}}'.format( + "./bin/build build-backend --jobs 8 --edition {} ${{DRONE_TAG}}".format( edition, ), ] else: - build_no = '${DRONE_BUILD_NUMBER}' + build_no = "${DRONE_BUILD_NUMBER}" cmds = [ - './bin/build build-backend --jobs 8 --edition {} --build-id {}{}'.format( - edition, build_no, variants_str, + "./bin/build build-backend --jobs 8 --edition {} --build-id {}{}".format( + edition, + build_no, + variants_str, ), ] return { - 'name': 'build-backend' + enterprise2_suffix(edition), - 'image': build_image, - 'depends_on': [ - 'wire-install', - 'compile-build-cmd', + "name": "build-backend" + enterprise2_suffix(edition), + "image": build_image, + "depends_on": [ + "wire-install", + "compile-build-cmd", ], - 'commands': cmds, + "commands": cmds, } - def build_frontend_step(edition, ver_mode): - build_no = '${DRONE_BUILD_NUMBER}' + """Build the frontend code using the Grafana build tool. + + Args: + edition: controls which edition of the frontend is built. + ver_mode: if ver_mode != 'release', use the DRONE_BUILD_NUMBER environment + variable as a build identifier. + + Returns: + Drone step. + """ + build_no = "${DRONE_BUILD_NUMBER}" # TODO: Use percentage for num jobs - if ver_mode == 'release': + if ver_mode == "release": cmds = [ - './bin/build build-frontend --jobs 8 ' + \ - '--edition {} ${{DRONE_TAG}}'.format(edition), + "./bin/build build-frontend --jobs 8 " + + "--edition {} ${{DRONE_TAG}}".format(edition), ] else: cmds = [ - './bin/build build-frontend --jobs 8 --edition {} '.format(edition) + \ - '--build-id {}'.format(build_no), + "./bin/build build-frontend --jobs 8 --edition {} ".format(edition) + + "--build-id {}".format(build_no), ] return { - 'name': 'build-frontend', - 'image': build_image, - 'environment': { - 'NODE_OPTIONS': '--max_old_space_size=8192', + "name": "build-frontend", + "image": build_image, + "environment": { + "NODE_OPTIONS": "--max_old_space_size=8192", }, - 'depends_on': [ - 'compile-build-cmd', - 'yarn-install', + "depends_on": [ + "compile-build-cmd", + "yarn-install", ], - 'commands': cmds, + "commands": cmds, } - def build_frontend_package_step(edition, ver_mode): - build_no = '${DRONE_BUILD_NUMBER}' + """Build the frontend packages using the Grafana build tool. + + Args: + edition: controls which edition of the frontend is built. + ver_mode: if ver_mode != 'release', use the DRONE_BUILD_NUMBER environment + variable as a build identifier. + + Returns: + Drone step. + """ + build_no = "${DRONE_BUILD_NUMBER}" # TODO: Use percentage for num jobs - if ver_mode == 'release': + if ver_mode == "release": cmds = [ - './bin/build build-frontend-packages --jobs 8 ' + \ - '--edition {} ${{DRONE_TAG}}'.format(edition), + "./bin/build build-frontend-packages --jobs 8 " + + "--edition {} ${{DRONE_TAG}}".format(edition), ] else: cmds = [ - './bin/build build-frontend-packages --jobs 8 --edition {} '.format(edition) + \ - '--build-id {}'.format(build_no), + "./bin/build build-frontend-packages --jobs 8 --edition {} ".format(edition) + + "--build-id {}".format(build_no), ] return { - 'name': 'build-frontend-packages', - 'image': build_image, - 'environment': { - 'NODE_OPTIONS': '--max_old_space_size=8192', + "name": "build-frontend-packages", + "image": build_image, + "environment": { + "NODE_OPTIONS": "--max_old_space_size=8192", }, - 'depends_on': [ - 'compile-build-cmd', - 'yarn-install', + "depends_on": [ + "compile-build-cmd", + "yarn-install", ], - 'commands': cmds, + "commands": cmds, } - def build_plugins_step(edition, ver_mode): - if ver_mode!='pr': + if ver_mode != "pr": env = { - 'GRAFANA_API_KEY': from_secret('grafana_api_key'), + "GRAFANA_API_KEY": from_secret("grafana_api_key"), } else: env = None return { - 'name': 'build-plugins', - 'image': build_image, - 'environment': env, - 'depends_on': [ - 'compile-build-cmd', - 'yarn-install', + "name": "build-plugins", + "image": build_image, + "environment": env, + "depends_on": [ + "compile-build-cmd", + "yarn-install", ], - 'commands': [ + "commands": [ # TODO: Use percentage for num jobs - './bin/build build-plugins --jobs 8 --edition {}'.format(edition), + "./bin/build build-plugins --jobs 8 --edition {}".format(edition), ], } - -def test_backend_step(edition): - if edition == 'enterprise2': - return { - 'name': 'test-backend' + enterprise2_suffix(edition), - 'image': build_image, - 'depends_on': [ - 'wire-install', - ], - 'commands': [ - 'go test -tags=pro -covermode=atomic -timeout=30m ./pkg/...', - ], - } - else: - return { - 'name': 'test-backend' + enterprise2_suffix(edition), - 'image': build_image, - 'depends_on': [ - 'wire-install', - ], - 'commands': [ - 'go test -short -covermode=atomic -timeout=30m ./pkg/...', - ], - } - - - -def test_backend_integration_step(edition): +def test_backend_step(): return { - 'name': 'test-backend-integration', - 'image': build_image, - 'depends_on': [ - 'wire-install', + "name": "test-backend", + "image": build_image, + "depends_on": [ + "wire-install", ], - 'commands': [ - 'go test -run Integration -covermode=atomic -timeout=30m ./pkg/...', + "commands": [ + "go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/...", ], } -def betterer_frontend_step(edition="oss"): +def test_backend_integration_step(): + return { + "name": "test-backend-integration", + "image": build_image, + "depends_on": [ + "wire-install", + ], + "commands": [ + "go test -run Integration -covermode=atomic -timeout=5m ./pkg/...", + ], + } + +def betterer_frontend_step(edition = "oss"): + """Run betterer on frontend code. + + Args: + edition: controls whether enterprise code is also included in the source. + Defaults to 'oss'. + + Returns: + Drone step. + """ deps = [] if edition == "enterprise": - deps.extend(['init-enterprise']) - deps.extend(['yarn-install']) + deps.extend(["init-enterprise"]) + deps.extend(["yarn-install"]) return { - 'name': 'betterer-frontend', - 'image': build_image, - 'depends_on': deps, - 'commands': [ - 'yarn betterer ci', + "name": "betterer-frontend", + "image": build_image, + "depends_on": deps, + "commands": [ + "yarn betterer ci", ], - 'failure': 'ignore', } +def test_frontend_step(edition = "oss"): + """Runs tests on frontend code. + Args: + edition: controls whether enterprise code is also included in the source. + Defaults to 'oss'. -def test_frontend_step(edition="oss"): + Returns: + Drone step. + """ deps = [] if edition == "enterprise": - deps.extend(['init-enterprise']) - deps.extend(['yarn-install']) + deps.extend(["init-enterprise"]) + deps.extend(["yarn-install"]) return { - 'name': 'test-frontend', - 'image': build_image, - 'environment': { - 'TEST_MAX_WORKERS': '50%', + "name": "test-frontend", + "image": build_image, + "environment": { + "TEST_MAX_WORKERS": "50%", }, - 'depends_on': deps, - 'commands': [ - 'yarn run ci:test-frontend', + "depends_on": deps, + "commands": [ + "yarn run ci:test-frontend", ], } - def lint_frontend_step(): return { - 'name': 'lint-frontend', - 'image': build_image, - 'environment': { - 'TEST_MAX_WORKERS': '50%', + "name": "lint-frontend", + "image": build_image, + "environment": { + "TEST_MAX_WORKERS": "50%", }, - 'depends_on': [ - 'yarn-install', + "depends_on": [ + "yarn-install", ], - 'commands': [ - 'yarn run prettier:check', - 'yarn run lint', - 'yarn run i18n:compile', # TODO: right place for this? - 'yarn run typecheck', + "commands": [ + "yarn run prettier:check", + "yarn run lint", + "yarn run i18n:compile", # TODO: right place for this? + "yarn run typecheck", ], } +def test_a11y_frontend_step(ver_mode, port = 3001): + """Runs automated accessiblity tests against the frontend. -def test_a11y_frontend_step(ver_mode, edition, port=3001): + Args: + ver_mode: controls whether the step is blocking or just reporting. + If ver_mode == 'pr', the step causes the pipeline to fail. + port: which port to grafana-server is expected to be listening on. + Defaults to 3001. + + Returns: + Drone step. + """ commands = [ - 'yarn wait-on http://$HOST:$PORT', + "yarn wait-on http://$HOST:$PORT", ] - failure = 'ignore' - if ver_mode == 'pr': - commands.extend([ - 'pa11y-ci --config .pa11yci-pr.conf.js', - ]) - failure = 'always' + failure = "ignore" + if ver_mode == "pr": + commands.extend( + [ + "pa11y-ci --config .pa11yci-pr.conf.js", + ], + ) + failure = "always" else: - commands.extend([ - 'pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json', - ]) + commands.extend( + [ + "pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json", + ], + ) return { - 'name': 'test-a11y-frontend' + enterprise2_suffix(edition), - 'image': 'grafana/docker-puppeteer:1.1.0', - 'depends_on': [ - 'grafana-server' + enterprise2_suffix(edition), + "name": "test-a11y-frontend", + # TODO which image should be used? + "image": "grafana/docker-puppeteer:1.1.0", + "depends_on": [ + "grafana-server", ], - 'environment': { - 'GRAFANA_MISC_STATS_API_KEY': from_secret('grafana_misc_stats_api_key'), - 'HOST': 'grafana-server' + enterprise2_suffix(edition), - 'PORT': port, + "environment": { + "GRAFANA_MISC_STATS_API_KEY": from_secret("grafana_misc_stats_api_key"), + "HOST": "grafana-server", + "PORT": port, }, - 'failure': failure, - 'commands': commands, + "failure": failure, + "commands": commands, } +def frontend_metrics_step(trigger = None): + """Reports frontend metrics to Grafana Cloud. -def frontend_metrics_step(edition, trigger=None): - if edition in ('enterprise', 'enterprise2'): - return None + Args: + trigger: a Drone trigger for the step. + Defaults to None. + Returns: + Drone step. + """ step = { - 'name': 'publish-frontend-metrics', - 'image': build_image, - 'depends_on': [ - 'test-a11y-frontend' + enterprise2_suffix(edition), + "name": "publish-frontend-metrics", + "image": build_image, + "depends_on": [ + "test-a11y-frontend", ], - 'environment': { - 'GRAFANA_MISC_STATS_API_KEY': from_secret('grafana_misc_stats_api_key'), + "environment": { + "GRAFANA_MISC_STATS_API_KEY": from_secret("grafana_misc_stats_api_key"), }, - 'failure': 'ignore', - 'commands': [ - './scripts/ci-frontend-metrics.sh | ./bin/build publish-metrics $${GRAFANA_MISC_STATS_API_KEY}', + "failure": "ignore", + "commands": [ + "./scripts/ci-frontend-metrics.sh | ./bin/build publish-metrics $${GRAFANA_MISC_STATS_API_KEY}", ], } if trigger: - step = dict(step, when=trigger) + step = dict(step, when = trigger) return step - def codespell_step(): return { - 'name': 'codespell', - 'image': build_image, - 'commands': [ + "name": "codespell", + "image": build_image, + "commands": [ # Important: all words have to be in lowercase, and separated by "\n". 'echo -e "unknwon\nreferer\nerrorstring\neror\niam\nwan" > words_to_ignore.txt', - 'codespell -I words_to_ignore.txt docs/', - 'rm words_to_ignore.txt', + "codespell -I words_to_ignore.txt docs/", + "rm words_to_ignore.txt", ], } +def package_step(edition, ver_mode): + """Packages Grafana with the Grafana build tool. -def package_step(edition, ver_mode, include_enterprise2=False, variants=None): + Args: + edition: controls which edition of Grafana is packaged. + ver_mode: controls whether the packages are signed for a release. + If ver_mode != 'release', use the DRONE_BUILD_NUMBER environment + variable as a build identifier. + + Returns: + Drone step. + """ deps = [ - 'build-plugins', - 'build-backend', - 'build-frontend', - 'build-frontend-packages', + "build-plugins", + "build-backend" + enterprise2_suffix(edition), + "build-frontend", + "build-frontend-packages", ] - if include_enterprise2: - sfx = '-enterprise2' - deps.extend([ - 'build-backend' + sfx, - ]) - variants_str = '' - if variants: - variants_str = ' --variants {}'.format(','.join(variants)) - - if ver_mode in ('main', 'release', 'release-branch'): - sign_args = ' --sign' + if ver_mode in ("main", "release", "release-branch"): + sign_args = " --sign" env = { - 'GRAFANA_API_KEY': from_secret('grafana_api_key'), - 'GPG_PRIV_KEY': from_secret('packages_gpg_private_key'), - 'GPG_PUB_KEY': from_secret('packages_gpg_public_key'), - 'GPG_KEY_PASSWORD': from_secret('packages_gpg_passphrase'), + "GRAFANA_API_KEY": from_secret("grafana_api_key"), + "GPG_PRIV_KEY": from_secret("packages_gpg_private_key"), + "GPG_PUB_KEY": from_secret("packages_gpg_public_key"), + "GPG_KEY_PASSWORD": from_secret("packages_gpg_passphrase"), } - test_args = '' + test_args = "" else: - sign_args = '' + sign_args = "" env = None - test_args = '. scripts/build/gpg-test-vars.sh && ' + + # TODO: env vars no longer needed by build if not signing + test_args = ". scripts/build/gpg-test-vars.sh && " # TODO: Use percentage for jobs - if ver_mode == 'release': + if ver_mode == "release": cmds = [ - '{}./bin/build package --jobs 8 --edition {} '.format(test_args, edition) + \ - '{} ${{DRONE_TAG}}'.format( - sign_args - ), + "{}./bin/build package --jobs 8 --edition {} ".format(test_args, edition) + + "{} ${{DRONE_TAG}}".format(sign_args), ] else: - build_no = '${DRONE_BUILD_NUMBER}' + build_no = "${DRONE_BUILD_NUMBER}" cmds = [ - '{}./bin/build package --jobs 8 --edition {} '.format(test_args, edition) + \ - '--build-id {}{}{}'.format(build_no, variants_str, sign_args), + "{}./bin/build package --jobs 8 --edition {} ".format(test_args, edition) + + "--build-id {}{}".format(build_no, sign_args), ] return { - 'name': 'package' + enterprise2_suffix(edition), - 'image': build_image, - 'depends_on': deps, - 'environment': env, - 'commands': cmds, + "name": "package" + enterprise2_suffix(edition), + "image": build_image, + "depends_on": deps, + "environment": env, + "commands": cmds, } +def grafana_server_step(edition, port = 3001): + """Runs the grafana-server binary as a service. -def grafana_server_step(edition, port=3001): - package_file_pfx = '' - if edition == 'enterprise2': - package_file_pfx = 'grafana' + enterprise2_suffix(edition) - elif edition == 'enterprise': - package_file_pfx = 'grafana-' + edition + Args: + edition: controls which edition of grafana-server to run. + port: port to listen on. + Defaults to 3001. - environment = { - 'PORT': port, - 'ARCH': 'linux-amd64' - } - if package_file_pfx: - environment['RUNDIR'] = 'scripts/grafana-server/tmp-{}'.format(package_file_pfx) + Returns: + Drone step. + """ + environment = {"PORT": port, "ARCH": "linux-amd64"} + if edition == "enterprise": + environment["RUNDIR"] = "scripts/grafana-server/tmp-grafana-enterprise" return { - 'name': 'grafana-server' + enterprise2_suffix(edition), - 'image': build_image, - 'detach': True, - 'depends_on': [ - 'build-plugins', - 'build-backend', - 'build-frontend', - 'build-frontend-packages', + "name": "grafana-server", + "image": build_image, + "detach": True, + "depends_on": [ + "build-plugins", + "build-backend", + "build-frontend", + "build-frontend-packages", ], - 'environment': environment, - 'commands': [ - './scripts/grafana-server/start-server', + "environment": environment, + "commands": [ + "./scripts/grafana-server/start-server", ], } - -def e2e_tests_step(suite, edition, port=3001, tries=None): - cmd = './bin/build e2e-tests --port {} --suite {}'.format(port, suite) +def e2e_tests_step(suite, port = 3001, tries = None): + cmd = "./bin/build e2e-tests --port {} --suite {}".format(port, suite) if tries: - cmd += ' --tries {}'.format(tries) + cmd += " --tries {}".format(tries) return { - 'name': 'end-to-end-tests-{}'.format(suite) + enterprise2_suffix(edition), - 'image': 'cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97', - 'depends_on': [ - 'grafana-server', + "name": "end-to-end-tests-{}".format(suite), + "image": "cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97", + "depends_on": [ + "grafana-server", ], - 'environment': { - 'HOST': 'grafana-server' + enterprise2_suffix(edition), + "environment": { + "HOST": "grafana-server", }, - 'commands': [ - 'apt-get install -y netcat', + "commands": [ + "apt-get install -y netcat", cmd, ], } +def cloud_plugins_e2e_tests_step(suite, cloud, trigger = None): + """Run cloud plugins end-to-end tests. + + Args: + suite: affects the pipeline name. + TODO: check if this actually affects step behavior. + cloud: used to determine cloud provider specific tests. + trigger: a Drone trigger for the step. + Defaults to None. + + Returns: + Drone step. + """ + environment = {} + when = {} + if trigger: + when = trigger + if cloud == "azure": + environment = { + "CYPRESS_CI": "true", + "HOST": "grafana-server", + "GITHUB_TOKEN": from_secret("github_token_pr"), + "AZURE_SP_APP_ID": from_secret("azure_sp_app_id"), + "AZURE_SP_PASSWORD": from_secret("azure_sp_app_pw"), + "AZURE_TENANT": from_secret("azure_tenant"), + } + when = dict( + when, + paths = { + "include": [ + "pkg/tsdb/azuremonitor/**", + "public/app/plugins/datasource/grafana-azure-monitor-datasource/**", + "e2e/cloud-plugins-suite/azure-monitor.spec.ts", + ], + }, + ) + branch = "${DRONE_SOURCE_BRANCH}".replace("/", "-") + step = { + "name": "end-to-end-tests-{}-{}".format(suite, cloud), + "image": "us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest", + "depends_on": [ + "grafana-server", + ], + "environment": environment, + "commands": ["cd /", "./cpp-e2e/scripts/ci-run.sh {} {}".format(cloud, branch)], + } + step = dict(step, when = when) + return step def build_docs_website_step(): return { - 'name': 'build-docs-website', + "name": "build-docs-website", # Use latest revision here, since we want to catch if it breaks - 'image': 'grafana/docs-base:latest', - 'commands': [ - 'mkdir -p /hugo/content/docs/grafana', - 'cp -r docs/sources/* /hugo/content/docs/grafana/latest/', - 'cd /hugo && make prod', + "image": "grafana/docs-base:latest", + "commands": [ + "mkdir -p /hugo/content/docs/grafana", + "cp -r docs/sources/* /hugo/content/docs/grafana/latest/", + "cd /hugo && make prod", ], } - -def copy_packages_for_docker_step(): +def copy_packages_for_docker_step(edition = None): return { - 'name': 'copy-packages-for-docker', - 'image': build_image, - 'depends_on': [ - 'package', + "name": "copy-packages-for-docker", + "image": build_image, + "depends_on": [ + "package" + enterprise2_suffix(edition), ], - 'commands': [ - 'ls dist/*.tar.gz*', - 'cp dist/*.tar.gz* packaging/docker/', + "commands": [ + "ls dist/*.tar.gz*", + "cp dist/*.tar.gz* packaging/docker/", ], } +def build_docker_images_step(edition, archs = None, ubuntu = False, publish = False): + """Build Docker images using the Grafana build tool. -def build_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publish=False): - cmd = './bin/build build-docker --edition {}'.format(edition) + Args: + edition: controls which repository the image is published to. + archs: a list of architectures to build the image for. + Defaults to None. + ubuntu: controls whether the final image is built from an Ubuntu base image. + Defaults to False. + publish: controls whether the built image is saved to a pre-release repository. + Defaults to False. + + Returns: + Drone step. + """ + cmd = "./bin/build build-docker --edition {}".format(edition) if publish: - cmd += ' --shouldSave' + cmd += " --shouldSave" - ubuntu_sfx = '' + ubuntu_sfx = "" if ubuntu: - ubuntu_sfx = '-ubuntu' - cmd += ' --ubuntu' + ubuntu_sfx = "-ubuntu" + cmd += " --ubuntu" if archs: - cmd += ' -archs {}'.format(','.join(archs)) + cmd += " -archs {}".format(",".join(archs)) + + environment = { + "GCP_KEY": from_secret("gcp_key"), + } + + if edition == "enterprise2": + environment.update( + {"DOCKER_ENTERPRISE2_REPO": from_secret("docker_enterprise2_repo")}, + ) return { - 'name': 'build-docker-images' + ubuntu_sfx, - 'image': 'google/cloud-sdk', - 'depends_on': [ - 'copy-packages-for-docker', - 'compile-build-cmd', + "name": "build-docker-images" + ubuntu_sfx, + "image": "google/cloud-sdk", + "depends_on": [ + "copy-packages-for-docker", + "compile-build-cmd", ], - 'commands': [ - cmd - ], - 'volumes': [{ - 'name': 'docker', - 'path': '/var/run/docker.sock' - }], - 'environment': { - 'GCP_KEY': from_secret('gcp_key'), - }, + "commands": [cmd], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], + "environment": environment, } def fetch_images_step(edition): return { - 'name': 'fetch-images-{}'.format(edition), - 'image': 'google/cloud-sdk', - 'environment': { - 'GCP_KEY': from_secret('gcp_key'), - 'DOCKER_USER': from_secret('docker_username'), - 'DOCKER_PASSWORD': from_secret('docker_password'), + "name": "fetch-images-{}".format(edition), + "image": "google/cloud-sdk", + "environment": { + "GCP_KEY": from_secret("gcp_key"), + "DOCKER_USER": from_secret("docker_username"), + "DOCKER_PASSWORD": from_secret("docker_password"), + "DOCKER_ENTERPRISE2_REPO": from_secret("docker_enterprise2_repo"), }, - 'commands': ['./bin/build artifacts docker fetch --edition {}'.format(edition)], - 'depends_on': ['compile-build-cmd'], - 'volumes': [{ - 'name': 'docker', - 'path': '/var/run/docker.sock' - }], + "commands": ["./bin/build artifacts docker fetch --edition {}".format(edition)], + "depends_on": ["compile-build-cmd"], + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], } +def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None): + """Generates a step for publishing public Docker images with grabpl. -def publish_images_step(edition, ver_mode, mode, docker_repo, trigger=None): - if mode == 'security': - mode = '--{} '.format(mode) + Args: + edition: controls which version of an image is fetched in the case of a release. + It also controls which publishing implementation is used. + ver_mode: controls whether the image needs to be built or retrieved from a previous build. + If ver_mode == 'release', the previously built image is fetched instead of being built again. + mode: uses to control the publishing of security images when mode == 'security'. + docker_repo: the Docker image name. + It is combined with the 'grafana/' library prefix. + trigger: a Drone trigger for the pipeline. + Defaults to None. + + Returns: + Drone step. + """ + name = docker_repo + docker_repo = "grafana/{}".format(docker_repo) + if mode == "security": + mode = "--{} ".format(mode) else: - mode = '' + mode = "" - cmd = './bin/grabpl artifacts docker publish {}--dockerhub-repo {}'.format( - mode, docker_repo) + environment = { + "GCP_KEY": from_secret("gcp_key"), + "DOCKER_USER": from_secret("docker_username"), + "DOCKER_PASSWORD": from_secret("docker_password"), + } - if ver_mode == 'release': - deps = ['fetch-images-{}'.format(edition)] - cmd += ' --version-tag ${TAG}' - else: - deps = ['build-docker-images', 'build-docker-images-ubuntu'] + cmd = "./bin/grabpl artifacts docker publish {}--dockerhub-repo {}".format( + mode, + docker_repo, + ) + + deps = ["build-docker-images", "build-docker-images-ubuntu"] + if ver_mode == "release": + deps = ["fetch-images-{}".format(edition)] + cmd += " --version-tag ${DRONE_TAG}" + + if edition == "enterprise2": + name = edition + docker_repo = "$${DOCKER_ENTERPRISE2_REPO}" + environment.update( + { + "GCP_KEY": from_secret("gcp_key_hg"), + "DOCKER_ENTERPRISE2_REPO": from_secret("docker_enterprise2_repo"), + }, + ) + cmd = "./bin/build artifacts docker publish-enterprise2 --dockerhub-repo {}".format( + docker_repo, + ) step = { - 'name': 'publish-images-{}'.format(docker_repo), - 'image': 'google/cloud-sdk', - 'environment': { - 'GCP_KEY': from_secret('gcp_key'), - 'DOCKER_USER': from_secret('docker_username'), - 'DOCKER_PASSWORD': from_secret('docker_password'), - }, - 'commands': [cmd], - 'depends_on': deps, - 'volumes': [{ - 'name': 'docker', - 'path': '/var/run/docker.sock' - }], + "name": "publish-images-{}".format(name), + "image": "google/cloud-sdk", + "environment": environment, + "commands": [cmd], + "depends_on": deps, + "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], } if trigger and ver_mode in ("release-branch", "main"): - step = dict(step, when=trigger) + step = dict(step, when = trigger) return step - -def postgres_integration_tests_step(edition, ver_mode): +def postgres_integration_tests_step(): cmds = [ - 'apt-get update', - 'apt-get install -yq postgresql-client', - 'dockerize -wait tcp://postgres:5432 -timeout 120s', - 'psql -p 5432 -h postgres -U grafanatest -d grafanatest -f ' + - 'devenv/docker/blocks/postgres_tests/setup.sql', - # Make sure that we don't use cached results for another database - 'go clean -testcache', - "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=30m {}'", - ] + "apt-get update", + "apt-get install -yq postgresql-client", + "dockerize -wait tcp://postgres:5432 -timeout 120s", + "psql -p 5432 -h postgres -U grafanatest -d grafanatest -f " + + "devenv/docker/blocks/postgres_tests/setup.sql", + # Make sure that we don't use cached results for another database + "go clean -testcache", + "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=5m {}'", + ] return { - 'name': 'postgres-integration-tests', - 'image': build_image, - 'depends_on': ['wire-install'], - 'environment': { - 'PGPASSWORD': 'grafanatest', - 'GRAFANA_TEST_DB': 'postgres', - 'POSTGRES_HOST': 'postgres', + "name": "postgres-integration-tests", + "image": build_image, + "depends_on": ["wire-install"], + "environment": { + "PGPASSWORD": "grafanatest", + "GRAFANA_TEST_DB": "postgres", + "POSTGRES_HOST": "postgres", }, - 'commands': cmds, + "commands": cmds, } - -def mysql_integration_tests_step(edition, ver_mode): +def mysql_integration_tests_step(): cmds = [ - 'apt-get update', - 'apt-get install -yq default-mysql-client', - 'dockerize -wait tcp://mysql:3306 -timeout 120s', - 'cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root -prootpass', - # Make sure that we don't use cached results for another database - 'go clean -testcache', - "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=30m {}'", - ] + "apt-get update", + "apt-get install -yq default-mysql-client", + "dockerize -wait tcp://mysql:3306 -timeout 120s", + "cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root -prootpass", + # Make sure that we don't use cached results for another database + "go clean -testcache", + "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=5m {}'", + ] return { - 'name': 'mysql-integration-tests', - 'image': build_image, - 'depends_on': ['wire-install'], - 'environment': { - 'GRAFANA_TEST_DB': 'mysql', - 'MYSQL_HOST': 'mysql', + "name": "mysql-integration-tests", + "image": build_image, + "depends_on": ["wire-install"], + "environment": { + "GRAFANA_TEST_DB": "mysql", + "MYSQL_HOST": "mysql", }, - 'commands': cmds, + "commands": cmds, } - def redis_integration_tests_step(): return { - 'name': 'redis-integration-tests', - 'image': build_image, - 'depends_on': ['wire-install'], - 'environment': { - 'REDIS_URL': 'redis://redis:6379/0', + "name": "redis-integration-tests", + "image": build_image, + "depends_on": ["wire-install"], + "environment": { + "REDIS_URL": "redis://redis:6379/0", }, - 'commands': [ - 'dockerize -wait tcp://redis:6379/0 -timeout 120s', - './bin/grabpl integration-tests', + "commands": [ + "dockerize -wait tcp://redis:6379/0 -timeout 120s", + "go clean -testcache", + "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=5m {}'", ], } - def memcached_integration_tests_step(): return { - 'name': 'memcached-integration-tests', - 'image': build_image, - 'depends_on': ['wire-install'], - 'environment': { - 'MEMCACHED_HOSTS': 'memcached:11211', + "name": "memcached-integration-tests", + "image": build_image, + "depends_on": ["wire-install"], + "environment": { + "MEMCACHED_HOSTS": "memcached:11211", }, - 'commands': [ - 'dockerize -wait tcp://memcached:11211 -timeout 120s', - './bin/grabpl integration-tests', + "commands": [ + "dockerize -wait tcp://memcached:11211 -timeout 120s", + "go clean -testcache", + "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=5m {}'", ], } +def release_canary_npm_packages_step(trigger = None): + """Releases canary NPM packages. -def release_canary_npm_packages_step(edition, trigger=None): - if edition in ('enterprise', 'enterprise2'): - return None + Args: + trigger: a Drone trigger for the step. + Defaults to None. + Returns: + Drone step. + """ step = { - 'name': 'release-canary-npm-packages', - 'image': build_image, - 'depends_on': end_to_end_tests_deps(edition), - 'environment': { - 'NPM_TOKEN': from_secret('npm_token'), + "name": "release-canary-npm-packages", + "image": build_image, + "depends_on": end_to_end_tests_deps(), + "environment": { + "NPM_TOKEN": from_secret("npm_token"), }, - 'commands': [ - './scripts/circle-release-canary-packages.sh', + "commands": [ + "./scripts/circle-release-canary-packages.sh", ], } if trigger: - step = dict(step, when=trigger) + step = dict(step, when = trigger) return step - def enterprise2_suffix(edition): - if edition == 'enterprise2': - return '-{}'.format(edition) - return '' + if edition == "enterprise2": + return "-{}".format(edition) + return "" +def upload_packages_step(edition, ver_mode, trigger = None): + """Upload packages to object storage. -def upload_packages_step(edition, ver_mode, trigger=None): - if ver_mode == 'main' and edition in ('enterprise', 'enterprise2'): - return None - - deps = [] - if edition in 'enterprise2' or not end_to_end_tests_deps(edition): - deps.extend([ - 'package' + enterprise2_suffix(edition), - ]) - else: - deps.extend(end_to_end_tests_deps(edition)) + Args: + edition: controls which edition of Grafana packages to upload. + ver_mode: when ver_mode == 'main', inhibit upload of enterprise + edition packages when executed. + trigger: a Drone trigger for the step. + Defaults to None. + Returns: + Drone step. + """ step = { - 'name': 'upload-packages' + enterprise2_suffix(edition), - 'image': publish_image, - 'depends_on': deps, - 'environment': { - 'GCP_KEY': from_secret('gcp_key'), - 'PRERELEASE_BUCKET': from_secret('prerelease_bucket'), + "name": "upload-packages" + enterprise2_suffix(edition), + "image": publish_image, + "depends_on": end_to_end_tests_deps(), + "environment": { + "GCP_KEY": from_secret("gcp_key"), + "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), }, - 'commands': ['./bin/grabpl upload-packages --edition {}'.format(edition),], + "commands": [ + "./bin/build upload-packages --edition {}".format(edition), + ], } if trigger and ver_mode in ("release-branch", "main"): - step = dict(step, when=trigger) + step = dict(step, when = trigger) return step - -def publish_packages_step(edition, ver_mode): - if ver_mode == 'release': - cmd = './bin/build publish packages --edition {} --gcp-key /tmp/gcpkey.json ${{DRONE_TAG}}'.format( - edition, - ) - elif ver_mode == 'main': - build_no = '${DRONE_BUILD_NUMBER}' - cmd = './bin/build publish packages --edition {} --gcp-key /tmp/gcpkey.json --build-id {}'.format( - edition, build_no, - ) - else: - fail('Unexpected version mode {}'.format(ver_mode)) - - return { - 'name': 'publish-packages-{}'.format(edition), - 'image': publish_image, - 'depends_on': [ - 'compile-build-cmd', - ], - 'environment': { - 'GRAFANA_COM_API_KEY': from_secret('grafana_api_key'), - 'GCP_KEY': from_secret('gcp_key'), - 'GPG_PRIV_KEY': from_secret('packages_gpg_private_key'), - 'GPG_PUB_KEY': from_secret('packages_gpg_public_key'), - 'GPG_KEY_PASSWORD': from_secret('packages_gpg_passphrase'), - }, - 'commands': [ - cmd, - ], - } - def publish_grafanacom_step(edition, ver_mode): - if ver_mode == 'release': - cmd = './bin/build publish grafana-com --edition {} ${{DRONE_TAG}}'.format( + """Publishes Grafana packages to grafana.com. + + Args: + edition: controls which edition of Grafana to publish to. + ver_mode: if ver_mode == 'main', pass the DRONE_BUILD_NUMBER environment + variable as the value for the --build-id option. + TODO: is this actually used by the grafanacom subcommand? I think it might + just use the environment varaiable directly. + + Returns: + Drone step. + """ + if ver_mode == "release": + cmd = "./bin/build publish grafana-com --edition {} ${{DRONE_TAG}}".format( edition, ) - elif ver_mode == 'main': - build_no = '${DRONE_BUILD_NUMBER}' - cmd = './bin/build publish grafana-com --edition {} --build-id {}'.format( - edition, build_no, + elif ver_mode == "main": + build_no = "${DRONE_BUILD_NUMBER}" + cmd = "./bin/build publish grafana-com --edition {} --build-id {}".format( + edition, + build_no, ) else: - fail('Unexpected version mode {}'.format(ver_mode)) + fail("Unexpected version mode {}".format(ver_mode)) return { - 'name': 'publish-grafanacom-{}'.format(edition), - 'image': publish_image, - 'depends_on': [ - 'publish-packages-{}'.format(edition), + "name": "publish-grafanacom-{}".format(edition), + "image": publish_image, + "depends_on": [ + "publish-linux-packages-deb", + "publish-linux-packages-rpm", ], - 'environment': { - 'GRAFANA_COM_API_KEY': from_secret('grafana_api_key'), - 'GCP_KEY': from_secret('gcp_key'), + "environment": { + "GRAFANA_COM_API_KEY": from_secret("grafana_api_key"), + "GCP_KEY": from_secret("gcp_key"), }, - 'commands': [ + "commands": [ cmd, ], } -def publish_linux_packages_step(edition, package_manager='deb'): +def publish_linux_packages_step(edition, package_manager = "deb"): return { - 'name': 'publish-linux-packages-{}'.format(package_manager), + "name": "publish-linux-packages-{}".format(package_manager), # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image - 'image': 'us.gcr.io/kubernetes-dev/package-publish:latest', - 'depends_on': [ - 'grabpl' - ], - 'privileged': True, - 'failure': 'ignore', # While we're testing it - 'settings': { - 'access_key_id': from_secret('packages_access_key_id'), - 'secret_access_key': from_secret('packages_secret_access_key'), - 'service_account_json': from_secret('packages_service_account'), - 'target_bucket': 'grafana-packages', - 'deb_distribution': 'auto', - 'gpg_passphrase': from_secret('packages_gpg_passphrase'), - 'gpg_public_key': from_secret('packages_gpg_public_key'), - 'gpg_private_key': from_secret('packages_gpg_private_key'), - 'package_path': 'gs://grafana-prerelease/artifacts/downloads/*${{DRONE_TAG}}/{}/**.{}'.format(edition, package_manager) - } + "image": "us.gcr.io/kubernetes-dev/package-publish:latest", + "depends_on": ["grabpl"], + "privileged": True, + "settings": { + "access_key_id": from_secret("packages_access_key_id"), + "secret_access_key": from_secret("packages_secret_access_key"), + "service_account_json": from_secret("packages_service_account"), + "target_bucket": "grafana-packages", + "deb_distribution": "auto", + "gpg_passphrase": from_secret("packages_gpg_passphrase"), + "gpg_public_key": from_secret("packages_gpg_public_key"), + "gpg_private_key": from_secret("packages_gpg_private_key"), + "package_path": "gs://grafana-prerelease/artifacts/downloads/*${{DRONE_TAG}}/{}/**.{}".format( + edition, + package_manager, + ), + }, } - def get_windows_steps(edition, ver_mode): - init_cmds = [] - sfx = '' - if edition in ('enterprise', 'enterprise2'): - sfx = '-{}'.format(edition) - else: - init_cmds.extend([ - '$$ProgressPreference = "SilentlyContinue"', - 'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format( - grabpl_version), - ]) - steps = [ - { - 'name': 'windows-init', - 'image': wix_image, - 'commands': init_cmds, - }, - ] - if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in ( - 'release', 'release-branch', - ): - bucket = '%PRERELEASE_BUCKET%/artifacts/downloads' - if ver_mode == 'release': - ver_part = '${DRONE_TAG}' - dir = 'release' - else: - dir = 'main' - bucket = 'grafana-downloads' - build_no = 'DRONE_BUILD_NUMBER' - ver_part = '--build-id $$env:{}'.format(build_no) - installer_commands = [ - '$$gcpKey = $$env:GCP_KEY', - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json', - # gcloud fails to read the file unless converted with dos2unix - 'dos2unix gcpkey.json', - 'gcloud auth activate-service-account --key-file=gcpkey.json', - 'rm gcpkey.json', - 'cp C:\\App\\nssm-2.24.zip .', - ] - if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in ( - 'release', - ): - installer_commands.extend([ - '.\\grabpl.exe windows-installer --edition {} {}'.format(edition, ver_part), - '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', - ]) - if ver_mode == 'main': - installer_commands.extend([ - 'gsutil cp $$fname gs://{}/{}/{}/'.format(bucket, edition, dir), - 'gsutil cp "$$fname.sha256" gs://{}/{}/{}/'.format(bucket, edition, dir), - ]) - else: - installer_commands.extend([ - 'gsutil cp $$fname gs://{}/{}/{}/{}/'.format(bucket, ver_part, edition, dir), - 'gsutil cp "$$fname.sha256" gs://{}/{}/{}/{}/'.format(bucket, ver_part, edition, dir), - ]) - steps.append({ - 'name': 'build-windows-installer', - 'image': wix_image, - 'depends_on': [ - 'windows-init', - ], - 'environment': { - 'GCP_KEY': from_secret('gcp_key'), - 'PRERELEASE_BUCKET': from_secret(prerelease_bucket), - 'GITHUB_TOKEN': from_secret('github_token') - }, - 'commands': installer_commands, - }) + """Generate the list of Windows steps. - if edition in ('enterprise', 'enterprise2'): - if ver_mode == 'release': - committish = '${DRONE_TAG}' - elif ver_mode == 'release-branch': - committish = '$$env:DRONE_BRANCH' + Args: + edition: used to differentiate steps for different Grafana editions. + ver_mode: used to differentiate steps for different version modes. + + Returns: + List of Drone steps. + """ + steps = [ + identify_runner_step("windows"), + ] + + if edition in ("enterprise", "enterprise2"): + if ver_mode == "release": + source = "${DRONE_TAG}" + elif ver_mode == "release-branch": + source = "$$env:DRONE_BRANCH" else: - committish = '$$env:DRONE_COMMIT' + source = "$$env:DRONE_COMMIT" + # For enterprise, we have to clone both OSS and enterprise and merge the latter into the former - download_grabpl_step_cmds = [ + download_grabpl_cmds = [ '$$ProgressPreference = "SilentlyContinue"', - 'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format( - grabpl_version), + "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( + grabpl_version, + ), ] + clone_cmds = [ 'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"', + "cd grafana-enterprise", + "git checkout {}".format(source), ] - clone_cmds.extend([ - 'cd grafana-enterprise', - 'git checkout {}'.format(committish), - ]) - steps.insert(0, { - 'name': 'clone', - 'image': wix_image, - 'environment': { - 'GITHUB_TOKEN': from_secret(github_token), - }, - 'commands': download_grabpl_step_cmds + clone_cmds, - }) - steps[1]['depends_on'] = [ - 'clone', - ] - steps[1]['commands'].extend([ + + init_cmds = [ # Need to move grafana-enterprise out of the way, so directory is empty and can be cloned into - 'cp -r grafana-enterprise C:\\App\\grafana-enterprise', - 'rm -r -force grafana-enterprise', - 'cp grabpl.exe C:\\App\\grabpl.exe', - 'rm -force grabpl.exe', - 'C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise', - 'cp C:\\App\\grabpl.exe grabpl.exe', - ]) - if 'environment' in steps[1]: - steps[1]['environment'] + {'GITHUB_TOKEN': from_secret(github_token)} + "cp -r grafana-enterprise C:\\App\\grafana-enterprise", + "rm -r -force grafana-enterprise", + "cp grabpl.exe C:\\App\\grabpl.exe", + "rm -force grabpl.exe", + "C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise", + "cp C:\\App\\grabpl.exe grabpl.exe", + ] + + steps.extend( + [ + { + "name": "clone", + "image": wix_image, + "environment": { + "GITHUB_TOKEN": from_secret("github_token"), + }, + "commands": download_grabpl_cmds + clone_cmds, + }, + { + "name": "windows-init", + "image": wix_image, + "commands": init_cmds, + "depends_on": ["clone"], + "environment": {"GITHUB_TOKEN": from_secret("github_token")}, + }, + ], + ) + else: + init_cmds = [ + '$$ProgressPreference = "SilentlyContinue"', + "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( + grabpl_version, + ), + ] + + steps.extend( + [ + { + "name": "windows-init", + "image": wix_image, + "commands": init_cmds, + }, + ], + ) + + if ( + ver_mode == "main" and (edition not in ("enterprise", "enterprise2")) + ) or ver_mode in ( + "release", + "release-branch", + ): + bucket = "%PRERELEASE_BUCKET%/artifacts/downloads" + if ver_mode == "release": + ver_part = "${DRONE_TAG}" + dir = "release" else: - steps[1]['environment'] = {'GITHUB_TOKEN': from_secret(github_token)} + dir = "main" + bucket = "grafana-downloads" + build_no = "DRONE_BUILD_NUMBER" + ver_part = "--build-id $$env:{}".format(build_no) + installer_commands = [ + "$$gcpKey = $$env:GCP_KEY", + "[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json", + # gcloud fails to read the file unless converted with dos2unix + "dos2unix gcpkey.json", + "gcloud auth activate-service-account --key-file=gcpkey.json", + "rm gcpkey.json", + "cp C:\\App\\nssm-2.24.zip .", + ] + if ( + ver_mode == "main" and (edition not in ("enterprise", "enterprise2")) + ) or ver_mode in ("release",): + installer_commands.extend( + [ + ".\\grabpl.exe windows-installer --edition {} {}".format( + edition, + ver_part, + ), + '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', + ], + ) + if ver_mode == "main": + installer_commands.extend( + [ + "gsutil cp $$fname gs://{}/{}/{}/".format(bucket, edition, dir), + 'gsutil cp "$$fname.sha256" gs://{}/{}/{}/'.format( + bucket, + edition, + dir, + ), + ], + ) + else: + installer_commands.extend( + [ + "gsutil cp $$fname gs://{}/{}/{}/{}/".format( + bucket, + ver_part, + edition, + dir, + ), + 'gsutil cp "$$fname.sha256" gs://{}/{}/{}/{}/'.format( + bucket, + ver_part, + edition, + dir, + ), + ], + ) + steps.append( + { + "name": "build-windows-installer", + "image": wix_image, + "depends_on": [ + "windows-init", + ], + "environment": { + "GCP_KEY": from_secret("gcp_key"), + "PRERELEASE_BUCKET": from_secret(prerelease_bucket), + "GITHUB_TOKEN": from_secret("github_token"), + }, + "commands": installer_commands, + }, + ) return steps -def verify_gen_cue_step(edition): - deps = [] - if edition in ('enterprise', 'enterprise2'): - deps.extend(['init-enterprise']) +def verify_gen_cue_step(): return { - 'name': 'verify-gen-cue', - 'image': build_image, - 'depends_on': deps, - 'commands': [ - '# It is required that code generated from Thema/CUE be committed and in sync with its inputs.', - '# The following command will fail if running code generators produces any diff in output.', - 'CODEGEN_VERIFY=1 make gen-cue', + "name": "verify-gen-cue", + "image": build_image, + "depends_on": [], + "commands": [ + "# It is required that code generated from Thema/CUE be committed and in sync with its inputs.", + "# The following command will fail if running code generators produces any diff in output.", + "CODEGEN_VERIFY=1 make gen-cue", ], } def trigger_test_release(): return { - 'name': 'trigger-test-release', - 'image': build_image, - 'environment': { - 'GITHUB_TOKEN': from_secret('github_token'), - 'DOWNSTREAM_REPO': from_secret('downstream'), - 'TEST_TAG': 'v0.0.0-test', + "name": "trigger-test-release", + "image": build_image, + "environment": { + "GITHUB_TOKEN": from_secret("github_token_pr"), + "DOWNSTREAM_REPO": from_secret("downstream"), + "TEST_TAG": "v0.0.0-test", }, - 'commands': [ + "commands": [ 'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" --depth=1', - 'cd grafana-enterprise', - 'git fetch origin "refs/tags/*:refs/tags/*"', - 'git tag -d $${TEST_TAG} && git push --delete origin $${TEST_TAG} && git tag $${TEST_TAG} && git push origin $${TEST_TAG}', - 'cd -', - 'git fetch origin "refs/tags/*:refs/tags/*"', - 'git remote add downstream https://$${GITHUB_TOKEN}@github.com/grafana/$${DOWNSTREAM_REPO}.git', - 'git tag -d $${TEST_TAG} && git push --delete downstream --quiet $${TEST_TAG} && git tag $${TEST_TAG} && git push downstream $${TEST_TAG} --quiet', + "cd grafana-enterprise", + 'git fetch origin "refs/tags/*:refs/tags/*" --quiet', + "if git show-ref --tags $${TEST_TAG} --quiet; then git tag -d $${TEST_TAG} && git push --delete origin $${TEST_TAG}; fi", + "git tag $${TEST_TAG} && git push origin $${TEST_TAG}", + "cd -", + 'git fetch https://$${GITHUB_TOKEN}@github.com/grafana/grafana.git "refs/tags/*:refs/tags/*" --quiet && git fetch --quiet', + "if git show-ref --tags $${TEST_TAG} --quiet; then git tag -d $${TEST_TAG} && git push --delete https://$${GITHUB_TOKEN}@github.com/grafana/grafana.git $${TEST_TAG}; fi", + "git tag $${TEST_TAG} && git push https://$${GITHUB_TOKEN}@github.com/grafana/grafana.git $${TEST_TAG}", ], - 'failure': 'ignore', - 'when': { - 'paths': { - 'include': [ - '.drone.yml', - 'pkg/build/**', - ] + "failure": "ignore", + "when": { + "paths": { + "include": [ + ".drone.yml", + "pkg/build/**", + ], }, - 'repo': [ - 'grafana/grafana', - ] - } + "repo": [ + "grafana/grafana", + ], + }, } def artifacts_page_step(): return { - 'name': 'artifacts-page', - 'image': build_image, - 'depends_on': [ - 'grabpl', + "name": "artifacts-page", + "image": build_image, + "depends_on": [ + "compile-build-cmd", ], - 'environment': { - 'GCP_KEY': from_secret('gcp_key'), + "environment": { + "GCP_KEY": from_secret("gcp_key"), }, - 'commands': [ - './bin/grabpl artifacts-page', + "commands": [ + "./bin/build artifacts-page", ], } -def end_to_end_tests_deps(edition): - if disable_tests: - return [] +def end_to_end_tests_deps(): return [ - 'end-to-end-tests-dashboards-suite' + enterprise2_suffix(edition), - 'end-to-end-tests-panels-suite' + enterprise2_suffix(edition), - 'end-to-end-tests-smoke-tests-suite' + enterprise2_suffix(edition), - 'end-to-end-tests-various-suite' + enterprise2_suffix(edition), + "end-to-end-tests-dashboards-suite", + "end-to-end-tests-panels-suite", + "end-to-end-tests-smoke-tests-suite", + "end-to-end-tests-various-suite", ] -def compile_build_cmd(edition='oss'): +def compile_build_cmd(edition = "oss"): dependencies = [] - if edition in ('enterprise', 'enterprise2'): - dependencies = ['init-enterprise',] + if edition in ("enterprise", "enterprise2"): + dependencies = [ + "init-enterprise", + ] return { - 'name': 'compile-build-cmd', - 'image': go_image, - 'commands': [ + "name": "compile-build-cmd", + "image": go_image, + "commands": [ "go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd", ], - 'depends_on': dependencies, - 'environment': { - 'CGO_ENABLED': 0, - }, -} + "depends_on": dependencies, + "environment": { + "CGO_ENABLED": 0, + }, + } def get_trigger_storybook(ver_mode): - trigger_storybook = '' - if ver_mode == 'release': - trigger_storybook = { - 'event': ['tag'] - } + """Generate a Drone trigger for UI changes that affect the Grafana UI storybook. + + Args: + ver_mode: affects whether the trigger is event tags or changed files. + + Returns: + Drone trigger. + """ + trigger_storybook = "" + if ver_mode == "release": + trigger_storybook = {"event": ["tag"]} else: trigger_storybook = { - 'paths': { - 'include': [ - 'packages/grafana-ui/**', + "paths": { + "include": [ + "packages/grafana-ui/**", ], - } + }, } return trigger_storybook diff --git a/scripts/drone/utils/utils.star b/scripts/drone/utils/utils.star index eef792b9207..e3c79beee89 100644 --- a/scripts/drone/utils/utils.star +++ b/scripts/drone/utils/utils.star @@ -1,91 +1,136 @@ +""" +This module contains utility functions for generating Drone pipelines. +""" + load( - 'scripts/drone/steps/lib.star', - 'download_grabpl_step', - 'slack_step', + "scripts/drone/steps/lib.star", + "slack_step", ) +load("scripts/drone/vault.star", "pull_secret") -load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token') - -failure_template = 'Build {{build.number}} failed for commit: : {{build.link}}\nBranch: \nAuthor: {{build.author}}' -drone_change_template = '`.drone.yml` and `starlark` files have been changed on the OSS repo, by: {{build.author}}. \nBranch: \nCommit hash: ' +failure_template = "Build {{build.number}} failed for commit: : {{build.link}}\nBranch: \nAuthor: {{build.author}}" +drone_change_template = "`.drone.yml` and `starlark` files have been changed on the OSS repo, by: {{build.author}}. \nBranch: \nCommit hash: " def pipeline( - name, edition, trigger, steps, services=[], platform='linux', depends_on=[], environment=None, volumes=[], - ): - if platform != 'windows': + name, + edition, + trigger, + steps, + services = [], + platform = "linux", + depends_on = [], + environment = None, + volumes = []): + """Generate a Drone Docker pipeline with commonly used values. + + In addition to the parameters provided, it configures: + - the use of an image pull secret + - a retry count for cloning + - a volume 'docker' that can be used to access the Docker socket + + Args: + name: controls the pipeline name. + edition: used to differentiate the pipeline for enterprise builds. + trigger: a Drone trigger for the pipeline. + steps: the Drone steps for the pipeline. + services: auxilliary services used during the pipeline. + Defaults to []. + platform: abstracts platform specific configuration primarily for different Drone behavior on Windows. + Defaults to 'linux'. + depends_on: list of pipelines that must have succeeded before this pipeline can start. + Defaults to []. + environment: environment variables passed through to pipeline steps. + Defaults to None. + volumes: additional volumes available to be mounted by pipeline steps. + Defaults to []. + + Returns: + Drone pipeline + """ + if platform != "windows": platform_conf = { - 'platform': { - 'os': 'linux', - 'arch': 'amd64' - }, + "platform": {"os": "linux", "arch": "amd64"}, # A shared cache is used on the host # To avoid issues with parallel builds, we run this repo on single build agents - 'node': { - 'type': 'no-parallel' - } + "node": {"type": "no-parallel"}, } else: platform_conf = { - 'platform': { - 'os': 'windows', - 'arch': 'amd64', - 'version': '1809', - } + "platform": { + "os": "windows", + "arch": "amd64", + "version": "1809", + }, } pipeline = { - 'kind': 'pipeline', - 'type': 'docker', - 'name': name, - 'trigger': trigger, - 'services': services, - 'steps': steps, - 'clone': { - 'retries': 3, + "kind": "pipeline", + "type": "docker", + "name": name, + "trigger": trigger, + "services": services, + "steps": steps, + "clone": { + "retries": 3, }, - 'volumes': [{ - 'name': 'docker', - 'host': { - 'path': '/var/run/docker.sock', + "volumes": [ + { + "name": "docker", + "host": { + "path": "/var/run/docker.sock", + }, }, - }], - 'depends_on': depends_on, - 'image_pull_secrets': [pull_secret], + ], + "depends_on": depends_on, + "image_pull_secrets": [pull_secret], } if environment: - pipeline.update({ - 'environment': environment, - }) + pipeline.update( + { + "environment": environment, + }, + ) - pipeline['volumes'].extend(volumes) + pipeline["volumes"].extend(volumes) pipeline.update(platform_conf) - if edition in ('enterprise', 'enterprise2'): + if edition in ("enterprise", "enterprise2"): # We have a custom clone step for enterprise - pipeline['clone'] = { - 'disable': True, + pipeline["clone"] = { + "disable": True, } return pipeline -def notify_pipeline(name, slack_channel, trigger, depends_on=[], template=None, secret=None): +def notify_pipeline( + name, + slack_channel, + trigger, + depends_on = [], + template = None, + secret = None): trigger = dict(trigger) return { - 'kind': 'pipeline', - 'type': 'docker', - 'platform': { - 'os': 'linux', - 'arch': 'amd64', + "kind": "pipeline", + "type": "docker", + "platform": { + "os": "linux", + "arch": "amd64", }, - 'name': name, - 'trigger': trigger, - 'steps': [ + "name": name, + "trigger": trigger, + "steps": [ slack_step(slack_channel, template, secret), ], - 'clone': { - 'retries': 3, + "clone": { + "retries": 3, }, - 'depends_on': depends_on, + "depends_on": depends_on, } - +# TODO: this overrides any existing dependencies because we're following the existing logic +# it should append to any existing dependencies +def with_deps(steps, deps = []): + for step in steps: + step["depends_on"] = deps + return steps diff --git a/scripts/drone/vault.star b/scripts/drone/vault.star index b7ebbd54e8b..09a6bae3009 100644 --- a/scripts/drone/vault.star +++ b/scripts/drone/vault.star @@ -1,97 +1,97 @@ -pull_secret = 'dockerconfigjson' -drone_token = 'drone_token' -prerelease_bucket = 'prerelease_bucket' -gcp_upload_artifacts_key = 'gcp_upload_artifacts_key' -azure_sp_app_id = 'azure_sp_app_id' -azure_sp_app_pw = 'azure_sp_app_pw' -azure_tenant = 'azure_tenant' -github_token = 'github_token' +""" +This module returns functions for generating Drone secrets fetched from Vault. +""" +pull_secret = "dockerconfigjson" +drone_token = "drone_token" +prerelease_bucket = "prerelease_bucket" +gcp_upload_artifacts_key = "gcp_upload_artifacts_key" +azure_sp_app_id = "azure_sp_app_id" +azure_sp_app_pw = "azure_sp_app_pw" +azure_tenant = "azure_tenant" def from_secret(secret): - return {'from_secret': secret} - + return {"from_secret": secret} def vault_secret(name, path, key): return { - 'kind': 'secret', - 'name': name, - 'get': { - 'path': path, - 'name': key, + "kind": "secret", + "name": name, + "get": { + "path": path, + "name": key, }, } - def secrets(): return [ - vault_secret(pull_secret, 'secret/data/common/gcr', '.dockerconfigjson'), - vault_secret('github_token', 'infra/data/ci/github/grafanabot', 'pat'), - vault_secret(drone_token, 'infra/data/ci/drone', 'machine-user-token'), - vault_secret(prerelease_bucket, 'infra/data/ci/grafana/prerelease', 'bucket'), + vault_secret(pull_secret, "secret/data/common/gcr", ".dockerconfigjson"), + vault_secret("github_token", "infra/data/ci/github/grafanabot", "pat"), + vault_secret(drone_token, "infra/data/ci/drone", "machine-user-token"), + vault_secret(prerelease_bucket, "infra/data/ci/grafana/prerelease", "bucket"), vault_secret( gcp_upload_artifacts_key, - 'infra/data/ci/grafana/releng/artifacts-uploader-service-account', - 'credentials.json', + "infra/data/ci/grafana/releng/artifacts-uploader-service-account", + "credentials.json", ), vault_secret( azure_sp_app_id, - 'infra/data/ci/datasources/cpp-azure-resourcemanager-credentials', - 'application_id', + "infra/data/ci/datasources/cpp-azure-resourcemanager-credentials", + "application_id", ), vault_secret( azure_sp_app_pw, - 'infra/data/ci/datasources/cpp-azure-resourcemanager-credentials', - 'application_secret', + "infra/data/ci/datasources/cpp-azure-resourcemanager-credentials", + "application_secret", ), vault_secret( azure_tenant, - 'infra/data/ci/datasources/cpp-azure-resourcemanager-credentials', - 'tenant_id', + "infra/data/ci/datasources/cpp-azure-resourcemanager-credentials", + "tenant_id", ), # Package publishing vault_secret( - 'packages_gpg_public_key', - 'infra/data/ci/packages-publish/gpg', - 'public-key-b64', + "packages_gpg_public_key", + "infra/data/ci/packages-publish/gpg", + "public-key-b64", ), vault_secret( - 'packages_gpg_private_key', - 'infra/data/ci/packages-publish/gpg', - 'private-key-b64', + "packages_gpg_private_key", + "infra/data/ci/packages-publish/gpg", + "private-key-b64", ), vault_secret( - 'packages_gpg_passphrase', - 'infra/data/ci/packages-publish/gpg', - 'passphrase', + "packages_gpg_passphrase", + "infra/data/ci/packages-publish/gpg", + "passphrase", ), vault_secret( - 'packages_service_account', - 'infra/data/ci/packages-publish/service-account', - 'credentials.json', + "packages_service_account", + "infra/data/ci/packages-publish/service-account", + "credentials.json", ), vault_secret( - 'packages_access_key_id', - 'infra/data/ci/packages-publish/bucket-credentials', - 'AccessID', + "packages_access_key_id", + "infra/data/ci/packages-publish/bucket-credentials", + "AccessID", ), vault_secret( - 'packages_secret_access_key', - 'infra/data/ci/packages-publish/bucket-credentials', - 'Secret', + "packages_secret_access_key", + "infra/data/ci/packages-publish/bucket-credentials", + "Secret", ), vault_secret( - 'aws_region', - 'secret/data/common/aws-marketplace', - 'aws_region', + "aws_region", + "secret/data/common/aws-marketplace", + "aws_region", ), vault_secret( - 'aws_access_key_id', - 'secret/data/common/aws-marketplace', - 'aws_access_key_id', + "aws_access_key_id", + "secret/data/common/aws-marketplace", + "aws_access_key_id", ), vault_secret( - 'aws_secret_access_key', - 'secret/data/common/aws-marketplace', - 'aws_secret_access_key', + "aws_secret_access_key", + "secret/data/common/aws-marketplace", + "aws_secret_access_key", ), ] diff --git a/scripts/drone/version.star b/scripts/drone/version.star index 80f0c6ac48f..f56a8274000 100644 --- a/scripts/drone/version.star +++ b/scripts/drone/version.star @@ -1,11 +1,20 @@ +""" +This module returns the pipeline used for version branches. +""" + load( - 'scripts/drone/events/release.star', - 'release_pipelines', + "scripts/drone/events/release.star", + "enterprise2_pipelines", + "enterprise_pipelines", + "oss_pipelines", ) -ver_mode = 'release-branch' +ver_mode = "release-branch" +trigger = {"ref": ["refs/heads/v[0-9]*"]} def version_branch_pipelines(): - return release_pipelines(ver_mode=ver_mode, trigger={ - 'ref': ['refs/heads/v[0-9]*'], - }) + return ( + oss_pipelines(ver_mode = ver_mode, trigger = trigger) + + enterprise_pipelines(ver_mode = ver_mode, trigger = trigger) + + enterprise2_pipelines(ver_mode = ver_mode, trigger = trigger) + )