From 20f001649ee99a994c85006e36c630bb8d9833f3 Mon Sep 17 00:00:00 2001 From: Guilherme Caulada Date: Tue, 18 Jul 2023 11:17:11 -0300 Subject: [PATCH] [v9.4.x] CI: Removes enterprise specific pipelines and steps (#71775) [WIP] CI: Removes enterprise specific pipelines and steps (#70815) * Removes enterprise specific pipelines and steps (#123) * Comment out enterprise related pipelines and steps * Suppress unused variable warning * Removes all edition arguments * Remove leftover comments * Remove redundant oss on pipelines and steps names * Remove leftover unused variable * Remove leftovers * Remove pipeline dependencies * Rename pipelines * Fix starlark --------- Co-authored-by: dsotirakis (cherry picked from commit 642a81ba75e79138246797302aba5c35575f030d) * Add editions for static assets (cherry picked from commit b13939b9af8263e01a6b85c16d859d56a0069cf8) Co-authored-by: Dimitris Sotirakis --- .drone.star | 10 - .drone.yml | 2651 +---------------- scripts/drone/events/main.star | 4 +- scripts/drone/events/pr.star | 1 + scripts/drone/events/release.star | 410 +-- scripts/drone/pipelines/aws_marketplace.star | 45 - scripts/drone/pipelines/build.star | 26 +- scripts/drone/pipelines/ci_images.star | 2 - scripts/drone/pipelines/docs.star | 1 - scripts/drone/pipelines/github.star | 43 - .../drone/pipelines/integration_tests.star | 13 +- scripts/drone/pipelines/lint_backend.star | 9 +- scripts/drone/pipelines/lint_frontend.star | 10 +- scripts/drone/pipelines/publish_images.star | 30 +- scripts/drone/pipelines/shellcheck.star | 1 - scripts/drone/pipelines/test_backend.star | 72 +- scripts/drone/pipelines/test_frontend.star | 60 +- .../drone/pipelines/trigger_downstream.star | 1 - scripts/drone/pipelines/verify_drone.star | 1 - scripts/drone/pipelines/verify_starlark.star | 1 - .../drone/pipelines/whats_new_checker.star | 1 - scripts/drone/pipelines/windows.star | 27 +- scripts/drone/rgm.star | 2 - scripts/drone/steps/lib.star | 439 +-- scripts/drone/utils/utils.star | 8 - scripts/drone/vault.star | 15 - scripts/drone/version.star | 6 +- 27 files changed, 362 insertions(+), 3527 deletions(-) diff --git a/.drone.star b/.drone.star index f496a0cb2df..b187ee88abb 100644 --- a/.drone.star +++ b/.drone.star @@ -11,8 +11,6 @@ load("scripts/drone/events/pr.star", "pr_pipelines") load("scripts/drone/events/main.star", "main_pipelines") load( "scripts/drone/events/release.star", - "enterprise2_pipelines", - "enterprise_pipelines", "integration_test_pipelines", "oss_pipelines", "publish_artifacts_pipelines", @@ -33,8 +31,6 @@ load( "publish_ci_build_container_image_pipeline", "publish_ci_windows_test_image_pipeline", ) -load("scripts/drone/pipelines/github.star", "publish_github_pipeline") -load("scripts/drone/pipelines/aws_marketplace.star", "publish_aws_marketplace_pipeline") load( "scripts/drone/pipelines/windows.star", "windows_test_backend", @@ -48,13 +44,7 @@ def main(_ctx): pr_pipelines() + main_pipelines() + oss_pipelines() + - enterprise_pipelines() + - enterprise2_pipelines() + publish_image_pipelines_public() + - publish_github_pipeline("public") + - publish_github_pipeline("security") + - publish_aws_marketplace_pipeline("public") + - publish_artifacts_pipelines("security") + publish_artifacts_pipelines("public") + publish_npm_pipelines() + publish_packages_pipeline() + diff --git a/.drone.yml b/.drone.yml index 1d4dc3c7036..9aae82dc826 100644 --- a/.drone.yml +++ b/.drone.yml @@ -135,6 +135,26 @@ steps: - yarn-install image: grafana/build-container:1.7.4 name: betterer-frontend +- commands: + - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" + | jq .head.repo.fork) + - if [ "$is_fork" != false ]; then return 1; fi + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + ../grafana-enterprise + - cd ../grafana-enterprise + - if git checkout ${DRONE_SOURCE_BRANCH}; then echo "checked out ${DRONE_SOURCE_BRANCH}"; + elif git checkout ${DRONE_TARGET_BRANCH}; then echo "git checkout ${DRONE_TARGET_BRANCH}"; + else git checkout main; fi + - cd ../ + - ln -s src grafana + - cd ./grafana-enterprise + - ./build.sh + environment: + GITHUB_TOKEN: + from_secret: github_token + failure: ignore + image: grafana/build-container:1.7.4 + name: clone-enterprise - commands: - yarn run ci:test-frontend depends_on: @@ -177,6 +197,26 @@ platform: os: linux services: [] steps: +- commands: + - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" + | jq .head.repo.fork) + - if [ "$is_fork" != false ]; then return 1; fi + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + ../grafana-enterprise + - cd ../grafana-enterprise + - if git checkout ${DRONE_SOURCE_BRANCH}; then echo "checked out ${DRONE_SOURCE_BRANCH}"; + elif git checkout ${DRONE_TARGET_BRANCH}; then echo "git checkout ${DRONE_TARGET_BRANCH}"; + else git checkout main; fi + - cd ../ + - ln -s src grafana + - cd ./grafana-enterprise + - ./build.sh + environment: + GITHUB_TOKEN: + from_secret: github_token + failure: ignore + image: grafana/build-container:1.7.4 + name: clone-enterprise - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.17.1 @@ -231,6 +271,26 @@ platform: os: linux services: [] steps: +- commands: + - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" + | jq .head.repo.fork) + - if [ "$is_fork" != false ]; then return 1; fi + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + ../grafana-enterprise + - cd ../grafana-enterprise + - if git checkout ${DRONE_SOURCE_BRANCH}; then echo "checked out ${DRONE_SOURCE_BRANCH}"; + elif git checkout ${DRONE_TARGET_BRANCH}; then echo "git checkout ${DRONE_TARGET_BRANCH}"; + else git checkout main; fi + - cd ../ + - ln -s src grafana + - cd ./grafana-enterprise + - ./build.sh + environment: + GITHUB_TOKEN: + from_secret: github_token + failure: ignore + image: grafana/build-container:1.7.4 + name: clone-enterprise - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.17.1 @@ -326,6 +386,26 @@ steps: CGO_ENABLED: 0 image: golang:1.20.4 name: compile-build-cmd +- commands: + - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" + | jq .head.repo.fork) + - if [ "$is_fork" != false ]; then return 1; fi + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + ../grafana-enterprise + - cd ../grafana-enterprise + - if git checkout ${DRONE_SOURCE_BRANCH}; then echo "checked out ${DRONE_SOURCE_BRANCH}"; + elif git checkout ${DRONE_TARGET_BRANCH}; then echo "git checkout ${DRONE_TARGET_BRANCH}"; + else git checkout main; fi + - cd ../ + - ln -s src grafana + - cd ./grafana-enterprise + - ./build.sh + environment: + GITHUB_TOKEN: + from_secret: github_token + failure: ignore + image: grafana/build-container:1.7.4 + name: clone-enterprise - commands: - make gen-go depends_on: [] @@ -476,7 +556,7 @@ steps: image: grafana/build-container:1.7.4 name: build-backend - commands: - - ./bin/build build-frontend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - ./bin/build build-frontend --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} depends_on: - compile-build-cmd - yarn-install @@ -485,7 +565,7 @@ steps: image: grafana/build-container:1.7.4 name: build-frontend - commands: - - ./bin/build build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - ./bin/build build-frontend-packages --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} depends_on: - compile-build-cmd - yarn-install @@ -750,6 +830,26 @@ services: - name: mysql path: /var/lib/mysql steps: +- commands: + - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" + | jq .head.repo.fork) + - if [ "$is_fork" != false ]; then return 1; fi + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + ../grafana-enterprise + - cd ../grafana-enterprise + - if git checkout ${DRONE_SOURCE_BRANCH}; then echo "checked out ${DRONE_SOURCE_BRANCH}"; + elif git checkout ${DRONE_TARGET_BRANCH}; then echo "git checkout ${DRONE_TARGET_BRANCH}"; + else git checkout main; fi + - cd ../ + - ln -s src grafana + - cd ./grafana-enterprise + - ./build.sh + environment: + GITHUB_TOKEN: + from_secret: github_token + failure: ignore + image: grafana/build-container:1.7.4 + name: clone-enterprise - commands: - mkdir -p bin - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/grabpl @@ -1346,7 +1446,7 @@ steps: image: grafana/build-container:1.7.4 name: build-backend - commands: - - ./bin/build build-frontend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - ./bin/build build-frontend --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} depends_on: - compile-build-cmd - yarn-install @@ -1355,7 +1455,7 @@ steps: image: grafana/build-container:1.7.4 name: build-frontend - commands: - - ./bin/build build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - ./bin/build build-frontend-packages --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} depends_on: - compile-build-cmd - yarn-install @@ -1862,29 +1962,6 @@ steps: -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: windows-init -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - - .\grabpl.exe windows-installer --edition oss --build-id $$env:DRONE_BUILD_NUMBER - - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp $$fname gs://grafana-downloads/oss/main/ - - gsutil cp "$$fname.sha256" gs://grafana-downloads/oss/main/ - depends_on: - - windows-init - environment: - GCP_KEY: - from_secret: gcp_key - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer trigger: branch: main event: @@ -2067,7 +2144,7 @@ environment: image_pull_secrets: - dockerconfigjson kind: pipeline -name: release-oss-build-e2e-publish +name: release-build-e2e-publish node: type: no-parallel platform: @@ -2148,7 +2225,7 @@ steps: image: grafana/build-container:1.7.4 name: build-plugins - commands: - - ./bin/build package --jobs 8 --edition oss --sign ${DRONE_TAG} + - ./bin/build package --jobs 8 --edition oss --sign $${DRONE_TAG} depends_on: - build-plugins - build-backend @@ -2370,7 +2447,7 @@ environment: image_pull_secrets: - dockerconfigjson kind: pipeline -name: release-oss-test-frontend +name: release-test-frontend node: type: no-parallel platform: @@ -2430,7 +2507,7 @@ environment: image_pull_secrets: - dockerconfigjson kind: pipeline -name: release-oss-test-backend +name: release-test-backend node: type: no-parallel platform: @@ -2502,14 +2579,14 @@ volumes: clone: retries: 3 depends_on: -- release-oss-build-e2e-publish -- release-oss-test-frontend +- release-build-e2e-publish +- release-test-frontend environment: EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline -name: release-oss-windows +name: release-windows platform: arch: amd64 os: windows @@ -2564,849 +2641,6 @@ volumes: path: //./pipe/docker_engine/ name: docker --- -clone: - disable: true -depends_on: [] -environment: - EDITION: enterprise -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-enterprise-build-e2e-publish -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.17.1 - name: identify-runner -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout ${DRONE_TAG} - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: clone-enterprise -- commands: - - mv bin/grabpl /tmp/ - - rmdir bin - - mv grafana-enterprise /tmp/ - - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise - ${DRONE_TAG} - - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - - mkdir bin - - mv /tmp/grabpl bin/ - depends_on: - - clone-enterprise - - grabpl - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: init-enterprise -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: - - init-enterprise - environment: - CGO_ENABLED: 0 - image: golang:1.20.4 - name: compile-build-cmd -- commands: - - make gen-go - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: wire-install -- commands: - - yarn install --immutable - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: yarn-install -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: verify-gen-cue -- commands: - - '# It is required that generated jsonnet is committed and in sync with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-jsonnet - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: verify-gen-jsonnet -- commands: - - ./bin/build build-backend --jobs 8 --edition enterprise ${DRONE_TAG} - depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.7.4 - name: build-backend -- commands: - - ./bin/build build-frontend --jobs 8 --edition enterprise ${DRONE_TAG} - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.4 - name: build-frontend -- commands: - - ./bin/build build-frontend-packages --jobs 8 --edition enterprise ${DRONE_TAG} - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.4 - name: build-frontend-packages -- commands: - - ./bin/build build-plugins --jobs 8 --edition enterprise - depends_on: - - compile-build-cmd - - yarn-install - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.4 - name: build-plugins -- commands: - - ./bin/build package --jobs 8 --edition enterprise --sign ${DRONE_TAG} - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - environment: - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.4 - name: package -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.7.4 - name: copy-packages-for-docker -- commands: - - ./bin/build build-docker --edition enterprise --shouldSave - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk:431.0.0 - name: build-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build build-docker --edition enterprise --shouldSave --ubuntu - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk:431.0.0 - name: build-docker-images-ubuntu - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./scripts/grafana-server/start-server - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - detach: true - environment: - ARCH: linux-amd64 - PORT: 3001 - RUNDIR: scripts/grafana-server/tmp-grafana-enterprise - image: grafana/build-container:1.7.4 - name: grafana-server -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite dashboards-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-dashboards-suite -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-smoke-tests-suite -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite panels-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-panels-suite -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite various-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-various-suite -- commands: - - apt-get update - - apt-get install -yq zip - - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ - - gsutil cp e2e/videos.zip gs://$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - export E2E_ARTIFACTS_VIDEO_ZIP=https://storage.googleapis.com/$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - 'echo "E2E Test artifacts uploaded to: $${E2E_ARTIFACTS_VIDEO_ZIP}"' - - 'curl -X POST https://api.github.com/repos/${DRONE_REPO}/statuses/${DRONE_COMMIT_SHA} - -H "Authorization: token $${GITHUB_TOKEN}" -d "{\"state\":\"success\",\"target_url\":\"$${E2E_ARTIFACTS_VIDEO_ZIP}\", - \"description\": \"Click on the details to download e2e recording videos\", \"context\": - \"e2e_artifacts\"}"' - depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - E2E_TEST_ARTIFACTS_BUCKET: releng-pipeline-artifacts-dev - GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY: - from_secret: gcp_upload_artifacts_key - GITHUB_TOKEN: - from_secret: github_token - failure: ignore - image: google/cloud-sdk:431.0.0 - name: e2e-tests-artifacts-upload - when: - status: - - success - - failure -- commands: - - ./bin/build upload-cdn --edition enterprise - depends_on: - - package - environment: - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-cdn-assets -- commands: - - ./bin/build upload-packages --edition enterprise - depends_on: - - package - environment: - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-packages -trigger: - event: - exclude: - - promote - ref: - exclude: - - refs/tags/*-cloud* - include: - - refs/tags/v* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - disable: true -depends_on: [] -environment: - EDITION: enterprise -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-enterprise-test-frontend -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout ${DRONE_TAG} - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: clone-enterprise -- commands: - - mv bin/grabpl /tmp/ - - rmdir bin - - mv grafana-enterprise /tmp/ - - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise - ${DRONE_TAG} - - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - - mkdir bin - - mv /tmp/grabpl bin/ - depends_on: - - clone-enterprise - - grabpl - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: init-enterprise -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.17.1 - name: identify-runner -- commands: - - yarn install --immutable - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: yarn-install -- commands: - - yarn betterer ci - depends_on: - - init-enterprise - - yarn-install - image: grafana/build-container:1.7.4 - name: betterer-frontend -- commands: - - yarn run ci:test-frontend - depends_on: - - init-enterprise - - yarn-install - environment: - TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.4 - name: test-frontend -trigger: - event: - exclude: - - promote - ref: - exclude: - - refs/tags/*-cloud* - include: - - refs/tags/v* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - disable: true -depends_on: [] -environment: - EDITION: enterprise -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-enterprise-test-backend -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout ${DRONE_TAG} - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: clone-enterprise -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - mv bin/grabpl /tmp/ - - rmdir bin - - mv grafana-enterprise /tmp/ - - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise - ${DRONE_TAG} - - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - - mkdir bin - - mv /tmp/grabpl bin/ - depends_on: - - clone-enterprise - - grabpl - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: init-enterprise -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.17.1 - name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: - - init-enterprise - environment: - CGO_ENABLED: 0 - image: golang:1.20.4 - name: compile-build-cmd -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: verify-gen-cue -- commands: - - '# It is required that generated jsonnet is committed and in sync with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-jsonnet - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: verify-gen-jsonnet -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.7.4 - name: wire-install -- commands: - - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... - depends_on: - - wire-install - image: grafana/build-container:1.7.4 - name: test-backend -- commands: - - go test -run Integration -covermode=atomic -timeout=5m ./pkg/... - depends_on: - - wire-install - image: grafana/build-container:1.7.4 - name: test-backend-integration -trigger: - event: - exclude: - - promote - ref: - exclude: - - refs/tags/*-cloud* - include: - - refs/tags/v* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - disable: true -depends_on: -- release-enterprise-build-e2e-publish -- release-enterprise-test-frontend -- release-enterprise-test-backend -environment: - EDITION: enterprise -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-enterprise-windows -platform: - arch: amd64 - os: windows - version: "1809" -services: [] -steps: -- commands: - - echo $env:DRONE_RUNNER_NAME - image: mcr.microsoft.com/windows:1809 - name: identify-runner -- commands: - - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/windows/grabpl.exe - -OutFile grabpl.exe - - git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout ${DRONE_TAG} - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/ci-wix:0.1.1 - name: clone -- commands: - - cp -r grafana-enterprise C:\App\grafana-enterprise - - rm -r -force grafana-enterprise - - cp grabpl.exe C:\App\grabpl.exe - - rm -force grabpl.exe - - C:\App\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\App\grafana-enterprise - ${DRONE_TAG} - - cp C:\App\grabpl.exe grabpl.exe - depends_on: - - clone - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/ci-wix:0.1.1 - name: windows-init -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - - .\grabpl.exe windows-installer --edition enterprise ${DRONE_TAG} - - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp $$fname gs://%PRERELEASE_BUCKET%/artifacts/downloads/${DRONE_TAG}/enterprise/release/ - - gsutil cp "$$fname.sha256" gs://%PRERELEASE_BUCKET%/artifacts/downloads/${DRONE_TAG}/enterprise/release/ - depends_on: - - windows-init - environment: - GCP_KEY: - from_secret: gcp_key - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer -trigger: - event: - exclude: - - promote - ref: - exclude: - - refs/tags/*-cloud* - include: - - refs/tags/v* -type: docker -volumes: -- host: - path: //./pipe/docker_engine/ - name: docker ---- -clone: - disable: true -depends_on: [] -environment: - EDITION: enterprise2 -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-enterprise2-build-e2e-publish -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.17.1 - name: identify-runner -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout ${DRONE_TAG} - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: clone-enterprise -- commands: - - mv bin/grabpl /tmp/ - - rmdir bin - - mv grafana-enterprise /tmp/ - - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise - ${DRONE_TAG} - - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - - mkdir bin - - mv /tmp/grabpl bin/ - depends_on: - - clone-enterprise - - grabpl - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: init-enterprise -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: - - init-enterprise - environment: - CGO_ENABLED: 0 - image: golang:1.20.4 - name: compile-build-cmd -- commands: - - make gen-go - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: wire-install -- commands: - - yarn install --immutable - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: yarn-install -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: verify-gen-cue -- commands: - - ./bin/build build-frontend --jobs 8 --edition enterprise ${DRONE_TAG} - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.4 - name: build-frontend -- commands: - - ./bin/build build-frontend-packages --jobs 8 --edition enterprise ${DRONE_TAG} - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.4 - name: build-frontend-packages -- commands: - - ./bin/build build-plugins --jobs 8 --edition enterprise - depends_on: - - compile-build-cmd - - yarn-install - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.4 - name: build-plugins -- commands: - - ./bin/build build-backend --jobs 8 --edition enterprise2 ${DRONE_TAG} - depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.7.4 - name: build-backend-enterprise2 -- commands: - - ./bin/build package --jobs 8 --edition enterprise2 --sign ${DRONE_TAG} - depends_on: - - build-plugins - - build-backend-enterprise2 - - build-frontend - - build-frontend-packages - environment: - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.4 - name: package-enterprise2 -- commands: - - ./bin/build upload-cdn --edition enterprise2 - depends_on: - - package-enterprise2 - environment: - ENTERPRISE2_CDN_PATH: - from_secret: enterprise2-cdn-path - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-cdn-assets-enterprise2 -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package-enterprise2 - image: grafana/build-container:1.7.4 - name: copy-packages-for-docker -- commands: - - ./bin/build build-docker --edition enterprise2 --shouldSave - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - DOCKER_ENTERPRISE2_REPO: - from_secret: docker_enterprise2_repo - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk:431.0.0 - name: build-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build build-docker --edition enterprise2 --shouldSave --ubuntu - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - DOCKER_ENTERPRISE2_REPO: - from_secret: docker_enterprise2_repo - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk:431.0.0 - name: build-docker-images-ubuntu - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build artifacts docker fetch --edition enterprise2 - depends_on: - - build-docker-images - - build-docker-images-ubuntu - environment: - DOCKER_ENTERPRISE2_REPO: - from_secret: docker_enterprise2_repo - DOCKER_PASSWORD: - from_secret: docker_password - DOCKER_USER: - from_secret: docker_username - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk:431.0.0 - name: fetch-images-enterprise2 - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build artifacts docker publish-enterprise2 --dockerhub-repo $${DOCKER_ENTERPRISE2_REPO} - depends_on: - - fetch-images-enterprise2 - environment: - DOCKER_ENTERPRISE2_REPO: - from_secret: docker_enterprise2_repo - DOCKER_PASSWORD: - from_secret: docker_password - DOCKER_USER: - from_secret: docker_username - GCP_KEY: - from_secret: gcp_key_hg - GITHUB_APP_ID: - from_secret: delivery-bot-app-id - GITHUB_APP_INSTALLATION_ID: - from_secret: delivery-bot-app-installation-id - GITHUB_APP_PRIVATE_KEY: - from_secret: delivery-bot-app-private-key - image: google/cloud-sdk:431.0.0 - name: publish-images-enterprise2 - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build upload-packages --edition enterprise2 - depends_on: - - package-enterprise2 - environment: - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-packages-enterprise2 -trigger: - event: - exclude: - - promote - ref: - exclude: - - refs/tags/*-cloud* - include: - - refs/tags/v* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql - temp: - medium: memory ---- clone: retries: 3 depends_on: [] @@ -3415,7 +2649,7 @@ environment: image_pull_secrets: - dockerconfigjson kind: pipeline -name: publish-docker-oss-public +name: publish-docker-public node: type: no-parallel platform: @@ -3445,8 +2679,6 @@ steps: depends_on: - compile-build-cmd environment: - DOCKER_ENTERPRISE2_REPO: - from_secret: docker_enterprise2_repo DOCKER_PASSWORD: from_secret: docker_password DOCKER_USER: @@ -3454,7 +2686,7 @@ steps: GCP_KEY: from_secret: gcp_key image: google/cloud-sdk:431.0.0 - name: fetch-images-oss + name: fetch-images volumes: - name: docker path: /var/run/docker.sock @@ -3462,7 +2694,7 @@ steps: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana --version-tag ${DRONE_TAG} depends_on: - - fetch-images-oss + - fetch-images environment: DOCKER_PASSWORD: from_secret: docker_password @@ -3485,7 +2717,7 @@ steps: - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana-oss --version-tag ${DRONE_TAG} depends_on: - - fetch-images-oss + - fetch-images environment: DOCKER_PASSWORD: from_secret: docker_password @@ -3519,339 +2751,7 @@ clone: retries: 3 depends_on: [] environment: - EDITION: enterprise -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: publish-docker-enterprise-public -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.17.1 - name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.4 - name: compile-build-cmd -- commands: - - ./bin/build artifacts docker fetch --edition enterprise - depends_on: - - compile-build-cmd - environment: - DOCKER_ENTERPRISE2_REPO: - from_secret: docker_enterprise2_repo - DOCKER_PASSWORD: - from_secret: docker_password - DOCKER_USER: - from_secret: docker_username - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk:431.0.0 - name: fetch-images-enterprise - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana-enterprise - --version-tag ${DRONE_TAG} - depends_on: - - fetch-images-enterprise - environment: - DOCKER_PASSWORD: - from_secret: docker_password - DOCKER_USER: - from_secret: docker_username - GCP_KEY: - from_secret: gcp_key - GITHUB_APP_ID: - from_secret: delivery-bot-app-id - GITHUB_APP_INSTALLATION_ID: - from_secret: delivery-bot-app-installation-id - GITHUB_APP_PRIVATE_KEY: - from_secret: delivery-bot-app-private-key - image: google/cloud-sdk:431.0.0 - name: publish-images-grafana-enterprise - volumes: - - name: docker - path: /var/run/docker.sock -trigger: - event: - - promote - target: - - public -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: enterprise2 -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: publish-github-public -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.4 - name: compile-build-cmd -- commands: - - ./bin/build artifacts docker fetch --edition enterprise2 - depends_on: - - compile-build-cmd - environment: - DOCKER_ENTERPRISE2_REPO: - from_secret: docker_enterprise2_repo - DOCKER_PASSWORD: - from_secret: docker_password - DOCKER_USER: - from_secret: docker_username - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk:431.0.0 - name: fetch-images-enterprise2 - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build publish github --repo $${GH_REGISTRY} --create - depends_on: - - fetch-images-enterprise2 - environment: - GH_REGISTRY: - from_secret: gh_registry - GH_TOKEN: - from_secret: github_token - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-github -trigger: - event: - - promote - target: - - public -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: enterprise2 -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: publish-github-security -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.4 - name: compile-build-cmd -- commands: - - ./bin/build artifacts docker fetch --edition enterprise2 - depends_on: - - compile-build-cmd - environment: - DOCKER_ENTERPRISE2_REPO: - from_secret: docker_enterprise2_repo - DOCKER_PASSWORD: - from_secret: docker_password - DOCKER_USER: - from_secret: docker_username - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk:431.0.0 - name: fetch-images-enterprise2 - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build publish github --repo $${GH_REGISTRY} --create - depends_on: - - fetch-images-enterprise2 - environment: - GH_REGISTRY: - from_secret: gh_registry - GH_TOKEN: - from_secret: github_token - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-github -trigger: - event: - - promote - target: - - security -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: -- publish-docker-enterprise-public -environment: - EDITION: enterprise2 -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: publish-aws-marketplace-public -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.4 - name: compile-build-cmd -- commands: - - ./bin/build publish aws --image grafana/grafana-enterprise --repo grafana-labs/grafanaenterprise - --product 422b46fb-bea6-4f27-8bcc-832117bd627e - depends_on: - - compile-build-cmd - environment: - AWS_ACCESS_KEY_ID: - from_secret: aws_access_key_id - AWS_REGION: - from_secret: aws_region - AWS_SECRET_ACCESS_KEY: - from_secret: aws_secret_access_key - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-aws-marketplace - volumes: - - name: docker - path: /var/run/docker.sock -trigger: - event: - - promote - target: - - public -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: all -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: publish-artifacts-security -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.4 - name: compile-build-cmd -- commands: - - ./bin/build artifacts packages --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET} - depends_on: - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-artifacts -- commands: - - ./bin/build artifacts static-assets --tag ${DRONE_TAG} - depends_on: - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - STATIC_ASSET_EDITIONS: - from_secret: static_asset_editions - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-static-assets -- commands: - - ./bin/build artifacts storybook --tag ${DRONE_TAG} - depends_on: - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-storybook -trigger: - event: - - promote - target: - - security -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: [] -environment: - EDITION: all + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -3882,7 +2782,7 @@ steps: image: grafana/grafana-ci-deploy:1.3.3 name: publish-artifacts - commands: - - ./bin/build artifacts static-assets --tag ${DRONE_TAG} + - ./bin/build artifacts static-assets --tag ${DRONE_TAG} --static-asset-editions=grafana-oss depends_on: - compile-build-cmd environment: @@ -3920,7 +2820,7 @@ clone: retries: 3 depends_on: [] environment: - EDITION: all + EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline @@ -3983,14 +2883,13 @@ clone: retries: 3 depends_on: - publish-artifacts-public -- publish-docker-oss-public -- publish-docker-enterprise-public +- publish-docker-public environment: EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline -name: publish-packages-oss +name: publish-packages node: type: no-parallel platform: @@ -4020,7 +2919,7 @@ steps: from_secret: packages_gpg_private_key gpg_public_key: from_secret: packages_gpg_public_key - package_path: gs://grafana-prerelease/artifacts/downloads/*${DRONE_TAG}/oss/**.deb + package_path: gs://grafana-prerelease/artifacts/downloads/*$${DRONE_TAG}/oss/**.deb secret_access_key: from_secret: packages_secret_access_key service_account_json: @@ -4041,7 +2940,7 @@ steps: from_secret: packages_gpg_private_key gpg_public_key: from_secret: packages_gpg_public_key - package_path: gs://grafana-prerelease/artifacts/downloads/*${DRONE_TAG}/oss/**.rpm + package_path: gs://grafana-prerelease/artifacts/downloads/*$${DRONE_TAG}/oss/**.rpm secret_access_key: from_secret: packages_secret_access_key service_account_json: @@ -4058,7 +2957,7 @@ steps: GRAFANA_COM_API_KEY: from_secret: grafana_api_key image: grafana/grafana-ci-deploy:1.3.3 - name: publish-grafanacom-oss + name: publish-grafanacom trigger: event: - promote @@ -4073,102 +2972,8 @@ volumes: clone: retries: 3 depends_on: -- publish-artifacts-public -- publish-docker-oss-public -- publish-docker-enterprise-public -environment: - EDITION: enterprise -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: publish-packages-enterprise -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.4 - name: compile-build-cmd -- depends_on: - - compile-build-cmd - image: us.gcr.io/kubernetes-dev/package-publish:latest - name: publish-linux-packages-deb - privileged: true - settings: - access_key_id: - from_secret: packages_access_key_id - deb_distribution: auto - gpg_passphrase: - from_secret: packages_gpg_passphrase - gpg_private_key: - from_secret: packages_gpg_private_key - gpg_public_key: - from_secret: packages_gpg_public_key - package_path: gs://grafana-prerelease/artifacts/downloads/*${DRONE_TAG}/enterprise/**.deb - secret_access_key: - from_secret: packages_secret_access_key - service_account_json: - from_secret: packages_service_account - target_bucket: grafana-packages -- depends_on: - - compile-build-cmd - image: us.gcr.io/kubernetes-dev/package-publish:latest - name: publish-linux-packages-rpm - privileged: true - settings: - access_key_id: - from_secret: packages_access_key_id - deb_distribution: auto - gpg_passphrase: - from_secret: packages_gpg_passphrase - gpg_private_key: - from_secret: packages_gpg_private_key - gpg_public_key: - from_secret: packages_gpg_public_key - package_path: gs://grafana-prerelease/artifacts/downloads/*${DRONE_TAG}/enterprise/**.rpm - secret_access_key: - from_secret: packages_secret_access_key - service_account_json: - from_secret: packages_service_account - target_bucket: grafana-packages -- commands: - - ./bin/build publish grafana-com --edition enterprise ${DRONE_TAG} - depends_on: - - publish-linux-packages-deb - - publish-linux-packages-rpm - environment: - GCP_KEY: - from_secret: gcp_key - GRAFANA_COM_API_KEY: - from_secret: grafana_api_key - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-grafanacom-enterprise -trigger: - event: - - promote - target: - - public -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: -- release-oss-build-e2e-publish -- release-enterprise-build-e2e-publish -- release-enterprise2-build-e2e-publish -- release-oss-windows -- release-enterprise-windows +- release-build-e2e-publish +- release-windows image_pull_secrets: - dockerconfigjson kind: pipeline @@ -4218,7 +3023,7 @@ environment: image_pull_secrets: - dockerconfigjson kind: pipeline -name: testing-oss-test-backend-windows +name: testing-test-backend-windows platform: arch: amd64 os: windows @@ -4423,7 +3228,7 @@ environment: image_pull_secrets: - dockerconfigjson kind: pipeline -name: release-branch-oss-build-e2e-publish +name: release-branch-build-e2e-publish node: type: no-parallel platform: @@ -4476,7 +3281,7 @@ steps: image: grafana/build-container:1.7.4 name: build-backend - commands: - - ./bin/build build-frontend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - ./bin/build build-frontend --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} depends_on: - compile-build-cmd - yarn-install @@ -4485,7 +3290,7 @@ steps: image: grafana/build-container:1.7.4 name: build-frontend - commands: - - ./bin/build build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - ./bin/build build-frontend-packages --jobs 8 --edition oss--build-id ${DRONE_BUILD_NUMBER} depends_on: - compile-build-cmd - yarn-install @@ -4702,7 +3507,7 @@ environment: image_pull_secrets: - dockerconfigjson kind: pipeline -name: release-branch-oss-test-frontend +name: release-branch-test-frontend node: type: no-parallel platform: @@ -4756,7 +3561,7 @@ environment: image_pull_secrets: - dockerconfigjson kind: pipeline -name: release-branch-oss-test-backend +name: release-branch-test-backend node: type: no-parallel platform: @@ -4827,7 +3632,7 @@ environment: image_pull_secrets: - dockerconfigjson kind: pipeline -name: release-branch-oss-integration-tests +name: release-branch-integration-tests node: type: no-parallel platform: @@ -4938,14 +3743,14 @@ volumes: clone: retries: 3 depends_on: -- release-branch-oss-build-e2e-publish -- release-branch-oss-test-frontend +- release-branch-build-e2e-publish +- release-branch-test-frontend environment: EDITION: oss image_pull_secrets: - dockerconfigjson kind: pipeline -name: release-branch-oss-windows +name: release-branch-windows platform: arch: amd64 os: windows @@ -4990,997 +3795,6 @@ volumes: path: //./pipe/docker_engine/ name: docker --- -clone: - disable: true -depends_on: [] -environment: - EDITION: enterprise -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-enterprise-build-e2e-publish -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.17.1 - name: identify-runner -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout ${DRONE_BRANCH} - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: clone-enterprise -- commands: - - mv bin/grabpl /tmp/ - - rmdir bin - - mv grafana-enterprise /tmp/ - - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise - - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - - mkdir bin - - mv /tmp/grabpl bin/ - depends_on: - - clone-enterprise - - grabpl - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: init-enterprise -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: - - init-enterprise - environment: - CGO_ENABLED: 0 - image: golang:1.20.4 - name: compile-build-cmd -- commands: - - make gen-go - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: wire-install -- commands: - - yarn install --immutable - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: yarn-install -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: verify-gen-cue -- commands: - - '# It is required that generated jsonnet is committed and in sync with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-jsonnet - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: verify-gen-jsonnet -- commands: - - ./bin/build build-backend --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} - depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.7.4 - name: build-backend -- commands: - - ./bin/build build-frontend --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.4 - name: build-frontend -- commands: - - ./bin/build build-frontend-packages --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.4 - name: build-frontend-packages -- commands: - - ./bin/build build-plugins --jobs 8 --edition enterprise - depends_on: - - compile-build-cmd - - yarn-install - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.4 - name: build-plugins -- commands: - - ./bin/build package --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} - --sign - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - environment: - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.4 - name: package -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.7.4 - name: copy-packages-for-docker -- commands: - - ./bin/build build-docker --edition enterprise --shouldSave - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk:431.0.0 - name: build-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build build-docker --edition enterprise --shouldSave --ubuntu - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk:431.0.0 - name: build-docker-images-ubuntu - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./scripts/grafana-server/start-server - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-frontend-packages - detach: true - environment: - ARCH: linux-amd64 - PORT: 3001 - RUNDIR: scripts/grafana-server/tmp-grafana-enterprise - image: grafana/build-container:1.7.4 - name: grafana-server -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite dashboards-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-dashboards-suite -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-smoke-tests-suite -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite panels-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-panels-suite -- commands: - - apt-get install -y netcat - - ./bin/build e2e-tests --port 3001 --suite various-suite --tries 3 - depends_on: - - grafana-server - environment: - HOST: grafana-server - image: cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 - name: end-to-end-tests-various-suite -- commands: - - apt-get update - - apt-get install -yq zip - - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ - - gsutil cp e2e/videos.zip gs://$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - export E2E_ARTIFACTS_VIDEO_ZIP=https://storage.googleapis.com/$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - 'echo "E2E Test artifacts uploaded to: $${E2E_ARTIFACTS_VIDEO_ZIP}"' - - 'curl -X POST https://api.github.com/repos/${DRONE_REPO}/statuses/${DRONE_COMMIT_SHA} - -H "Authorization: token $${GITHUB_TOKEN}" -d "{\"state\":\"success\",\"target_url\":\"$${E2E_ARTIFACTS_VIDEO_ZIP}\", - \"description\": \"Click on the details to download e2e recording videos\", \"context\": - \"e2e_artifacts\"}"' - depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - E2E_TEST_ARTIFACTS_BUCKET: releng-pipeline-artifacts-dev - GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY: - from_secret: gcp_upload_artifacts_key - GITHUB_TOKEN: - from_secret: github_token - failure: ignore - image: google/cloud-sdk:431.0.0 - name: e2e-tests-artifacts-upload - when: - status: - - success - - failure -- commands: - - ./bin/build upload-cdn --edition enterprise - depends_on: - - package - environment: - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-cdn-assets - when: - repo: - - grafana/grafana -- commands: - - ./bin/build upload-packages --edition enterprise - depends_on: - - package - environment: - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-packages - when: - repo: - - grafana/grafana -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql - temp: - medium: memory ---- -clone: - disable: true -depends_on: [] -environment: - EDITION: enterprise -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-enterprise-test-frontend -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout ${DRONE_BRANCH} - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: clone-enterprise -- commands: - - mv bin/grabpl /tmp/ - - rmdir bin - - mv grafana-enterprise /tmp/ - - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise - - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - - mkdir bin - - mv /tmp/grabpl bin/ - depends_on: - - clone-enterprise - - grabpl - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: init-enterprise -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.17.1 - name: identify-runner -- commands: - - yarn install --immutable - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: yarn-install -- commands: - - yarn betterer ci - depends_on: - - init-enterprise - - yarn-install - image: grafana/build-container:1.7.4 - name: betterer-frontend -- commands: - - yarn run ci:test-frontend - depends_on: - - init-enterprise - - yarn-install - environment: - TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.7.4 - name: test-frontend -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - disable: true -depends_on: [] -environment: - EDITION: enterprise -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-enterprise-test-backend -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout ${DRONE_BRANCH} - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: clone-enterprise -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - mv bin/grabpl /tmp/ - - rmdir bin - - mv grafana-enterprise /tmp/ - - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise - - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - - mkdir bin - - mv /tmp/grabpl bin/ - depends_on: - - clone-enterprise - - grabpl - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: init-enterprise -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.17.1 - name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: - - init-enterprise - environment: - CGO_ENABLED: 0 - image: golang:1.20.4 - name: compile-build-cmd -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: verify-gen-cue -- commands: - - '# It is required that generated jsonnet is committed and in sync with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-jsonnet - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: verify-gen-jsonnet -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.7.4 - name: wire-install -- commands: - - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... - depends_on: - - wire-install - image: grafana/build-container:1.7.4 - name: test-backend -- commands: - - go test -run Integration -covermode=atomic -timeout=5m ./pkg/... - depends_on: - - wire-install - image: grafana/build-container:1.7.4 - name: test-backend-integration -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - disable: true -depends_on: [] -environment: - EDITION: enterprise -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-enterprise-integration-tests -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: -- environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_DB: grafanatest - POSTGRES_PASSWORD: grafanatest - POSTGRES_USER: grafanatest - image: postgres:12.3-alpine - name: postgres - volumes: - - name: postgres - path: /var/lib/postgresql/data/pgdata -- environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:5.7.39 - name: mysql - volumes: - - name: mysql - path: /var/lib/mysql -- environment: {} - image: redis:6.2.1-alpine - name: redis -- environment: {} - image: memcached:1.6.9-alpine - name: memcached -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.17.1 - name: identify-runner -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout ${DRONE_BRANCH} - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: clone-enterprise -- commands: - - mv bin/grabpl /tmp/ - - rmdir bin - - mv grafana-enterprise /tmp/ - - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise - - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - - mkdir bin - - mv /tmp/grabpl bin/ - depends_on: - - clone-enterprise - - grabpl - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: init-enterprise -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: verify-gen-cue -- commands: - - '# It is required that generated jsonnet is committed and in sync with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-jsonnet - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: verify-gen-jsonnet -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.7.4 - name: wire-install -- commands: - - apt-get update - - apt-get install -yq postgresql-client - - dockerize -wait tcp://postgres:5432 -timeout 120s - - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' - depends_on: - - wire-install - environment: - GRAFANA_TEST_DB: postgres - PGPASSWORD: grafanatest - POSTGRES_HOST: postgres - image: grafana/build-container:1.7.4 - name: postgres-integration-tests -- commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root - -prootpass - - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' - depends_on: - - wire-install - environment: - GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql - image: grafana/build-container:1.7.4 - name: mysql-integration-tests -- commands: - - dockerize -wait tcp://redis:6379/0 -timeout 120s - - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' - depends_on: - - wire-install - environment: - REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.7.4 - name: redis-integration-tests -- commands: - - dockerize -wait tcp://memcached:11211 -timeout 120s - - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' - depends_on: - - wire-install - environment: - MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.7.4 - name: memcached-integration-tests -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql - temp: - medium: memory ---- -clone: - disable: true -depends_on: -- release-branch-enterprise-build-e2e-publish -- release-branch-enterprise-test-frontend -- release-branch-enterprise-test-backend -environment: - EDITION: enterprise -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-enterprise-windows -platform: - arch: amd64 - os: windows - version: "1809" -services: [] -steps: -- commands: - - echo $env:DRONE_RUNNER_NAME - image: mcr.microsoft.com/windows:1809 - name: identify-runner -- commands: - - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/windows/grabpl.exe - -OutFile grabpl.exe - - git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout $$env:DRONE_BRANCH - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/ci-wix:0.1.1 - name: clone -- commands: - - cp -r grafana-enterprise C:\App\grafana-enterprise - - rm -r -force grafana-enterprise - - cp grabpl.exe C:\App\grabpl.exe - - rm -force grabpl.exe - - C:\App\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\App\grafana-enterprise - $$env:DRONE_BRANCH - - cp C:\App\grabpl.exe grabpl.exe - depends_on: - - clone - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/ci-wix:0.1.1 - name: windows-init -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - depends_on: - - windows-init - environment: - GCP_KEY: - from_secret: gcp_key - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: //./pipe/docker_engine/ - name: docker ---- -clone: - disable: true -depends_on: [] -environment: - EDITION: enterprise2 -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release-branch-enterprise2-build-e2e-publish -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.17.1 - name: identify-runner -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout ${DRONE_BRANCH} - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: clone-enterprise -- commands: - - mv bin/grabpl /tmp/ - - rmdir bin - - mv grafana-enterprise /tmp/ - - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise - - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - - mkdir bin - - mv /tmp/grabpl bin/ - depends_on: - - clone-enterprise - - grabpl - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: init-enterprise -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: - - init-enterprise - environment: - CGO_ENABLED: 0 - image: golang:1.20.4 - name: compile-build-cmd -- commands: - - make gen-go - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: wire-install -- commands: - - yarn install --immutable - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: yarn-install -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: verify-gen-cue -- commands: - - ./bin/build build-frontend --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.4 - name: build-frontend -- commands: - - ./bin/build build-frontend-packages --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} - depends_on: - - compile-build-cmd - - yarn-install - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.7.4 - name: build-frontend-packages -- commands: - - ./bin/build build-plugins --jobs 8 --edition enterprise - depends_on: - - compile-build-cmd - - yarn-install - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.4 - name: build-plugins -- commands: - - ./bin/build build-backend --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER} - --variants linux-amd64 - depends_on: - - wire-install - - compile-build-cmd - image: grafana/build-container:1.7.4 - name: build-backend-enterprise2 -- commands: - - ./bin/build package --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER} - --sign - depends_on: - - build-plugins - - build-backend-enterprise2 - - build-frontend - - build-frontend-packages - environment: - GPG_KEY_PASSWORD: - from_secret: packages_gpg_passphrase - GPG_PRIV_KEY: - from_secret: packages_gpg_private_key - GPG_PUB_KEY: - from_secret: packages_gpg_public_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.7.4 - name: package-enterprise2 -- commands: - - ./bin/build upload-cdn --edition enterprise2 - depends_on: - - package-enterprise2 - environment: - ENTERPRISE2_CDN_PATH: - from_secret: enterprise2-cdn-path - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-cdn-assets-enterprise2 -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package-enterprise2 - image: grafana/build-container:1.7.4 - name: copy-packages-for-docker -- commands: - - ./bin/build build-docker --edition enterprise2 --shouldSave - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - DOCKER_ENTERPRISE2_REPO: - from_secret: docker_enterprise2_repo - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk:431.0.0 - name: build-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build build-docker --edition enterprise2 --shouldSave --ubuntu - depends_on: - - copy-packages-for-docker - - compile-build-cmd - environment: - DOCKER_ENTERPRISE2_REPO: - from_secret: docker_enterprise2_repo - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk:431.0.0 - name: build-docker-images-ubuntu - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build artifacts docker fetch --edition enterprise2 - depends_on: - - build-docker-images - - build-docker-images-ubuntu - environment: - DOCKER_ENTERPRISE2_REPO: - from_secret: docker_enterprise2_repo - DOCKER_PASSWORD: - from_secret: docker_password - DOCKER_USER: - from_secret: docker_username - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk:431.0.0 - name: fetch-images-enterprise2 - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build artifacts docker publish-enterprise2 --dockerhub-repo $${DOCKER_ENTERPRISE2_REPO} - depends_on: - - fetch-images-enterprise2 - environment: - DOCKER_ENTERPRISE2_REPO: - from_secret: docker_enterprise2_repo - DOCKER_PASSWORD: - from_secret: docker_password - DOCKER_USER: - from_secret: docker_username - GCP_KEY: - from_secret: gcp_key_hg - GITHUB_APP_ID: - from_secret: delivery-bot-app-id - GITHUB_APP_INSTALLATION_ID: - from_secret: delivery-bot-app-installation-id - GITHUB_APP_PRIVATE_KEY: - from_secret: delivery-bot-app-private-key - image: google/cloud-sdk:431.0.0 - name: publish-images-enterprise2 - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/build upload-packages --edition enterprise2 - depends_on: - - package-enterprise2 - environment: - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.3 - name: upload-packages-enterprise2 -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql - temp: - medium: memory ---- clone: retries: 3 depends_on: [] @@ -5989,7 +3803,7 @@ environment: image_pull_secrets: - dockerconfigjson kind: pipeline -name: integration-tests-oss +name: integration-tests node: type: no-parallel platform: @@ -6101,179 +3915,6 @@ volumes: clone: disable: true depends_on: [] -environment: - EDITION: enterprise -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: integration-tests-enterprise -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: -- environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_DB: grafanatest - POSTGRES_PASSWORD: grafanatest - POSTGRES_USER: grafanatest - image: postgres:12.3-alpine - name: postgres - volumes: - - name: postgres - path: /var/lib/postgresql/data/pgdata -- environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:5.7.39 - name: mysql - volumes: - - name: mysql - path: /var/lib/mysql -- environment: {} - image: redis:6.2.1-alpine - name: redis -- environment: {} - image: memcached:1.6.9-alpine - name: memcached -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.17.1 - name: identify-runner -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout ${DRONE_TAG} - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: clone-enterprise -- commands: - - mv bin/grabpl /tmp/ - - rmdir bin - - mv grafana-enterprise /tmp/ - - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise - ${DRONE_TAG} - - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - - mkdir bin - - mv /tmp/grabpl bin/ - depends_on: - - clone-enterprise - - grabpl - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: init-enterprise -- commands: - - '# It is required that code generated from Thema/CUE be committed and in sync - with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-cue - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: verify-gen-cue -- commands: - - '# It is required that generated jsonnet is committed and in sync with its inputs.' - - '# The following command will fail if running code generators produces any diff - in output.' - - CODEGEN_VERIFY=1 make gen-jsonnet - depends_on: - - init-enterprise - image: grafana/build-container:1.7.4 - name: verify-gen-jsonnet -- commands: - - make gen-go - depends_on: - - verify-gen-cue - image: grafana/build-container:1.7.4 - name: wire-install -- commands: - - apt-get update - - apt-get install -yq postgresql-client - - dockerize -wait tcp://postgres:5432 -timeout 120s - - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' - depends_on: - - wire-install - environment: - GRAFANA_TEST_DB: postgres - PGPASSWORD: grafanatest - POSTGRES_HOST: postgres - image: grafana/build-container:1.7.4 - name: postgres-integration-tests -- commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root - -prootpass - - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' - depends_on: - - wire-install - environment: - GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql - image: grafana/build-container:1.7.4 - name: mysql-integration-tests -- commands: - - dockerize -wait tcp://redis:6379/0 -timeout 120s - - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' - depends_on: - - wire-install - environment: - REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.7.4 - name: redis-integration-tests -- commands: - - dockerize -wait tcp://memcached:11211 -timeout 120s - - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' - depends_on: - - wire-install - environment: - MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.7.4 - name: memcached-integration-tests -trigger: - event: - - promote - target: integration-tests -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql - temp: - medium: memory ---- -clone: - disable: true -depends_on: [] image_pull_secrets: - dockerconfigjson kind: pipeline @@ -6681,24 +4322,6 @@ get: kind: secret name: packages_secret_access_key --- -get: - name: aws_region - path: secret/data/common/aws-marketplace -kind: secret -name: aws_region ---- -get: - name: aws_access_key_id - path: secret/data/common/aws-marketplace -kind: secret -name: aws_access_key_id ---- -get: - name: aws_secret_access_key - path: secret/data/common/aws-marketplace -kind: secret -name: aws_secret_access_key ---- get: name: static_asset_editions path: infra/data/ci/grafana-release-eng/artifact-publishing @@ -6778,6 +4401,6 @@ kind: secret name: github_token --- kind: signature -hmac: 1a98cd9e5f6971a50cc1fb18c0b637c50f7e5a6ebda9664b63658e356b21a312 +hmac: f0116e20032a6984e32c3611ea2562a3c30ff9214cebbc7e307927243bcb5616 ... diff --git a/scripts/drone/events/main.star b/scripts/drone/events/main.star index ab003e15abb..7c592c53ae8 100644 --- a/scripts/drone/events/main.star +++ b/scripts/drone/events/main.star @@ -87,8 +87,8 @@ def main_pipelines(): test_backend(trigger, ver_mode), lint_backend_pipeline(trigger, ver_mode), build_e2e(trigger, ver_mode), - integration_tests(trigger, prefix = ver_mode), - windows(trigger, edition = "oss", ver_mode = ver_mode), + integration_tests(trigger, prefix = ver_mode, ver_mode = ver_mode), + windows(trigger, ver_mode = ver_mode), notify_pipeline( name = "notify-drone-changes", slack_channel = "slack-webhooks-test", diff --git a/scripts/drone/events/pr.star b/scripts/drone/events/pr.star index 14605cd7909..476ef660e76 100644 --- a/scripts/drone/events/pr.star +++ b/scripts/drone/events/pr.star @@ -130,6 +130,7 @@ def pr_pipelines(): ], ), prefix = ver_mode, + ver_mode = ver_mode, ), docs_pipelines(ver_mode, trigger_docs_pr()), shellcheck_pipeline(), diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index 34760f2d33d..cc49f3a4c9f 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -10,25 +10,19 @@ load( "build_frontend_step", "build_plugins_step", "build_storybook_step", - "clone_enterprise_step", "compile_build_cmd", "copy_packages_for_docker_step", "download_grabpl_step", "e2e_tests_artifacts", "e2e_tests_step", - "fetch_images_step", "get_windows_steps", "grafana_server_step", "identify_runner_step", - "init_enterprise_step", - "memcached_integration_tests_step", "mysql_integration_tests_step", "package_step", "postgres_integration_tests_step", "publish_grafanacom_step", - "publish_images_step", "publish_linux_packages_step", - "redis_integration_tests_step", "store_storybook_step", "trigger_oss", "upload_cdn_step", @@ -46,17 +40,14 @@ load( load( "scripts/drone/utils/utils.star", "pipeline", - "with_deps", ) load( "scripts/drone/pipelines/test_frontend.star", "test_frontend", - "test_frontend_enterprise", ) load( "scripts/drone/pipelines/test_backend.star", "test_backend", - "test_backend_enterprise", ) load("scripts/drone/vault.star", "from_secret", "prerelease_bucket") load( @@ -158,19 +149,18 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger): ] build_steps = [ - build_backend_step(edition = "oss", ver_mode = ver_mode), - build_frontend_step(edition = "oss", ver_mode = ver_mode), - build_frontend_package_step(edition = "oss", ver_mode = ver_mode), - build_plugins_step(edition = "oss", ver_mode = ver_mode), - package_step(edition = "oss", ver_mode = ver_mode), + build_backend_step(ver_mode = ver_mode), + build_frontend_step(ver_mode = ver_mode), + build_frontend_package_step(ver_mode = ver_mode), + build_plugins_step(ver_mode = ver_mode), + package_step(ver_mode = ver_mode), copy_packages_for_docker_step(), - build_docker_images_step(edition = "oss", publish = True), + build_docker_images_step(publish = True), build_docker_images_step( - edition = "oss", publish = True, ubuntu = True, ), - grafana_server_step(edition = "oss"), + grafana_server_step(), e2e_tests_step("dashboards-suite", tries = 3), e2e_tests_step("smoke-tests-suite", tries = 3), e2e_tests_step("panels-suite", tries = 3), @@ -187,9 +177,8 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger): ): publish_steps.extend( [ - upload_cdn_step(edition = "oss", ver_mode = ver_mode, trigger = trigger_oss), + upload_cdn_step(ver_mode = ver_mode, trigger = trigger_oss), upload_packages_step( - edition = "oss", ver_mode = ver_mode, trigger = trigger_oss, ), @@ -218,23 +207,13 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger): integration_test_steps = [] volumes = [] - windows_pipeline = pipeline( - name = "{}-oss-windows".format(ver_mode), - edition = "oss", - trigger = trigger, - steps = get_windows_steps(edition = "oss", ver_mode = ver_mode), - platform = "windows", - depends_on = [ - "{}-oss-build-e2e-publish".format(ver_mode), - "{}-oss-test-frontend".format(ver_mode), - ], - environment = environment, - ) - + windows_pipeline_dependencies = [ + "{}-build-e2e-publish".format(ver_mode), + "{}-test-frontend".format(ver_mode), + ] pipelines.extend([ pipeline( - name = "{}-oss-build-e2e-publish".format(ver_mode), - edition = "oss", + name = "{}-build-e2e-publish".format(ver_mode), trigger = trigger, services = [], steps = init_steps + build_steps + publish_steps, @@ -247,8 +226,7 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger): if ver_mode not in ("release"): pipelines.append(pipeline( - name = "{}-oss-integration-tests".format(ver_mode), - edition = "oss", + name = "{}-integration-tests".format(ver_mode), trigger = trigger, services = services, steps = [ @@ -263,291 +241,19 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger): volumes = volumes, )) - pipelines.append(windows_pipeline) - - return pipelines - -def enterprise_pipelines(ver_mode = ver_mode, trigger = release_trigger): - """Generates all pipelines used for Grafana Enterprise. - - Args: - ver_mode: controls which steps are included in the pipeline. - Defaults to 'release'. - trigger: controls which events can trigger the pipeline execution. - Defaults to tag events for tags with a 'v' prefix. - - Returns: - List of Drone pipelines. - """ - if ver_mode == "release": - committish = "${DRONE_TAG}" - elif ver_mode == "release-branch": - committish = "${DRONE_BRANCH}" - else: - committish = "${DRONE_COMMIT}" - - environment = {"EDITION": "enterprise"} - - services = integration_test_services(edition = "enterprise") - volumes = integration_test_services_volumes() - - init_steps = [ - download_grabpl_step(), - identify_runner_step(), - clone_enterprise_step(committish = committish), - init_enterprise_step(ver_mode), - compile_build_cmd("enterprise"), - ] + with_deps( - [ - wire_install_step(), - yarn_install_step(), - verify_gen_cue_step(), - verify_gen_jsonnet_step(), - ], - [ - "init-enterprise", - ], - ) - - build_steps = [ - build_backend_step(edition = "enterprise", ver_mode = ver_mode), - build_frontend_step(edition = "enterprise", ver_mode = ver_mode), - build_frontend_package_step(edition = "enterprise", ver_mode = ver_mode), - build_plugins_step(edition = "enterprise", ver_mode = ver_mode), - package_step( - edition = "enterprise", - ver_mode = ver_mode, - ), - copy_packages_for_docker_step(), - build_docker_images_step(edition = "enterprise", publish = True), - build_docker_images_step( - edition = "enterprise", - publish = True, - ubuntu = True, - ), - grafana_server_step(edition = "enterprise"), - e2e_tests_step("dashboards-suite", tries = 3), - e2e_tests_step("smoke-tests-suite", tries = 3), - e2e_tests_step("panels-suite", tries = 3), - e2e_tests_step("various-suite", tries = 3), - e2e_tests_artifacts(), - ] - - publish_steps = [] - - if ver_mode in ( - "release", - "release-branch", - ): - upload_packages_enterprise = upload_packages_step( - edition = "enterprise", - ver_mode = ver_mode, - trigger = trigger_oss, - ) - upload_packages_enterprise["depends_on"] = ["package"] - - publish_steps.extend( - [ - upload_cdn_step( - edition = "enterprise", - ver_mode = ver_mode, - trigger = trigger_oss, - ), - upload_packages_enterprise, - ], - ) - - integration_test_steps = [ - postgres_integration_tests_step(), - mysql_integration_tests_step(), - redis_integration_tests_step(), - memcached_integration_tests_step(), - ] - - # We don't need to run integration tests at release time since they have - # been run multiple times before: - if ver_mode in ("release"): - integration_test_steps = [] - volumes = [] - windows_pipeline = pipeline( - name = "{}-enterprise-windows".format(ver_mode), - edition = "enterprise", + name = "{}-windows".format(ver_mode), trigger = trigger, - steps = get_windows_steps(edition = "enterprise", ver_mode = ver_mode), + steps = get_windows_steps(ver_mode = ver_mode), platform = "windows", - depends_on = [ - "{}-enterprise-build-e2e-publish".format(ver_mode), - "{}-enterprise-test-frontend".format(ver_mode), - "{}-enterprise-test-backend".format(ver_mode), - ], + depends_on = windows_pipeline_dependencies, environment = environment, ) - pipelines = [ - pipeline( - name = "{}-enterprise-build-e2e-publish".format(ver_mode), - edition = "enterprise", - trigger = trigger, - services = [], - steps = init_steps + build_steps + publish_steps, - environment = environment, - volumes = volumes, - ), - test_frontend_enterprise(trigger, ver_mode, committish = committish), - test_backend_enterprise(trigger, ver_mode, committish = committish), - ] - - if ver_mode not in ("release"): - pipelines.append(pipeline( - name = "{}-enterprise-integration-tests".format(ver_mode), - edition = "enterprise", - trigger = trigger, - services = services, - steps = [ - download_grabpl_step(), - identify_runner_step(), - clone_enterprise_step(committish = committish), - init_enterprise_step(ver_mode), - ] + - with_deps( - [ - verify_gen_cue_step(), - verify_gen_jsonnet_step(), - ], - [ - "init-enterprise", - ], - ) + - [ - wire_install_step(), - ] + - integration_test_steps, - environment = environment, - volumes = volumes, - )) - pipelines.append(windows_pipeline) return pipelines -def enterprise2_pipelines(prefix = "", ver_mode = ver_mode, trigger = release_trigger): - """Generate the next generation of pipelines for Grafana Enterprise. - - Args: - prefix: a prefix for the pipeline name used to differentiate multiple instances of - the same pipeline. - Defaults to ''. - ver_mode: controls which steps are included in the pipeline. - Defaults to 'release'. - trigger: controls which events can trigger the pipeline execution. - Defaults to tag events for tags with a 'v' prefix. - - Returns: - List of Drone pipelines. - """ - if ver_mode == "release": - committish = "${DRONE_TAG}" - elif ver_mode == "release-branch": - committish = "${DRONE_BRANCH}" - else: - committish = "${DRONE_COMMIT}" - - environment = { - "EDITION": "enterprise2", - } - - volumes = integration_test_services_volumes() - - init_steps = [ - download_grabpl_step(), - identify_runner_step(), - clone_enterprise_step(committish = committish), - init_enterprise_step(ver_mode), - compile_build_cmd("enterprise"), - ] + with_deps( - [ - wire_install_step(), - yarn_install_step(), - verify_gen_cue_step(), - ], - [ - "init-enterprise", - ], - ) - - build_steps = [ - build_frontend_step(edition = "enterprise", ver_mode = ver_mode), - build_frontend_package_step(edition = "enterprise", ver_mode = ver_mode), - build_plugins_step(edition = "enterprise", ver_mode = ver_mode), - build_backend_step( - edition = "enterprise2", - ver_mode = ver_mode, - variants = ["linux-amd64"], - ), - ] - - fetch_images = fetch_images_step("enterprise2") - fetch_images.update( - {"depends_on": ["build-docker-images", "build-docker-images-ubuntu"]}, - ) - - upload_cdn = upload_cdn_step(edition = "enterprise2", ver_mode = ver_mode) - upload_cdn["environment"].update( - {"ENTERPRISE2_CDN_PATH": from_secret("enterprise2-cdn-path")}, - ) - - build_steps.extend( - [ - package_step( - edition = "enterprise2", - ver_mode = ver_mode, - ), - upload_cdn, - copy_packages_for_docker_step(edition = "enterprise2"), - build_docker_images_step( - edition = "enterprise2", - publish = True, - ), - build_docker_images_step( - edition = "enterprise2", - publish = True, - ubuntu = True, - ), - fetch_images, - publish_images_step( - "enterprise2", - "release", - docker_repo = "${{DOCKER_ENTERPRISE2_REPO}}", - ), - ], - ) - - publish_steps = [] - - if ver_mode in ( - "release", - "release-branch", - ): - step = upload_packages_step(edition = "enterprise2", ver_mode = ver_mode) - step["depends_on"] = ["package-enterprise2"] - - publish_steps.append(step) - - pipelines = [ - pipeline( - name = "{}{}-enterprise2-build-e2e-publish".format(prefix, ver_mode), - edition = "enterprise", - trigger = trigger, - services = [], - steps = init_steps + build_steps + publish_steps, - volumes = volumes, - environment = environment, - ), - ] - - return pipelines - def publish_artifacts_step(): return { "name": "publish-artifacts", @@ -572,7 +278,7 @@ def publish_static_assets_step(): "STATIC_ASSET_EDITIONS": from_secret("static_asset_editions"), }, "commands": [ - "./bin/build artifacts static-assets --tag ${DRONE_TAG}", + "./bin/build artifacts static-assets --tag ${DRONE_TAG} --static-asset-editions=grafana-oss", ], "depends_on": ["compile-build-cmd"], } @@ -617,13 +323,12 @@ def publish_artifacts_pipelines(mode): name = "publish-artifacts-{}".format(mode), trigger = trigger, steps = steps, - edition = "all", - environment = {"EDITION": "all"}, + environment = {"EDITION": "oss"}, ), ] def publish_packages_pipeline(): - """Generates pipelines used for publishing packages for both OSS and enterprise. + """Generates pipelines used for publishing packages for OSS. Returns: List of Drone pipelines. One for each of OSS and enterprise packages. @@ -635,40 +340,24 @@ def publish_packages_pipeline(): } oss_steps = [ compile_build_cmd(), - publish_linux_packages_step(edition = "oss", package_manager = "deb"), - publish_linux_packages_step(edition = "oss", package_manager = "rpm"), - publish_grafanacom_step(edition = "oss", ver_mode = "release"), + publish_linux_packages_step(package_manager = "deb"), + publish_linux_packages_step(package_manager = "rpm"), + publish_grafanacom_step(ver_mode = "release"), ] - enterprise_steps = [ - compile_build_cmd(), - publish_linux_packages_step(edition = "enterprise", package_manager = "deb"), - publish_linux_packages_step(edition = "enterprise", package_manager = "rpm"), - publish_grafanacom_step(edition = "enterprise", ver_mode = "release"), - ] deps = [ "publish-artifacts-public", - "publish-docker-oss-public", - "publish-docker-enterprise-public", + "publish-docker-public", ] return [ pipeline( - name = "publish-packages-oss", + name = "publish-packages", trigger = trigger, steps = oss_steps, - edition = "all", depends_on = deps, environment = {"EDITION": "oss"}, ), - pipeline( - name = "publish-packages-enterprise", - trigger = trigger, - steps = enterprise_steps, - edition = "all", - depends_on = deps, - environment = {"EDITION": "enterprise"}, - ), ] def publish_npm_pipelines(): @@ -688,8 +377,7 @@ def publish_npm_pipelines(): name = "publish-npm-packages-public", trigger = trigger, steps = steps, - edition = "all", - environment = {"EDITION": "all"}, + environment = {"EDITION": "oss"}, ), ] @@ -702,7 +390,7 @@ def integration_test_pipelines(): particular build. Returns: - List of Drone pipelines (one for enterprise and one for oss integration tests) + List of Drone pipelines """ trigger = { "event": ["promote"], @@ -714,15 +402,9 @@ def integration_test_pipelines(): postgres_integration_tests_step(), mysql_integration_tests_step(), ] - enterprise_integration_test_steps = oss_integration_test_steps + [ - redis_integration_tests_step(), - memcached_integration_tests_step(), - ] - source = "${DRONE_TAG}" pipelines.append(pipeline( - name = "integration-tests-oss", - edition = "oss", + name = "integration-tests", trigger = trigger, services = integration_test_services(edition = "oss"), steps = [ @@ -737,34 +419,6 @@ def integration_test_pipelines(): volumes = volumes, )) - pipelines.append(pipeline( - name = "integration-tests-enterprise", - edition = "enterprise", - trigger = trigger, - services = integration_test_services(edition = "enterprise"), - steps = [ - download_grabpl_step(), - identify_runner_step(), - clone_enterprise_step(committish = source), - init_enterprise_step(ver_mode), - ] + - with_deps( - [ - verify_gen_cue_step(), - verify_gen_jsonnet_step(), - ], - [ - "init-enterprise", - ], - ) + - [ - wire_install_step(), - ] + - enterprise_integration_test_steps, - environment = {"EDITION": "enterprise"}, - volumes = volumes, - )) - return pipelines def verify_release_pipeline( @@ -774,11 +428,8 @@ def verify_release_pipeline( version = "${DRONE_TAG}", trigger = release_trigger, depends_on = [ - "release-oss-build-e2e-publish", - "release-enterprise-build-e2e-publish", - "release-enterprise2-build-e2e-publish", - "release-oss-windows", - "release-enterprise-windows", + "release-build-e2e-publish", + "release-windows", ]): """ Runs a script that 'gsutil stat's every artifact that should have been produced by the pre-release process. @@ -804,7 +455,6 @@ def verify_release_pipeline( return pipeline( depends_on = depends_on, name = name, - edition = "all", trigger = trigger, steps = [step], ) diff --git a/scripts/drone/pipelines/aws_marketplace.star b/scripts/drone/pipelines/aws_marketplace.star index 82d40a4fa57..e69de29bb2d 100644 --- a/scripts/drone/pipelines/aws_marketplace.star +++ b/scripts/drone/pipelines/aws_marketplace.star @@ -1,45 +0,0 @@ -""" -This module contains steps and pipelines publishing to AWS Marketplace. -""" - -load( - "scripts/drone/steps/lib.star", - "compile_build_cmd", -) -load("scripts/drone/vault.star", "from_secret") -load( - "scripts/drone/utils/utils.star", - "pipeline", -) -load( - "scripts/drone/utils/images.star", - "images", -) - -def publish_aws_marketplace_step(): - return { - "name": "publish-aws-marketplace", - "image": images["publish_image"], - "commands": ["./bin/build publish aws --image grafana/grafana-enterprise --repo grafana-labs/grafanaenterprise --product 422b46fb-bea6-4f27-8bcc-832117bd627e"], - "depends_on": ["compile-build-cmd"], - "environment": { - "AWS_REGION": from_secret("aws_region"), - "AWS_ACCESS_KEY_ID": from_secret("aws_access_key_id"), - "AWS_SECRET_ACCESS_KEY": from_secret("aws_secret_access_key"), - }, - "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], - } - -def publish_aws_marketplace_pipeline(mode): - trigger = { - "event": ["promote"], - "target": [mode], - } - return [pipeline( - name = "publish-aws-marketplace-{}".format(mode), - trigger = trigger, - steps = [compile_build_cmd(), publish_aws_marketplace_step()], - edition = "", - depends_on = ["publish-docker-enterprise-public"], - environment = {"EDITION": "enterprise2"}, - )] diff --git a/scripts/drone/pipelines/build.star b/scripts/drone/pipelines/build.star index 66e38273374..9013c697ce5 100644 --- a/scripts/drone/pipelines/build.star +++ b/scripts/drone/pipelines/build.star @@ -48,8 +48,8 @@ def build_e2e(trigger, ver_mode): Returns: Drone pipeline. """ - edition = "oss" - environment = {"EDITION": edition} + + environment = {"EDITION": "oss"} init_steps = [ identify_runner_step(), download_grabpl_step(), @@ -72,12 +72,12 @@ def build_e2e(trigger, ver_mode): build_steps.extend( [ - build_backend_step(edition = edition, ver_mode = ver_mode), - build_frontend_step(edition = edition, ver_mode = ver_mode), - build_frontend_package_step(edition = edition, ver_mode = ver_mode), - build_plugins_step(edition = edition, ver_mode = ver_mode), - package_step(edition = edition, ver_mode = ver_mode), - grafana_server_step(edition = edition), + build_backend_step(ver_mode = ver_mode), + build_frontend_step(ver_mode = ver_mode), + build_frontend_package_step(ver_mode = ver_mode), + build_plugins_step(ver_mode = ver_mode), + package_step(ver_mode = ver_mode), + grafana_server_step(), e2e_tests_step("dashboards-suite"), e2e_tests_step("smoke-tests-suite"), e2e_tests_step("panels-suite"), @@ -100,34 +100,28 @@ def build_e2e(trigger, ver_mode): store_storybook_step(trigger = trigger_oss, ver_mode = ver_mode), frontend_metrics_step(trigger = trigger_oss), build_docker_images_step( - edition = edition, publish = False, ), build_docker_images_step( - edition = edition, publish = False, ubuntu = True, ), publish_images_step( docker_repo = "grafana", - edition = edition, trigger = trigger_oss, ver_mode = ver_mode, ), publish_images_step( docker_repo = "grafana-oss", - edition = edition, trigger = trigger_oss, ver_mode = ver_mode, ), release_canary_npm_packages_step(trigger = trigger_oss), upload_packages_step( - edition = edition, trigger = trigger_oss, ver_mode = ver_mode, ), upload_cdn_step( - edition = edition, trigger = trigger_oss, ver_mode = ver_mode, ), @@ -140,18 +134,15 @@ def build_e2e(trigger, ver_mode): archs = [ "amd64", ], - edition = edition, ), build_docker_images_step( archs = [ "amd64", ], - edition = edition, ubuntu = True, ), publish_images_step( docker_repo = "grafana", - edition = edition, trigger = trigger_oss, ver_mode = ver_mode, ), @@ -164,7 +155,6 @@ def build_e2e(trigger, ver_mode): return pipeline( name = "{}-build-e2e{}".format(ver_mode, publish_suffix), - edition = "oss", environment = environment, services = [], steps = init_steps + build_steps, diff --git a/scripts/drone/pipelines/ci_images.star b/scripts/drone/pipelines/ci_images.star index efda2339e35..65df476754b 100644 --- a/scripts/drone/pipelines/ci_images.star +++ b/scripts/drone/pipelines/ci_images.star @@ -28,7 +28,6 @@ def publish_ci_windows_test_image_pipeline(): pl = pipeline( name = "publish-ci-windows-test-image", trigger = trigger, - edition = "", platform = "windows", steps = [ { @@ -79,7 +78,6 @@ def publish_ci_build_container_image_pipeline(): pl = pipeline( name = "publish-ci-build-container-image", trigger = trigger, - edition = "", steps = [ { "name": "validate-version", diff --git a/scripts/drone/pipelines/docs.star b/scripts/drone/pipelines/docs.star index d60fcdb6cad..5f2cbdb8ef3 100644 --- a/scripts/drone/pipelines/docs.star +++ b/scripts/drone/pipelines/docs.star @@ -39,7 +39,6 @@ def docs_pipelines(ver_mode, trigger): return pipeline( name = "{}-docs".format(ver_mode), - edition = "oss", trigger = trigger, services = [], steps = steps, diff --git a/scripts/drone/pipelines/github.star b/scripts/drone/pipelines/github.star index fc43638c504..e69de29bb2d 100644 --- a/scripts/drone/pipelines/github.star +++ b/scripts/drone/pipelines/github.star @@ -1,43 +0,0 @@ -""" -This module contains steps and pipelines relating to GitHub. -""" - -load( - "scripts/drone/steps/lib.star", - "compile_build_cmd", - "fetch_images_step", -) -load("scripts/drone/vault.star", "from_secret") -load( - "scripts/drone/utils/utils.star", - "pipeline", -) -load( - "scripts/drone/utils/images.star", - "images", -) - -def publish_github_step(): - return { - "name": "publish-github", - "image": images["publish_image"], - "commands": ["./bin/build publish github --repo $${GH_REGISTRY} --create"], - "depends_on": ["fetch-images-enterprise2"], - "environment": { - "GH_TOKEN": from_secret("github_token"), - "GH_REGISTRY": from_secret("gh_registry"), - }, - } - -def publish_github_pipeline(mode): - trigger = { - "event": ["promote"], - "target": [mode], - } - return [pipeline( - name = "publish-github-{}".format(mode), - trigger = trigger, - steps = [compile_build_cmd(), fetch_images_step("enterprise2"), publish_github_step()], - edition = "", - environment = {"EDITION": "enterprise2"}, - )] diff --git a/scripts/drone/pipelines/integration_tests.star b/scripts/drone/pipelines/integration_tests.star index a5ceb011d78..b63b634755b 100644 --- a/scripts/drone/pipelines/integration_tests.star +++ b/scripts/drone/pipelines/integration_tests.star @@ -6,6 +6,7 @@ load( "scripts/drone/steps/lib.star", "compile_build_cmd", "download_grabpl_step", + "enterprise_setup_step", "identify_runner_step", "mysql_integration_tests_step", "postgres_integration_tests_step", @@ -23,12 +24,13 @@ load( "pipeline", ) -def integration_tests(trigger, prefix): +def integration_tests(trigger, prefix, ver_mode): """Generate a pipeline for integration tests. Args: trigger: controls which events can trigger the pipeline execution. prefix: used in the naming of the pipeline. + ver_mode: defines the event / origin of this build. In this function, if it is set to pr, then it will attempt to clone grafana-enterprise. Otherwise it has no effect. Returns: Drone pipeline. @@ -38,7 +40,13 @@ def integration_tests(trigger, prefix): services = integration_test_services(edition = "oss") volumes = integration_test_services_volumes() - init_steps = [ + init_steps = [] + + if ver_mode == "pr": + # In pull requests, attempt to clone grafana enterprise. + init_steps.append(enterprise_setup_step()) + + init_steps += [ download_grabpl_step(), compile_build_cmd(), identify_runner_step(), @@ -54,7 +62,6 @@ def integration_tests(trigger, prefix): return pipeline( name = "{}-integration-tests".format(prefix), - edition = "oss", trigger = trigger, environment = environment, services = services, diff --git a/scripts/drone/pipelines/lint_backend.star b/scripts/drone/pipelines/lint_backend.star index 73788b1e800..f8253d0c544 100644 --- a/scripts/drone/pipelines/lint_backend.star +++ b/scripts/drone/pipelines/lint_backend.star @@ -5,6 +5,7 @@ This module returns the pipeline used for linting backend code. load( "scripts/drone/steps/lib.star", "compile_build_cmd", + "enterprise_setup_step", "identify_runner_step", "lint_backend_step", "lint_drone_step", @@ -33,9 +34,14 @@ def lint_backend_pipeline(trigger, ver_mode): init_steps = [ identify_runner_step(), compile_build_cmd(), - wire_step, ] + if ver_mode == "pr": + # In pull requests, attempt to clone grafana enterprise. + init_steps.append(enterprise_setup_step()) + + init_steps.append(wire_step) + test_steps = [ lint_backend_step(), ] @@ -45,7 +51,6 @@ def lint_backend_pipeline(trigger, ver_mode): return pipeline( name = "{}-lint-backend".format(ver_mode), - edition = "oss", trigger = trigger, services = [], steps = init_steps + test_steps, diff --git a/scripts/drone/pipelines/lint_frontend.star b/scripts/drone/pipelines/lint_frontend.star index 7bdb542982f..8093fa4d27c 100644 --- a/scripts/drone/pipelines/lint_frontend.star +++ b/scripts/drone/pipelines/lint_frontend.star @@ -4,6 +4,7 @@ This module returns the pipeline used for linting frontend code. load( "scripts/drone/steps/lib.star", + "enterprise_setup_step", "identify_runner_step", "lint_frontend_step", "yarn_install_step", @@ -25,7 +26,13 @@ def lint_frontend_pipeline(trigger, ver_mode): """ environment = {"EDITION": "oss"} - init_steps = [ + init_steps = [] + + if ver_mode == "pr": + # In pull requests, attempt to clone grafana enterprise. + init_steps = [enterprise_setup_step()] + + init_steps += [ identify_runner_step(), yarn_install_step(), ] @@ -36,7 +43,6 @@ def lint_frontend_pipeline(trigger, ver_mode): return pipeline( name = "{}-lint-frontend".format(ver_mode), - edition = "oss", trigger = trigger, services = [], steps = init_steps + test_steps, diff --git a/scripts/drone/pipelines/publish_images.star b/scripts/drone/pipelines/publish_images.star index a644e13dc02..4f099eba389 100644 --- a/scripts/drone/pipelines/publish_images.star +++ b/scripts/drone/pipelines/publish_images.star @@ -15,13 +15,10 @@ load( "pipeline", ) -def publish_image_steps(edition, docker_repo): +def publish_image_steps(docker_repo): """Generates the steps used for publising Docker images using grabpl. Args: - edition: controls which version of an image is fetched in the case of a release. - It also controls which publishing implementation is used. - If edition == 'oss', it additionally publishes the grafana/grafana-oss repository. docker_repo: the Docker image name. It is combined with the 'grafana/' library prefix. @@ -32,15 +29,11 @@ def publish_image_steps(edition, docker_repo): identify_runner_step(), download_grabpl_step(), compile_build_cmd(), - fetch_images_step(edition), - publish_images_step(edition, "release", docker_repo), + fetch_images_step(), + publish_images_step("release", docker_repo), + publish_images_step("release", "grafana-oss"), ] - if edition == "oss": - steps.append( - publish_images_step(edition, "release", "grafana-oss"), - ) - return steps def publish_image_pipelines_public(): @@ -56,20 +49,9 @@ def publish_image_pipelines_public(): } return [ pipeline( - name = "publish-docker-oss-{}".format(mode), + name = "publish-docker-{}".format(mode), trigger = trigger, - steps = publish_image_steps(edition = "oss", docker_repo = "grafana"), - edition = "", + steps = publish_image_steps(docker_repo = "grafana"), environment = {"EDITION": "oss"}, ), - pipeline( - name = "publish-docker-enterprise-{}".format(mode), - trigger = trigger, - steps = publish_image_steps( - edition = "enterprise", - docker_repo = "grafana-enterprise", - ), - edition = "", - environment = {"EDITION": "enterprise"}, - ), ] diff --git a/scripts/drone/pipelines/shellcheck.star b/scripts/drone/pipelines/shellcheck.star index b8ef57265d7..5b8d481094c 100644 --- a/scripts/drone/pipelines/shellcheck.star +++ b/scripts/drone/pipelines/shellcheck.star @@ -46,7 +46,6 @@ def shellcheck_pipeline(): ] return pipeline( name = "pr-shellcheck", - edition = "oss", trigger = trigger, services = [], steps = steps, diff --git a/scripts/drone/pipelines/test_backend.star b/scripts/drone/pipelines/test_backend.star index 4b9dfb119a1..627b398a3bf 100644 --- a/scripts/drone/pipelines/test_backend.star +++ b/scripts/drone/pipelines/test_backend.star @@ -5,15 +5,12 @@ This module returns the pipeline used for testing backend code. load( "scripts/drone/utils/utils.star", "pipeline", - "with_deps", ) load( "scripts/drone/steps/lib.star", - "clone_enterprise_step", "compile_build_cmd", - "download_grabpl_step", + "enterprise_setup_step", "identify_runner_step", - "init_enterprise_step", "test_backend_integration_step", "test_backend_step", "verify_gen_cue_step", @@ -33,9 +30,15 @@ def test_backend(trigger, ver_mode): """ environment = {"EDITION": "oss"} - steps = [ + steps = [] + + if ver_mode == "pr": + # In pull requests, attempt to clone grafana enterprise. + steps.append(enterprise_setup_step()) + + steps += [ identify_runner_step(), - compile_build_cmd(edition = "oss"), + compile_build_cmd(), verify_gen_cue_step(), verify_gen_jsonnet_step(), wire_install_step(), @@ -43,63 +46,8 @@ def test_backend(trigger, ver_mode): test_backend_integration_step(), ] - pipeline_name = "{}-test-backend".format(ver_mode) - if ver_mode in ("release-branch", "release"): - pipeline_name = "{}-{}-test-backend".format(ver_mode, "oss") - return pipeline( - name = pipeline_name, - edition = "oss", - trigger = trigger, - steps = steps, - environment = environment, - ) - -def test_backend_enterprise(trigger, ver_mode, committish, edition = "enterprise"): - """Generates the pipeline used for testing backend enterprise code. - - Args: - trigger: a Drone trigger for the pipeline. - ver_mode: affects the pipeline name. - committish: controls what revision of enterprise code to test with. - edition: affects the clone step in the pipeline and also affects the pipeline name. - - Returns: - Drone pipeline. - """ - environment = {"EDITION": edition} - - steps = ( - [ - clone_enterprise_step(committish), - download_grabpl_step(), - init_enterprise_step(ver_mode), - identify_runner_step(), - compile_build_cmd(edition), - ] + - with_deps( - [ - verify_gen_cue_step(), - verify_gen_jsonnet_step(), - ], - [ - "init-enterprise", - ], - ) + - [ - wire_install_step(), - test_backend_step(), - test_backend_integration_step(), - ] - ) - - pipeline_name = "{}-test-backend".format(ver_mode) - if ver_mode in ("release-branch", "release"): - pipeline_name = "{}-{}-test-backend".format(ver_mode, edition) - - return pipeline( - name = pipeline_name, - edition = edition, + name = "{}-test-backend".format(ver_mode), trigger = trigger, steps = steps, environment = environment, diff --git a/scripts/drone/pipelines/test_frontend.star b/scripts/drone/pipelines/test_frontend.star index a5bbacedabe..17e0b708fae 100644 --- a/scripts/drone/pipelines/test_frontend.star +++ b/scripts/drone/pipelines/test_frontend.star @@ -5,15 +5,13 @@ This module returns the pipeline used for testing backend code. load( "scripts/drone/utils/utils.star", "pipeline", - "with_deps", ) load( "scripts/drone/steps/lib.star", "betterer_frontend_step", - "clone_enterprise_step", "download_grabpl_step", + "enterprise_setup_step", "identify_runner_step", - "init_enterprise_step", "test_frontend_step", "yarn_install_step", ) @@ -34,57 +32,19 @@ def test_frontend(trigger, ver_mode): identify_runner_step(), download_grabpl_step(), yarn_install_step(), - betterer_frontend_step(edition = "oss"), - test_frontend_step(edition = "oss"), + betterer_frontend_step(), ] - pipeline_name = "{}-test-frontend".format(ver_mode) - if ver_mode in ("release-branch", "release"): - pipeline_name = "{}-{}-test-frontend".format(ver_mode, "oss") + test_step = test_frontend_step() + + if ver_mode == "pr": + # In pull requests, attempt to clone grafana enterprise. + steps.append(enterprise_setup_step()) + + steps.append(test_step) return pipeline( - name = pipeline_name, - edition = "oss", - trigger = trigger, - steps = steps, - environment = environment, - ) - -def test_frontend_enterprise(trigger, ver_mode, committish, edition = "enterprise"): - """Generates the pipeline used for testing frontend enterprise code. - - Args: - trigger: a Drone trigger for the pipeline. - ver_mode: affects the pipeline name. - committish: controls what revision of enterprise code to test with. - edition: affects the clone step in the pipeline and also affects the pipeline name. - - Returns: - Drone pipeline. - """ - environment = {"EDITION": edition} - - steps = ( - [ - download_grabpl_step(), - clone_enterprise_step(committish), - init_enterprise_step(ver_mode), - identify_runner_step(), - ] + - with_deps([yarn_install_step()], ["init-enterprise"]) + - [ - betterer_frontend_step(edition), - test_frontend_step(edition), - ] - ) - - pipeline_name = "{}-test-frontend".format(ver_mode) - if ver_mode in ("release-branch", "release"): - pipeline_name = "{}-{}-test-frontend".format(ver_mode, edition) - - return pipeline( - name = pipeline_name, - edition = edition, + name = "{}-test-frontend".format(ver_mode), trigger = trigger, steps = steps, environment = environment, diff --git a/scripts/drone/pipelines/trigger_downstream.star b/scripts/drone/pipelines/trigger_downstream.star index d3403de50f1..d35e2ce090a 100644 --- a/scripts/drone/pipelines/trigger_downstream.star +++ b/scripts/drone/pipelines/trigger_downstream.star @@ -36,7 +36,6 @@ def enterprise_downstream_pipeline(): ] return pipeline( name = "main-trigger-downstream", - edition = "oss", trigger = trigger, services = [], steps = steps, diff --git a/scripts/drone/pipelines/verify_drone.star b/scripts/drone/pipelines/verify_drone.star index 8975abec2c8..366b702bd08 100644 --- a/scripts/drone/pipelines/verify_drone.star +++ b/scripts/drone/pipelines/verify_drone.star @@ -22,7 +22,6 @@ def verify_drone(trigger, ver_mode): ] return pipeline( name = "{}-verify-drone".format(ver_mode), - edition = "oss", trigger = trigger, services = [], steps = steps, diff --git a/scripts/drone/pipelines/verify_starlark.star b/scripts/drone/pipelines/verify_starlark.star index b95962b2eb5..9e058893d7b 100644 --- a/scripts/drone/pipelines/verify_starlark.star +++ b/scripts/drone/pipelines/verify_starlark.star @@ -22,7 +22,6 @@ def verify_starlark(trigger, ver_mode): ] return pipeline( name = "{}-verify-starlark".format(ver_mode), - edition = "oss", trigger = trigger, services = [], steps = steps, diff --git a/scripts/drone/pipelines/whats_new_checker.star b/scripts/drone/pipelines/whats_new_checker.star index b73acb2be0b..e15a8d7a291 100644 --- a/scripts/drone/pipelines/whats_new_checker.star +++ b/scripts/drone/pipelines/whats_new_checker.star @@ -35,7 +35,6 @@ def whats_new_checker_pipeline(trigger): ] return pipeline( name = "release-whatsnew-checker", - edition = "oss", trigger = trigger, services = [], steps = steps, diff --git a/scripts/drone/pipelines/windows.star b/scripts/drone/pipelines/windows.star index f20759c18e7..fd5b99c9bf0 100644 --- a/scripts/drone/pipelines/windows.star +++ b/scripts/drone/pipelines/windows.star @@ -10,7 +10,6 @@ load( "scripts/drone/steps/lib.star", "get_windows_steps", "windows_clone_step", - "windows_init_enterprise_steps", "windows_test_backend_step", "windows_wire_install_step", ) @@ -34,23 +33,19 @@ def windows_test_backend(trigger, edition, ver_mode): windows_clone_step(), ] - if edition == "enterprise": - steps.extend(windows_init_enterprise_steps(ver_mode)) - else: - steps.extend([{ - "name": "windows-init", - "image": windows_images["windows_go_image"], - "depends_on": ["clone"], - "commands": [], - }]) + steps.extend([{ + "name": "windows-init", + "image": windows_images["windows_go_image"], + "depends_on": ["clone"], + "commands": [], + }]) steps.extend([ windows_wire_install_step(edition), windows_test_backend_step(), ]) pl = pipeline( - name = "{}-{}-test-backend-windows".format(ver_mode, edition), - edition = edition, + name = "{}-test-backend-windows".format(ver_mode), trigger = trigger, steps = steps, depends_on = [], @@ -62,25 +57,23 @@ def windows_test_backend(trigger, edition, ver_mode): } return pl -def windows(trigger, edition, ver_mode): +def windows(trigger, ver_mode): """Generates the pipeline used for building Grafana on Windows. Args: trigger: a Drone trigger for the pipeline. - edition: controls whether enterprise code is included in the pipeline steps. ver_mode: controls whether a pre-release or actual release pipeline is generated. Also indirectly controls which version of enterprise code is used. Returns: Drone pipeline. """ - environment = {"EDITION": edition} + environment = {"EDITION": "oss"} return pipeline( name = "main-windows", - edition = edition, trigger = dict(trigger, repo = ["grafana/grafana"]), - steps = get_windows_steps(edition, ver_mode), + steps = get_windows_steps(ver_mode), depends_on = [ "main-test-frontend", "main-test-backend", diff --git a/scripts/drone/rgm.star b/scripts/drone/rgm.star index b7e81d65925..2d78a8a8aaf 100644 --- a/scripts/drone/rgm.star +++ b/scripts/drone/rgm.star @@ -70,7 +70,6 @@ def rgm_main(): return pipeline( name = "rgm-main-prerelease", - edition = "all", trigger = trigger, steps = rgm_build(), depends_on = ["main-test-backend", "main-test-frontend"], @@ -95,7 +94,6 @@ tag_trigger = { def rgm_tag(): return pipeline( name = "rgm-tag-prerelease", - edition = "all", trigger = tag_trigger, steps = rgm_build(script = "drone_publish_tag.sh"), depends_on = [], diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 1a58673ae8a..9c526e694b8 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -88,130 +88,65 @@ def identify_runner_step(platform = "linux"): ], } -def clone_enterprise_step(committish = "${DRONE_COMMIT}"): +def enterprise_setup_step(source = "${DRONE_SOURCE_BRANCH}", canFail = True, isPromote = False): + """Setup the enterprise source into the ./grafana-enterprise directory. + + Args: + source: controls which revision of grafana-enterprise is checked out, if it exists. The name 'source' derives from the 'source branch' of a pull request. + canFail: controls whether the step can fail. This is useful for pull requests where the enterprise source may not exist. + isPromote: controls whether or not this step is being used in a promote pipeline. If it is, then the clone enterprise step will not check if the pull request is a fork. + Returns: + Drone step. + """ + step = clone_enterprise_step_pr(source = source, target = "${DRONE_TARGET_BRANCH}", canFail = canFail, location = "../grafana-enterprise", isPromote = isPromote) + step["commands"] += [ + "cd ../", + "ln -s src grafana", + "cd ./grafana-enterprise", + "./build.sh", + ] + + return step + +def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFail = False, location = "grafana-enterprise", isPromote = False): """Clone the enterprise source into the ./grafana-enterprise directory. Args: - committish: controls which revision of grafana-enterprise is cloned. - + source: controls which revision of grafana-enterprise is checked out, if it exists. The name 'source' derives from the 'source branch' of a pull request. + target: controls which revision of grafana-enterprise is checked out, if it 'source' does not exist. The name 'target' derives from the 'target branch' of a pull request. If this does not exist, then 'main' will be checked out. + canFail: controls whether or not this step is allowed to fail. If it fails and this is true, then the pipeline will continue. canFail is used in pull request pipelines where enterprise may be cloned but may not clone in forks. + location: the path where grafana-enterprise is cloned. + isPromote: controls whether or not this step is being used in a promote pipeline. If it is, then the step will not check if the pull request is a fork. Returns: Drone step. """ - return { + + if isPromote: + check = [] + else: + check = [ + 'is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork)', + 'if [ "$is_fork" != false ]; then return 1; fi', # Only clone if we're confident that 'fork' is 'false'. Fail if it's also empty. + ] + + step = { "name": "clone-enterprise", "image": images["build_image"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, "commands": [ - 'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git"', - "cd grafana-enterprise", - "git checkout {}".format(committish), + ] + check + [ + 'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" ' + location, + "cd {}".format(location), + 'if git checkout {0}; then echo "checked out {0}"; elif git checkout {1}; then echo "git checkout {1}"; else git checkout main; fi'.format(source, target), ], } -def init_enterprise_step(ver_mode): - """Adds the enterprise deployment configuration into the source directory. + if canFail: + step["failure"] = "ignore" - Args: - ver_mode: controls what revision of the OSS source to use. - If ver_mode is 'release', the step uses the tagged revision. - Otherwise, the DRONE_SOURCE_BRANCH is used. - - Returns: - Drone step. - """ - source_commit = "" - if ver_mode == "release": - source_commit = " ${DRONE_TAG}" - environment = { - "GITHUB_TOKEN": from_secret("github_token"), - } - token = "--github-token $${GITHUB_TOKEN}" - elif ver_mode == "release-branch": - environment = { - "GITHUB_TOKEN": from_secret("github_token"), - } - token = "--github-token $${GITHUB_TOKEN}" - else: - environment = {} - token = "" - return { - "name": "init-enterprise", - "image": images["build_image"], - "depends_on": [ - "clone-enterprise", - "grabpl", - ], - "environment": environment, - "commands": [ - "mv bin/grabpl /tmp/", - "rmdir bin", - "mv grafana-enterprise /tmp/", - "/tmp/grabpl init-enterprise {} /tmp/grafana-enterprise{}".format( - token, - source_commit, - ).rstrip(), - "mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json", - "mkdir bin", - "mv /tmp/grabpl bin/", - ], - } - -def windows_init_enterprise_steps(ver_mode): - """Performs init-enterprise steps in a Windows environment - - Args: - ver_mode: in what mode should this be run - - Returns: - A list of steps setting up an enterprise folder - """ - if ver_mode == "release": - source = "${DRONE_TAG}" - elif ver_mode == "release-branch": - source = "$$env:DRONE_BRANCH" - else: - source = "main" - - clone_cmds = [ - 'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"', - "cd grafana-enterprise", - "git checkout {}".format(source), - ] - - init_cmds = [ - # Need to move grafana-enterprise out of the way, so directory is empty and can be cloned into - "cp -r grafana-enterprise C:\\App\\grafana-enterprise", - "rm -r -force grafana-enterprise", - "cp grabpl.exe C:\\App\\grabpl.exe", - "rm -force grabpl.exe", - "C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise {}".format(source), - "cp C:\\App\\grabpl.exe grabpl.exe", - ] - - steps = [] - steps.extend( - [ - download_grabpl_step(platform = "windows"), - { - "name": "clone", - "image": windows_images["wix_image"], - "environment": { - "GITHUB_TOKEN": from_secret("github_token"), - }, - "commands": clone_cmds, - }, - { - "name": "windows-init", - "image": windows_images["wix_image"], - "commands": init_cmds, - "depends_on": ["clone"], - "environment": {"GITHUB_TOKEN": from_secret("github_token")}, - }, - ], - ) - return steps + return step def download_grabpl_step(platform = "linux"): if platform == "windows": @@ -443,11 +378,10 @@ def e2e_tests_artifacts(): ], } -def upload_cdn_step(edition, ver_mode, trigger = None): +def upload_cdn_step(ver_mode, trigger = None): """Uploads CDN assets using the Grafana build tool. Args: - edition: controls the output directory for the CDN assets. ver_mode: only uses the step trigger when ver_mode == 'release-branch' or 'main' trigger: a Drone trigger for the step. Defaults to None. @@ -455,41 +389,29 @@ def upload_cdn_step(edition, ver_mode, trigger = None): Returns: Drone step. """ - deps = [] - if edition in "enterprise2": - deps.extend( - [ - "package" + enterprise2_suffix(edition), - ], - ) - else: - deps.extend( - [ - "grafana-server", - ], - ) step = { - "name": "upload-cdn-assets" + enterprise2_suffix(edition), + "name": "upload-cdn-assets", "image": images["publish_image"], - "depends_on": deps, + "depends_on": [ + "grafana-server", + ], "environment": { "GCP_KEY": from_secret("gcp_key"), "PRERELEASE_BUCKET": from_secret(prerelease_bucket), }, "commands": [ - "./bin/build upload-cdn --edition {}".format(edition), + "./bin/build upload-cdn --edition oss", ], } if trigger and ver_mode in ("release-branch", "main"): step = dict(step, when = trigger) return step -def build_backend_step(edition, ver_mode, variants = None): +def build_backend_step(ver_mode, variants = None): """Build the backend code using the Grafana build tool. Args: - edition: controls which edition of the backend is built. ver_mode: if ver_mode != 'release', pass the DRONE_BUILD_NUMBER environment variable as the value for the --build-id option. TODO: is this option actually used by the build-backend subcommand? @@ -507,22 +429,19 @@ def build_backend_step(edition, ver_mode, variants = None): # TODO: Convert number of jobs to percentage if ver_mode == "release": cmds = [ - "./bin/build build-backend --jobs 8 --edition {} ${{DRONE_TAG}}".format( - edition, - ), + "./bin/build build-backend --jobs 8 --edition oss ${DRONE_TAG}", ] else: build_no = "${DRONE_BUILD_NUMBER}" cmds = [ - "./bin/build build-backend --jobs 8 --edition {} --build-id {}{}".format( - edition, + "./bin/build build-backend --jobs 8 --edition oss --build-id {}{}".format( build_no, variants_str, ), ] return { - "name": "build-backend" + enterprise2_suffix(edition), + "name": "build-backend", "image": images["build_image"], "depends_on": [ "wire-install", @@ -531,11 +450,10 @@ def build_backend_step(edition, ver_mode, variants = None): "commands": cmds, } -def build_frontend_step(edition, ver_mode): +def build_frontend_step(ver_mode): """Build the frontend code using the Grafana build tool. Args: - edition: controls which edition of the frontend is built. ver_mode: if ver_mode != 'release', use the DRONE_BUILD_NUMBER environment variable as a build identifier. @@ -548,11 +466,11 @@ def build_frontend_step(edition, ver_mode): if ver_mode == "release": cmds = [ "./bin/build build-frontend --jobs 8 " + - "--edition {} ${{DRONE_TAG}}".format(edition), + "--edition oss ${DRONE_TAG}", ] else: cmds = [ - "./bin/build build-frontend --jobs 8 --edition {} ".format(edition) + + "./bin/build build-frontend --jobs 8 --edition oss" + "--build-id {}".format(build_no), ] @@ -569,11 +487,10 @@ def build_frontend_step(edition, ver_mode): "commands": cmds, } -def build_frontend_package_step(edition, ver_mode): +def build_frontend_package_step(ver_mode): """Build the frontend packages using the Grafana build tool. Args: - edition: controls which edition of the frontend is built. ver_mode: if ver_mode != 'release', use the DRONE_BUILD_NUMBER environment variable as a build identifier. @@ -586,11 +503,11 @@ def build_frontend_package_step(edition, ver_mode): if ver_mode == "release": cmds = [ "./bin/build build-frontend-packages --jobs 8 " + - "--edition {} ${{DRONE_TAG}}".format(edition), + "--edition oss ${DRONE_TAG}", ] else: cmds = [ - "./bin/build build-frontend-packages --jobs 8 --edition {} ".format(edition) + + "./bin/build build-frontend-packages --jobs 8 --edition oss" + "--build-id {}".format(build_no), ] @@ -607,7 +524,7 @@ def build_frontend_package_step(edition, ver_mode): "commands": cmds, } -def build_plugins_step(edition, ver_mode): +def build_plugins_step(ver_mode): if ver_mode != "pr": env = { "GRAFANA_API_KEY": from_secret("grafana_api_key"), @@ -624,7 +541,7 @@ def build_plugins_step(edition, ver_mode): ], "commands": [ # TODO: Use percentage for num jobs - "./bin/build build-plugins --jobs 8 --edition {}".format(edition), + "./bin/build build-plugins --jobs 8 --edition oss", ], } @@ -656,50 +573,40 @@ def test_backend_integration_step(): ], } -def betterer_frontend_step(edition = "oss"): +def betterer_frontend_step(): """Run betterer on frontend code. - Args: - edition: controls whether enterprise code is also included in the source. - Defaults to 'oss'. - Returns: Drone step. """ - deps = [] - if edition == "enterprise": - deps.extend(["init-enterprise"]) - deps.extend(["yarn-install"]) + return { "name": "betterer-frontend", "image": images["build_image"], - "depends_on": deps, + "depends_on": [ + "yarn-install", + ], "commands": [ "yarn betterer ci", ], } -def test_frontend_step(edition = "oss"): +def test_frontend_step(): """Runs tests on frontend code. - Args: - edition: controls whether enterprise code is also included in the source. - Defaults to 'oss'. - Returns: Drone step. """ - deps = [] - if edition == "enterprise": - deps.extend(["init-enterprise"]) - deps.extend(["yarn-install"]) + return { "name": "test-frontend", "image": images["build_image"], "environment": { "TEST_MAX_WORKERS": "50%", }, - "depends_on": deps, + "depends_on": [ + "yarn-install", + ], "commands": [ "yarn run ci:test-frontend", ], @@ -809,11 +716,10 @@ def codespell_step(): ], } -def package_step(edition, ver_mode): +def package_step(ver_mode): """Packages Grafana with the Grafana build tool. Args: - edition: controls which edition of Grafana is packaged. ver_mode: controls whether the packages are signed for a release. If ver_mode != 'release', use the DRONE_BUILD_NUMBER environment variable as a build identifier. @@ -823,7 +729,7 @@ def package_step(edition, ver_mode): """ deps = [ "build-plugins", - "build-backend" + enterprise2_suffix(edition), + "build-backend", "build-frontend", "build-frontend-packages", ] @@ -847,29 +753,28 @@ def package_step(edition, ver_mode): # TODO: Use percentage for jobs if ver_mode == "release": cmds = [ - "{}./bin/build package --jobs 8 --edition {} ".format(test_args, edition) + - "{} ${{DRONE_TAG}}".format(sign_args), + "{}./bin/build package --jobs 8 --edition oss ".format(test_args) + + "{} $${{DRONE_TAG}}".format(sign_args), ] else: build_no = "${DRONE_BUILD_NUMBER}" cmds = [ - "{}./bin/build package --jobs 8 --edition {} ".format(test_args, edition) + + "{}./bin/build package --jobs 8 --edition oss ".format(test_args) + "--build-id {}{}".format(build_no, sign_args), ] return { - "name": "package" + enterprise2_suffix(edition), + "name": "package", "image": images["build_image"], "depends_on": deps, "environment": env, "commands": cmds, } -def grafana_server_step(edition, port = 3001): +def grafana_server_step(port = 3001): """Runs the grafana-server binary as a service. Args: - edition: controls which edition of grafana-server to run. port: port to listen on. Defaults to 3001. @@ -877,8 +782,6 @@ def grafana_server_step(edition, port = 3001): Drone step. """ environment = {"PORT": port, "ARCH": "linux-amd64"} - if edition == "enterprise": - environment["RUNDIR"] = "scripts/grafana-server/tmp-grafana-enterprise" return { "name": "grafana-server", @@ -976,12 +879,12 @@ def build_docs_website_step(): ], } -def copy_packages_for_docker_step(edition = None): +def copy_packages_for_docker_step(): return { "name": "copy-packages-for-docker", "image": images["build_image"], "depends_on": [ - "package" + enterprise2_suffix(edition), + "package", ], "commands": [ "ls dist/*.tar.gz*", @@ -989,11 +892,10 @@ def copy_packages_for_docker_step(edition = None): ], } -def build_docker_images_step(edition, archs = None, ubuntu = False, publish = False): +def build_docker_images_step(archs = None, ubuntu = False, publish = False): """Build Docker images using the Grafana build tool. Args: - edition: controls which repository the image is published to. archs: a list of architectures to build the image for. Defaults to None. ubuntu: controls whether the final image is built from an Ubuntu base image. @@ -1004,7 +906,7 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa Returns: Drone step. """ - cmd = "./bin/build build-docker --edition {}".format(edition) + cmd = "./bin/build build-docker --edition oss" if publish: cmd += " --shouldSave" @@ -1020,11 +922,6 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa "GCP_KEY": from_secret("gcp_key"), } - if edition == "enterprise2": - environment.update( - {"DOCKER_ENTERPRISE2_REPO": from_secret("docker_enterprise2_repo")}, - ) - return { "name": "build-docker-images" + ubuntu_sfx, "image": images["cloudsdk_image"], @@ -1037,27 +934,24 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa "environment": environment, } -def fetch_images_step(edition): +def fetch_images_step(): return { - "name": "fetch-images-{}".format(edition), + "name": "fetch-images", "image": images["cloudsdk_image"], "environment": { "GCP_KEY": from_secret("gcp_key"), "DOCKER_USER": from_secret("docker_username"), "DOCKER_PASSWORD": from_secret("docker_password"), - "DOCKER_ENTERPRISE2_REPO": from_secret("docker_enterprise2_repo"), }, - "commands": ["./bin/build artifacts docker fetch --edition {}".format(edition)], + "commands": ["./bin/build artifacts docker fetch --edition oss"], "depends_on": ["compile-build-cmd"], "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], } -def publish_images_step(edition, ver_mode, docker_repo, trigger = None): +def publish_images_step(ver_mode, docker_repo, trigger = None): """Generates a step for publishing public Docker images with grabpl. Args: - edition: controls which version of an image is fetched in the case of a release. - It also controls which publishing implementation is used. ver_mode: controls whether the image needs to be built or retrieved from a previous build. If ver_mode == 'release', the previously built image is fetched instead of being built again. docker_repo: the Docker image name. @@ -1086,22 +980,9 @@ def publish_images_step(edition, ver_mode, docker_repo, trigger = None): deps = ["build-docker-images", "build-docker-images-ubuntu"] if ver_mode == "release": - deps = ["fetch-images-{}".format(edition)] + deps = ["fetch-images"] cmd += " --version-tag ${DRONE_TAG}" - if edition == "enterprise2": - name = edition - docker_repo = "$${DOCKER_ENTERPRISE2_REPO}" - environment.update( - { - "GCP_KEY": from_secret("gcp_key_hg"), - "DOCKER_ENTERPRISE2_REPO": from_secret("docker_enterprise2_repo"), - }, - ) - cmd = "./bin/build artifacts docker publish-enterprise2 --dockerhub-repo {}".format( - docker_repo, - ) - if ver_mode == "pr": environment = { "DOCKER_USER": from_secret("docker_username_pr"), @@ -1225,16 +1106,10 @@ def release_canary_npm_packages_step(trigger = None): step = dict(step, when = trigger) return step -def enterprise2_suffix(edition): - if edition == "enterprise2": - return "-{}".format(edition) - return "" - -def upload_packages_step(edition, ver_mode, trigger = None): +def upload_packages_step(ver_mode, trigger = None): """Upload packages to object storage. Args: - edition: controls which edition of Grafana packages to upload. ver_mode: when ver_mode == 'main', inhibit upload of enterprise edition packages when executed. trigger: a Drone trigger for the step. @@ -1244,7 +1119,7 @@ def upload_packages_step(edition, ver_mode, trigger = None): Drone step. """ step = { - "name": "upload-packages" + enterprise2_suffix(edition), + "name": "upload-packages", "image": images["publish_image"], "depends_on": end_to_end_tests_deps(), "environment": { @@ -1252,18 +1127,17 @@ def upload_packages_step(edition, ver_mode, trigger = None): "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), }, "commands": [ - "./bin/build upload-packages --edition {}".format(edition), + "./bin/build upload-packages --edition oss", ], } if trigger and ver_mode in ("release-branch", "main"): step = dict(step, when = trigger) return step -def publish_grafanacom_step(edition, ver_mode): +def publish_grafanacom_step(ver_mode): """Publishes Grafana packages to grafana.com. Args: - edition: controls which edition of Grafana to publish to. ver_mode: if ver_mode == 'main', pass the DRONE_BUILD_NUMBER environment variable as the value for the --build-id option. TODO: is this actually used by the grafanacom subcommand? I think it might @@ -1273,20 +1147,17 @@ def publish_grafanacom_step(edition, ver_mode): Drone step. """ if ver_mode == "release": - cmd = "./bin/build publish grafana-com --edition {} ${{DRONE_TAG}}".format( - edition, - ) + cmd = "./bin/build publish grafana-com --edition oss ${DRONE_TAG}" elif ver_mode == "main": build_no = "${DRONE_BUILD_NUMBER}" - cmd = "./bin/build publish grafana-com --edition {} --build-id {}".format( - edition, + cmd = "./bin/build publish grafana-com --edition oss --build-id {}".format( build_no, ) else: fail("Unexpected version mode {}".format(ver_mode)) return { - "name": "publish-grafanacom-{}".format(edition), + "name": "publish-grafanacom", "image": images["publish_image"], "depends_on": [ "publish-linux-packages-deb", @@ -1301,7 +1172,7 @@ def publish_grafanacom_step(edition, ver_mode): ], } -def publish_linux_packages_step(edition, package_manager = "deb"): +def publish_linux_packages_step(package_manager = "deb"): return { "name": "publish-linux-packages-{}".format(package_manager), # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image @@ -1317,8 +1188,7 @@ def publish_linux_packages_step(edition, package_manager = "deb"): "gpg_passphrase": from_secret("packages_gpg_passphrase"), "gpg_public_key": from_secret("packages_gpg_public_key"), "gpg_private_key": from_secret("packages_gpg_private_key"), - "package_path": "gs://grafana-prerelease/artifacts/downloads/*${{DRONE_TAG}}/{}/**.{}".format( - edition, + "package_path": "gs://grafana-prerelease/artifacts/downloads/*$${{DRONE_TAG}}/oss/**.{}".format( package_manager, ), }, @@ -1337,11 +1207,10 @@ def windows_clone_step(): ], } -def get_windows_steps(edition, ver_mode): +def get_windows_steps(ver_mode): """Generate the list of Windows steps. Args: - edition: used to differentiate steps for different Grafana editions. ver_mode: used to differentiate steps for different version modes. Returns: @@ -1351,80 +1220,24 @@ def get_windows_steps(edition, ver_mode): identify_runner_step("windows"), ] - if edition in ("enterprise", "enterprise2"): - if ver_mode == "release": - committish = "${DRONE_TAG}" - elif ver_mode == "release-branch": - committish = "$$env:DRONE_BRANCH" - else: - committish = "$$env:DRONE_COMMIT" + init_cmds = [ + '$$ProgressPreference = "SilentlyContinue"', + "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( + grabpl_version, + ), + ] - # For enterprise, we have to clone both OSS and enterprise and merge the latter into the former - download_grabpl_cmds = [ - '$$ProgressPreference = "SilentlyContinue"', - "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( - grabpl_version, - ), - ] + steps.extend( + [ + { + "name": "windows-init", + "image": windows_images["wix_image"], + "commands": init_cmds, + }, + ], + ) - clone_cmds = [ - 'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"', - "cd grafana-enterprise", - "git checkout {}".format(committish), - ] - - init_cmds = [ - # Need to move grafana-enterprise out of the way, so directory is empty and can be cloned into - "cp -r grafana-enterprise C:\\App\\grafana-enterprise", - "rm -r -force grafana-enterprise", - "cp grabpl.exe C:\\App\\grabpl.exe", - "rm -force grabpl.exe", - "C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise {}".format(committish), - "cp C:\\App\\grabpl.exe grabpl.exe", - ] - - steps.extend( - [ - { - "name": "clone", - "image": windows_images["wix_image"], - "environment": { - "GITHUB_TOKEN": from_secret("github_token"), - }, - "commands": download_grabpl_cmds + clone_cmds, - }, - { - "name": "windows-init", - "image": windows_images["wix_image"], - "commands": init_cmds, - "depends_on": ["clone"], - "environment": {"GITHUB_TOKEN": from_secret("github_token")}, - }, - ], - ) - else: - init_cmds = [ - '$$ProgressPreference = "SilentlyContinue"', - "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( - grabpl_version, - ), - ] - - steps.extend( - [ - { - "name": "windows-init", - "image": windows_images["wix_image"], - "commands": init_cmds, - }, - ], - ) - - # TODO: Run windows backend tests - - if ( - ver_mode == "main" and (edition not in ("enterprise", "enterprise2")) - ) or ver_mode in ( + if ver_mode in ( "release", "release-branch", ): @@ -1446,13 +1259,11 @@ def get_windows_steps(edition, ver_mode): "rm gcpkey.json", "cp C:\\App\\nssm-2.24.zip .", ] - if ( - ver_mode == "main" and (edition not in ("enterprise", "enterprise2")) - ) or ver_mode in ("release",): + + if ver_mode in ("release",): installer_commands.extend( [ - ".\\grabpl.exe windows-installer --edition {} {}".format( - edition, + ".\\grabpl.exe windows-installer --edition oss {}".format( ver_part, ), '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', @@ -1461,10 +1272,9 @@ def get_windows_steps(edition, ver_mode): if ver_mode == "main": installer_commands.extend( [ - "gsutil cp $$fname gs://{}/{}/{}/".format(bucket, edition, dir), - 'gsutil cp "$$fname.sha256" gs://{}/{}/{}/'.format( + "gsutil cp $$fname gs://{}/oss/{}/".format(bucket, dir), + 'gsutil cp "$$fname.sha256" gs://{}/oss/{}/'.format( bucket, - edition, dir, ), ], @@ -1472,16 +1282,14 @@ def get_windows_steps(edition, ver_mode): else: installer_commands.extend( [ - "gsutil cp $$fname gs://{}/{}/{}/{}/".format( + "gsutil cp $$fname gs://{}/{}/oss/{}/".format( bucket, ver_part, - edition, dir, ), - 'gsutil cp "$$fname.sha256" gs://{}/{}/{}/{}/'.format( + 'gsutil cp "$$fname.sha256" gs://{}/{}/oss/{}/'.format( bucket, ver_part, - edition, dir, ), ], @@ -1570,12 +1378,9 @@ def end_to_end_tests_deps(): "end-to-end-tests-various-suite", ] -def compile_build_cmd(edition = "oss"): +def compile_build_cmd(): dependencies = [] - if edition in ("enterprise", "enterprise2"): - dependencies = [ - "init-enterprise", - ] + return { "name": "compile-build-cmd", "image": images["go_image"], diff --git a/scripts/drone/utils/utils.star b/scripts/drone/utils/utils.star index 82af7ca9604..e3756297464 100644 --- a/scripts/drone/utils/utils.star +++ b/scripts/drone/utils/utils.star @@ -13,7 +13,6 @@ drone_change_template = "`.drone.yml` and `starlark` files have been changed on def pipeline( name, - edition, trigger, steps, services = [], @@ -30,7 +29,6 @@ def pipeline( Args: name: controls the pipeline name. - edition: used to differentiate the pipeline for enterprise builds. trigger: a Drone trigger for the pipeline. steps: the Drone steps for the pipeline. services: auxilliary services used during the pipeline. @@ -98,12 +96,6 @@ def pipeline( pipeline["volumes"].extend(volumes) pipeline.update(platform_conf) - if edition in ("enterprise", "enterprise2"): - # We have a custom clone step for enterprise - pipeline["clone"] = { - "disable": True, - } - return pipeline def notify_pipeline( diff --git a/scripts/drone/vault.star b/scripts/drone/vault.star index 7fba1e1c613..fb1178385da 100644 --- a/scripts/drone/vault.star +++ b/scripts/drone/vault.star @@ -90,21 +90,6 @@ def secrets(): "infra/data/ci/packages-publish/bucket-credentials", "Secret", ), - vault_secret( - "aws_region", - "secret/data/common/aws-marketplace", - "aws_region", - ), - vault_secret( - "aws_access_key_id", - "secret/data/common/aws-marketplace", - "aws_access_key_id", - ), - vault_secret( - "aws_secret_access_key", - "secret/data/common/aws-marketplace", - "aws_secret_access_key", - ), vault_secret( "static_asset_editions", "infra/data/ci/grafana-release-eng/artifact-publishing", diff --git a/scripts/drone/version.star b/scripts/drone/version.star index f56a8274000..140cd4d35d8 100644 --- a/scripts/drone/version.star +++ b/scripts/drone/version.star @@ -4,8 +4,6 @@ This module returns the pipeline used for version branches. load( "scripts/drone/events/release.star", - "enterprise2_pipelines", - "enterprise_pipelines", "oss_pipelines", ) @@ -14,7 +12,5 @@ trigger = {"ref": ["refs/heads/v[0-9]*"]} def version_branch_pipelines(): return ( - oss_pipelines(ver_mode = ver_mode, trigger = trigger) + - enterprise_pipelines(ver_mode = ver_mode, trigger = trigger) + - enterprise2_pipelines(ver_mode = ver_mode, trigger = trigger) + oss_pipelines(ver_mode = ver_mode, trigger = trigger) )