diff --git a/.drone.star b/.drone.star index 9d2a65ebc63..2504f1a394c 100644 --- a/.drone.star +++ b/.drone.star @@ -7,7 +7,7 @@ load('scripts/drone/events/pr.star', 'pr_pipelines') load('scripts/drone/events/main.star', 'main_pipelines') load('scripts/drone/pipelines/docs.star', 'docs_pipelines') -load('scripts/drone/events/release.star', 'release_pipelines', 'publish_artifacts_pipelines', 'publish_npm_pipelines', 'publish_packages_pipeline', 'artifacts_page_pipeline') +load('scripts/drone/events/release.star', 'oss_pipelines', 'enterprise_pipelines', 'enterprise2_pipelines', 'publish_artifacts_pipelines', 'publish_npm_pipelines', 'publish_packages_pipeline', 'artifacts_page_pipeline') load('scripts/drone/pipelines/publish_images.star', 'publish_image_pipelines_public', 'publish_image_pipelines_security') load('scripts/drone/version.star', 'version_branch_pipelines') load('scripts/drone/events/cron.star', 'cronjobs') @@ -15,7 +15,8 @@ load('scripts/drone/vault.star', 'secrets') def main(ctx): edition = 'oss' - return pr_pipelines(edition=edition) + main_pipelines(edition=edition) + release_pipelines() + \ + return pr_pipelines(edition=edition) + main_pipelines(edition=edition) + oss_pipelines() + enterprise_pipelines() + enterprise2_pipelines() + \ + enterprise2_pipelines(prefix='custom-', trigger = {'event': ['custom']},) + \ publish_image_pipelines_public() + publish_image_pipelines_security() + \ publish_artifacts_pipelines('security') + publish_artifacts_pipelines('public') + \ publish_npm_pipelines('public') + publish_packages_pipeline() + artifacts_page_pipeline() + \ diff --git a/.drone.yml b/.drone.yml index 35a0338455a..5fb3cee20a8 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1535,7 +1535,7 @@ steps: GCP_KEY: from_secret: gcp_key image: google/cloud-sdk - name: publish-images-grafana/grafana + name: publish-images-grafana volumes: - name: docker path: /var/run/docker.sock @@ -1555,7 +1555,7 @@ steps: GCP_KEY: from_secret: gcp_key image: google/cloud-sdk - name: publish-images-grafana/grafana-oss + name: publish-images-grafana-oss volumes: - name: docker path: /var/run/docker.sock @@ -2692,7 +2692,6 @@ steps: - build-backend - build-frontend - build-frontend-packages - - build-backend-enterprise2 environment: GPG_KEY_PASSWORD: from_secret: gpg_key_password @@ -2844,10 +2843,9 @@ steps: - ./bin/build package --jobs 8 --edition enterprise2 --sign ${DRONE_TAG} depends_on: - build-plugins - - build-backend + - build-backend-enterprise2 - build-frontend - build-frontend-packages - - build-backend-enterprise2 environment: GPG_KEY_PASSWORD: from_secret: gpg_key_password @@ -3357,6 +3355,503 @@ volumes: path: /var/run/docker.sock name: docker --- +clone: + disable: true +depends_on: [] +environment: + EDITION: enterprise2 +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: release-enterprise2-build-e2e-publish +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.17/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15.6 + name: identify-runner +- commands: + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + - cd grafana-enterprise + - git checkout ${DRONE_TAG} + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.6.4 + name: clone-enterprise +- commands: + - mv bin/grabpl /tmp/ + - rmdir bin + - mv grafana-enterprise /tmp/ + - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise + ${DRONE_TAG} + - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json + - mkdir bin + - mv /tmp/grabpl bin/ + depends_on: + - clone-enterprise + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.6.4 + name: init-enterprise +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: + - init-enterprise + environment: + CGO_ENABLED: 0 + image: golang:1.19.3 + name: compile-build-cmd +- commands: + - make gen-go + depends_on: + - init-enterprise + image: grafana/build-container:1.6.4 + name: wire-install +- commands: + - yarn install --immutable + depends_on: + - init-enterprise + image: grafana/build-container:1.6.4 + name: yarn-install +- commands: + - '# It is required that code generated from Thema/CUE be committed and in sync + with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - CODEGEN_VERIFY=1 make gen-cue + depends_on: + - init-enterprise + image: grafana/build-container:1.6.4 + name: verify-gen-cue +- commands: + - ./bin/build build-frontend --jobs 8 --edition enterprise ${DRONE_TAG} + depends_on: + - compile-build-cmd + - yarn-install + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: grafana/build-container:1.6.4 + name: build-frontend +- commands: + - ./bin/build build-frontend-packages --jobs 8 --edition enterprise ${DRONE_TAG} + depends_on: + - compile-build-cmd + - yarn-install + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: grafana/build-container:1.6.4 + name: build-frontend-packages +- commands: + - ./bin/build build-plugins --jobs 8 --edition enterprise + depends_on: + - compile-build-cmd + - yarn-install + environment: + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:1.6.4 + name: build-plugins +- commands: + - ./bin/build build-backend --jobs 8 --edition enterprise2 ${DRONE_TAG} + depends_on: + - wire-install + - compile-build-cmd + image: grafana/build-container:1.6.4 + name: build-backend-enterprise2 +- commands: + - ./bin/build package --jobs 8 --edition enterprise2 --sign ${DRONE_TAG} + depends_on: + - build-plugins + - build-backend-enterprise2 + - build-frontend + - build-frontend-packages + environment: + GPG_KEY_PASSWORD: + from_secret: gpg_key_password + GPG_PRIV_KEY: + from_secret: gpg_priv_key + GPG_PUB_KEY: + from_secret: gpg_pub_key + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:1.6.4 + name: package-enterprise2 +- commands: + - ./bin/build upload-cdn --edition enterprise2 + depends_on: + - package-enterprise2 + environment: + ENTERPRISE2_CDN_PATH: + from_secret: enterprise2-cdn-path + GCP_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.3 + name: upload-cdn-assets-enterprise2 +- commands: + - ls dist/*.tar.gz* + - cp dist/*.tar.gz* packaging/docker/ + depends_on: + - package-enterprise2 + image: grafana/build-container:1.6.4 + name: copy-packages-for-docker +- commands: + - ./bin/build build-docker --edition enterprise2 --shouldSave + depends_on: + - copy-packages-for-docker + - compile-build-cmd + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: build-docker-images + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build build-docker --edition enterprise2 --shouldSave --ubuntu + depends_on: + - copy-packages-for-docker + - compile-build-cmd + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: build-docker-images-ubuntu + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build artifacts docker fetch --edition enterprise2 + depends_on: + - build-docker-images + - build-docker-images-ubuntu + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: fetch-images-enterprise2 + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build artifacts docker publish-enterprise2 --dockerhub-repo $${DOCKER_ENTERPRISE2_REPO} + depends_on: + - fetch-images-enterprise2 + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key_hg + image: google/cloud-sdk + name: publish-images-enterprise2 + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build upload-packages --edition enterprise2 + depends_on: + - package-enterprise2 + environment: + GCP_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.3 + name: upload-packages-enterprise2 +trigger: + event: + exclude: + - promote + ref: + - refs/tags/v* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory +--- +clone: + disable: true +depends_on: [] +environment: + EDITION: enterprise2 +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: custom-release-enterprise2-build-e2e-publish +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.17/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15.6 + name: identify-runner +- commands: + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + - cd grafana-enterprise + - git checkout ${DRONE_TAG} + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.6.4 + name: clone-enterprise +- commands: + - mv bin/grabpl /tmp/ + - rmdir bin + - mv grafana-enterprise /tmp/ + - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise + ${DRONE_TAG} + - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json + - mkdir bin + - mv /tmp/grabpl bin/ + depends_on: + - clone-enterprise + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.6.4 + name: init-enterprise +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: + - init-enterprise + environment: + CGO_ENABLED: 0 + image: golang:1.19.3 + name: compile-build-cmd +- commands: + - make gen-go + depends_on: + - init-enterprise + image: grafana/build-container:1.6.4 + name: wire-install +- commands: + - yarn install --immutable + depends_on: + - init-enterprise + image: grafana/build-container:1.6.4 + name: yarn-install +- commands: + - '# It is required that code generated from Thema/CUE be committed and in sync + with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - CODEGEN_VERIFY=1 make gen-cue + depends_on: + - init-enterprise + image: grafana/build-container:1.6.4 + name: verify-gen-cue +- commands: + - ./bin/build build-frontend --jobs 8 --edition enterprise ${DRONE_TAG} + depends_on: + - compile-build-cmd + - yarn-install + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: grafana/build-container:1.6.4 + name: build-frontend +- commands: + - ./bin/build build-frontend-packages --jobs 8 --edition enterprise ${DRONE_TAG} + depends_on: + - compile-build-cmd + - yarn-install + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: grafana/build-container:1.6.4 + name: build-frontend-packages +- commands: + - ./bin/build build-plugins --jobs 8 --edition enterprise + depends_on: + - compile-build-cmd + - yarn-install + environment: + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:1.6.4 + name: build-plugins +- commands: + - ./bin/build build-backend --jobs 8 --edition enterprise2 ${DRONE_TAG} + depends_on: + - wire-install + - compile-build-cmd + image: grafana/build-container:1.6.4 + name: build-backend-enterprise2 +- commands: + - ./bin/build package --jobs 8 --edition enterprise2 --sign ${DRONE_TAG} + depends_on: + - build-plugins + - build-backend-enterprise2 + - build-frontend + - build-frontend-packages + environment: + GPG_KEY_PASSWORD: + from_secret: gpg_key_password + GPG_PRIV_KEY: + from_secret: gpg_priv_key + GPG_PUB_KEY: + from_secret: gpg_pub_key + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:1.6.4 + name: package-enterprise2 +- commands: + - ./bin/build upload-cdn --edition enterprise2 + depends_on: + - package-enterprise2 + environment: + ENTERPRISE2_CDN_PATH: + from_secret: enterprise2-cdn-path + GCP_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.3 + name: upload-cdn-assets-enterprise2 +- commands: + - ls dist/*.tar.gz* + - cp dist/*.tar.gz* packaging/docker/ + depends_on: + - package-enterprise2 + image: grafana/build-container:1.6.4 + name: copy-packages-for-docker +- commands: + - ./bin/build build-docker --edition enterprise2 --shouldSave + depends_on: + - copy-packages-for-docker + - compile-build-cmd + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: build-docker-images + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build build-docker --edition enterprise2 --shouldSave --ubuntu + depends_on: + - copy-packages-for-docker + - compile-build-cmd + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: build-docker-images-ubuntu + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build artifacts docker fetch --edition enterprise2 + depends_on: + - build-docker-images + - build-docker-images-ubuntu + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: fetch-images-enterprise2 + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build artifacts docker publish-enterprise2 --dockerhub-repo $${DOCKER_ENTERPRISE2_REPO} + depends_on: + - fetch-images-enterprise2 + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key_hg + image: google/cloud-sdk + name: publish-images-enterprise2 + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build upload-packages --edition enterprise2 + depends_on: + - package-enterprise2 + environment: + GCP_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.3 + name: upload-packages-enterprise2 +trigger: + event: + - custom +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory +--- clone: retries: 3 depends_on: [] @@ -3391,6 +3886,8 @@ steps: depends_on: - compile-build-cmd environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo DOCKER_PASSWORD: from_secret: docker_password DOCKER_USER: @@ -3415,13 +3912,13 @@ steps: GCP_KEY: from_secret: gcp_key image: google/cloud-sdk - name: publish-images-grafana/grafana + name: publish-images-grafana volumes: - name: docker path: /var/run/docker.sock - commands: - - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana-oss --version-tag - ${DRONE_TAG} + - ./bin/grabpl artifacts docker publish --dockerhub-repo grafana/grafana/grafana-oss + --version-tag ${DRONE_TAG} depends_on: - fetch-images-oss environment: @@ -3481,6 +3978,8 @@ steps: depends_on: - compile-build-cmd environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo DOCKER_PASSWORD: from_secret: docker_password DOCKER_USER: @@ -3505,7 +4004,7 @@ steps: GCP_KEY: from_secret: gcp_key image: google/cloud-sdk - name: publish-images-grafana/grafana-enterprise + name: publish-images-grafana-enterprise volumes: - name: docker path: /var/run/docker.sock @@ -3554,6 +4053,8 @@ steps: depends_on: - compile-build-cmd environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo DOCKER_PASSWORD: from_secret: docker_password DOCKER_USER: @@ -3578,7 +4079,7 @@ steps: GCP_KEY: from_secret: gcp_key image: google/cloud-sdk - name: publish-images-grafana/grafana-enterprise + name: publish-images-grafana-enterprise volumes: - name: docker path: /var/run/docker.sock @@ -4693,7 +5194,6 @@ steps: - build-backend - build-frontend - build-frontend-packages - - build-backend-enterprise2 environment: GPG_KEY_PASSWORD: from_secret: gpg_key_password @@ -4852,10 +5352,9 @@ steps: --variants linux-amd64 --sign depends_on: - build-plugins - - build-backend + - build-backend-enterprise2 - build-frontend - build-frontend-packages - - build-backend-enterprise2 environment: GPG_KEY_PASSWORD: from_secret: gpg_key_password @@ -5332,6 +5831,254 @@ volumes: path: /var/run/docker.sock name: docker --- +clone: + disable: true +depends_on: [] +environment: + EDITION: enterprise2 +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: release-branch-enterprise2-build-e2e-publish +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.17/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15.6 + name: identify-runner +- commands: + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + - cd grafana-enterprise + - git checkout ${DRONE_BRANCH} + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.6.4 + name: clone-enterprise +- commands: + - mv bin/grabpl /tmp/ + - rmdir bin + - mv grafana-enterprise /tmp/ + - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise + - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json + - mkdir bin + - mv /tmp/grabpl bin/ + depends_on: + - clone-enterprise + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.6.4 + name: init-enterprise +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: + - init-enterprise + environment: + CGO_ENABLED: 0 + image: golang:1.19.3 + name: compile-build-cmd +- commands: + - make gen-go + depends_on: + - init-enterprise + image: grafana/build-container:1.6.4 + name: wire-install +- commands: + - yarn install --immutable + depends_on: + - init-enterprise + image: grafana/build-container:1.6.4 + name: yarn-install +- commands: + - '# It is required that code generated from Thema/CUE be committed and in sync + with its inputs.' + - '# The following command will fail if running code generators produces any diff + in output.' + - CODEGEN_VERIFY=1 make gen-cue + depends_on: + - init-enterprise + image: grafana/build-container:1.6.4 + name: verify-gen-cue +- commands: + - ./bin/build build-frontend --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} + depends_on: + - compile-build-cmd + - yarn-install + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: grafana/build-container:1.6.4 + name: build-frontend +- commands: + - ./bin/build build-frontend-packages --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} + depends_on: + - compile-build-cmd + - yarn-install + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: grafana/build-container:1.6.4 + name: build-frontend-packages +- commands: + - ./bin/build build-plugins --jobs 8 --edition enterprise + depends_on: + - compile-build-cmd + - yarn-install + environment: + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:1.6.4 + name: build-plugins +- commands: + - ./bin/build build-backend --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER} + --variants linux-amd64 + depends_on: + - wire-install + - compile-build-cmd + image: grafana/build-container:1.6.4 + name: build-backend-enterprise2 +- commands: + - ./bin/build package --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER} + --variants linux-amd64 --sign + depends_on: + - build-plugins + - build-backend-enterprise2 + - build-frontend + - build-frontend-packages + environment: + GPG_KEY_PASSWORD: + from_secret: gpg_key_password + GPG_PRIV_KEY: + from_secret: gpg_priv_key + GPG_PUB_KEY: + from_secret: gpg_pub_key + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:1.6.4 + name: package-enterprise2 +- commands: + - ./bin/build upload-cdn --edition enterprise2 + depends_on: + - package-enterprise2 + environment: + ENTERPRISE2_CDN_PATH: + from_secret: enterprise2-cdn-path + GCP_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.3 + name: upload-cdn-assets-enterprise2 +- commands: + - ls dist/*.tar.gz* + - cp dist/*.tar.gz* packaging/docker/ + depends_on: + - package-enterprise2 + image: grafana/build-container:1.6.4 + name: copy-packages-for-docker +- commands: + - ./bin/build build-docker --edition enterprise2 --shouldSave + depends_on: + - copy-packages-for-docker + - compile-build-cmd + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: build-docker-images + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build build-docker --edition enterprise2 --shouldSave --ubuntu + depends_on: + - copy-packages-for-docker + - compile-build-cmd + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: build-docker-images-ubuntu + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build artifacts docker fetch --edition enterprise2 + depends_on: + - build-docker-images + - build-docker-images-ubuntu + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: fetch-images-enterprise2 + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build artifacts docker publish-enterprise2 --dockerhub-repo $${DOCKER_ENTERPRISE2_REPO} + depends_on: + - fetch-images-enterprise2 + environment: + DOCKER_ENTERPRISE2_REPO: + from_secret: docker_enterprise2_repo + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USER: + from_secret: docker_username + GCP_KEY: + from_secret: gcp_key_hg + image: google/cloud-sdk + name: publish-images-enterprise2 + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - ./bin/build upload-packages --edition enterprise2 + depends_on: + - package-enterprise2 + environment: + GCP_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.3 + name: upload-packages-enterprise2 +trigger: + ref: + - refs/heads/v[0-9]* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory +--- clone: retries: 3 kind: pipeline @@ -5570,6 +6317,6 @@ kind: secret name: packages_secret_access_key --- kind: signature -hmac: 2f3c13716c7cd1e42004a0f0d7057d5024d38bceedb22370b3554a79e786a46c +hmac: 1eca02a75c7e67666e8be67a9f121b9a70f5c280ee4d231a647be9d696e267dc ... diff --git a/pkg/build/cmd/genversions.go b/pkg/build/cmd/genversions.go index 81879f42b55..0d293499430 100644 --- a/pkg/build/cmd/genversions.go +++ b/pkg/build/cmd/genversions.go @@ -19,6 +19,11 @@ func GenerateMetadata(c *cli.Context) (config.Metadata, error) { return config.Metadata{}, err } + tag, ok := os.LookupEnv("DRONE_TAG") + if !ok { + fmt.Println("DRONE_TAG envvar not present, %w", err) + } + var releaseMode config.ReleaseMode switch event { case string(config.PullRequestMode): @@ -30,6 +35,13 @@ func GenerateMetadata(c *cli.Context) (config.Metadata, error) { } releaseMode = config.ReleaseMode{Mode: mode} case config.Custom: + if edition, _ := os.LookupEnv("EDITION"); edition == string(config.EditionEnterprise2) { + releaseMode = config.ReleaseMode{Mode: config.TagMode} + if tag != "" { + version = strings.TrimPrefix(tag, "v") + } + break + } mode, err := config.CheckDroneTargetBranch() if err != nil { return config.Metadata{}, err @@ -41,9 +53,8 @@ func GenerateMetadata(c *cli.Context) (config.Metadata, error) { releaseMode = config.ReleaseMode{Mode: mode} } case config.Tag, config.Promote: - tag, ok := os.LookupEnv("DRONE_TAG") - if !ok || tag == "" { - return config.Metadata{}, err + if tag == "" { + return config.Metadata{}, fmt.Errorf("DRONE_TAG envvar not present for a tag/promotion event, %w", err) } version = strings.TrimPrefix(tag, "v") mode, err := config.CheckSemverSuffix() diff --git a/pkg/build/cmd/main.go b/pkg/build/cmd/main.go index c523b2a755c..a75033a6ade 100644 --- a/pkg/build/cmd/main.go +++ b/pkg/build/cmd/main.go @@ -187,6 +187,18 @@ func main() { &editionFlag, }, }, + { + Name: "publish-enterprise2", + Usage: "Handle Grafana Enterprise2 Docker images", + ArgsUsage: "[version]", + Action: Enterprise2, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "dockerhub-repo", + Usage: "DockerHub repo to push images", + }, + }, + }, }, }, }, diff --git a/pkg/build/cmd/publishimages_enterprise2.go b/pkg/build/cmd/publishimages_enterprise2.go new file mode 100644 index 00000000000..80ca4188684 --- /dev/null +++ b/pkg/build/cmd/publishimages_enterprise2.go @@ -0,0 +1,100 @@ +package main + +import ( + "fmt" + "log" + "os" + "os/exec" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/build/docker" + "github.com/grafana/grafana/pkg/build/gcloud" + "github.com/urfave/cli/v2" +) + +func Enterprise2(c *cli.Context) error { + if c.NArg() > 0 { + if err := cli.ShowSubcommandHelp(c); err != nil { + return cli.NewExitError(err.Error(), 1) + } + return cli.NewExitError("", 1) + } + + if err := gcloud.ActivateServiceAccount(); err != nil { + return fmt.Errorf("couldn't activate service account, err: %w", err) + } + + metadata, err := GenerateMetadata(c) + if err != nil { + return err + } + + buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode) + if err != nil { + return err + } + + cfg := docker.Config{ + Archs: buildConfig.Docker.Architectures, + Distribution: buildConfig.Docker.Distribution, + DockerHubRepo: c.String("dockerhub-repo"), + Tag: metadata.GrafanaVersion, + } + + err = dockerLoginEnterprise2() + if err != nil { + return err + } + + var distributionStr []string + for _, distribution := range cfg.Distribution { + switch distribution { + case alpine: + distributionStr = append(distributionStr, "") + case ubuntu: + distributionStr = append(distributionStr, "-ubuntu") + default: + return fmt.Errorf("unrecognized distribution %q", distribution) + } + } + + for _, distribution := range distributionStr { + var imageFileNames []string + for _, arch := range cfg.Archs { + imageFilename := fmt.Sprintf("%s:%s%s-%s", cfg.DockerHubRepo, cfg.Tag, distribution, arch) + err := docker.PushImage(imageFilename) + if err != nil { + return err + } + imageFileNames = append(imageFileNames, imageFilename) + } + manifest := fmt.Sprintf("%s:%s%s", cfg.DockerHubRepo, cfg.Tag, distribution) + args := []string{"manifest", "create", manifest} + args = append(args, imageFileNames...) + + //nolint:gosec + cmd := exec.Command("docker", args...) + cmd.Env = append(os.Environ(), "DOCKER_CLI_EXPERIMENTAL=enabled") + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create Docker manifest: %w\n%s", err, output) + } + + err = docker.PushManifest(manifest) + if err != nil { + return err + } + } + + return nil +} + +func dockerLoginEnterprise2() error { + log.Println("Docker login...") + cmd := exec.Command("gcloud", "auth", "configure-docker") + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("error logging in to DockerHub: %s %q", out, err) + } + + log.Println("Successful login!") + return nil +} diff --git a/pkg/build/docker/build.go b/pkg/build/docker/build.go index f77d54cd43e..7267d4a094b 100644 --- a/pkg/build/docker/build.go +++ b/pkg/build/docker/build.go @@ -85,6 +85,11 @@ func BuildImage(version string, arch config.Architecture, grafanaDir string, use var additionalDockerRepo string var tags []string var imageFileBase string + var dockerEnterprise2Repo string + if repo, ok := os.LookupEnv("DOCKER_ENTERPRISE2_REPO"); ok { + dockerEnterprise2Repo = repo + } + switch edition { case config.EditionOSS: dockerRepo = "grafana/grafana-image-tags" @@ -94,6 +99,10 @@ func BuildImage(version string, arch config.Architecture, grafanaDir string, use dockerRepo = "grafana/grafana-enterprise-image-tags" imageFileBase = "grafana-enterprise" editionStr = "-enterprise" + case config.EditionEnterprise2: + dockerRepo = dockerEnterprise2Repo + imageFileBase = "grafana-enterprise2" + editionStr = "-enterprise2" default: return []string{}, fmt.Errorf("unrecognized edition %s", edition) } diff --git a/pkg/build/docker/push.go b/pkg/build/docker/push.go new file mode 100644 index 00000000000..da87ef2cb16 --- /dev/null +++ b/pkg/build/docker/push.go @@ -0,0 +1,62 @@ +package docker + +import ( + "fmt" + "log" + "os" + "os/exec" + "time" +) + +const ( + tries = 3 + sleepTime = 30 +) + +func PushImage(newImage string) error { + var err error + for i := 0; i < tries; i++ { + log.Printf("push attempt #%d...", i+1) + var out []byte + cmd := exec.Command("docker", "push", newImage) + cmd.Dir = "." + out, err = cmd.CombinedOutput() + if err != nil { + log.Printf("output: %s", out) + log.Printf("sleep for %d, before retrying...", sleepTime) + time.Sleep(sleepTime * time.Second) + } else { + log.Printf("Successfully pushed %s!", newImage) + break + } + } + if err != nil { + return fmt.Errorf("error pushing images to DockerHub: %q", err) + } + return nil +} + +func PushManifest(manifest string) error { + log.Printf("Pushing Docker manifest %s...", manifest) + + var err error + for i := 0; i < tries; i++ { + log.Printf("push attempt #%d...", i+1) + var out []byte + cmd := exec.Command("docker", "manifest", "push", manifest) + cmd.Env = append(os.Environ(), "DOCKER_CLI_EXPERIMENTAL=enabled") + out, err = cmd.CombinedOutput() + if err != nil { + log.Printf("output: %s", out) + log.Printf("sleep for %d, before retrying...", sleepTime) + time.Sleep(sleepTime * time.Second) + } else { + log.Printf("Successful manifest push! %s", string(out)) + break + } + } + if err != nil { + return fmt.Errorf("failed to push manifest, err: %w", err) + } + return nil +} diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index ce5442e555c..a0049840a0d 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -73,6 +73,16 @@ load( load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token', 'prerelease_bucket') +ver_mode='release' +release_trigger = { + 'event': { + 'exclude': [ + 'promote' + ] + }, + 'ref': ['refs/tags/v*',], +} + def store_npm_packages_step(): return { 'name': 'store-npm-packages', @@ -122,7 +132,7 @@ def release_npm_packages_step(): ], } -def get_oss_pipelines(trigger, ver_mode): +def oss_pipelines(ver_mode=ver_mode, trigger=release_trigger): environment = {'EDITION': 'oss'} edition = 'oss' services = integration_test_services(edition=edition) @@ -223,7 +233,7 @@ def get_oss_pipelines(trigger, ver_mode): pipelines.extend([windows_pipeline]) return pipelines -def get_enterprise_pipelines(trigger, ver_mode): +def enterprise_pipelines(ver_mode=ver_mode, trigger=release_trigger): environment = {'EDITION': 'enterprise'} edition = 'enterprise' services = integration_test_services(edition=edition) @@ -350,6 +360,78 @@ def get_enterprise_pipelines(trigger, ver_mode): return pipelines +def enterprise2_pipelines(prefix='', ver_mode=ver_mode, trigger=release_trigger): + environment = { + 'EDITION': 'enterprise2', + } + edition = 'enterprise' + services = integration_test_services(edition=edition) + volumes = integration_test_services_volumes() + package_steps = [] + publish_steps = [] + should_publish = ver_mode == 'release' + should_upload = should_publish or ver_mode in ('release-branch',) + include_enterprise = edition == 'enterprise' + edition2 = 'enterprise2' + init_steps = [ + download_grabpl_step(), + identify_runner_step(), + clone_enterprise_step(ver_mode), + init_enterprise_step(ver_mode), + compile_build_cmd(edition), + ] + + build_steps = [ + build_frontend_step(edition=edition, ver_mode=ver_mode), + build_frontend_package_step(edition=edition, ver_mode=ver_mode), + build_plugins_step(edition=edition, ver_mode=ver_mode), + ] + + if include_enterprise: + build_steps.extend([ + build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-amd64']), + ]) + + fetch_images = fetch_images_step(edition2) + fetch_images.update({'depends_on': ['build-docker-images', 'build-docker-images-ubuntu']}) + upload_cdn = upload_cdn_step(edition=edition2, ver_mode=ver_mode) + upload_cdn['environment'].update({'ENTERPRISE2_CDN_PATH': from_secret('enterprise2-cdn-path')}) + + build_steps.extend([ + package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise, variants=['linux-amd64']), + upload_cdn, + copy_packages_for_docker_step(edition=edition2), + build_docker_images_step(edition=edition2, ver_mode=ver_mode, publish=True), + build_docker_images_step(edition=edition2, ver_mode=ver_mode, ubuntu=True, publish=True), + fetch_images, + publish_images_step(edition2, 'release', mode=edition2, docker_repo='${{DOCKER_ENTERPRISE2_REPO}}'), + ]) + + if should_upload: + step = upload_packages_step(edition=edition2, ver_mode=ver_mode) + if step: + publish_steps.append(step) + + deps_on_clone_enterprise_step = { + 'depends_on': [ + 'init-enterprise', + ] + } + + for step in [wire_install_step(), yarn_install_step(), verify_gen_cue_step(edition)]: + step.update(deps_on_clone_enterprise_step) + init_steps.extend([step]) + + pipelines = [ + pipeline( + name='{}{}-enterprise2-build{}-publish'.format(prefix, ver_mode, get_e2e_suffix()), edition=edition, trigger=trigger, services=[], + steps=init_steps + build_steps + package_steps + publish_steps, + volumes=volumes, environment=environment, + ), + ] + + return pipelines + def publish_artifacts_step(mode): security = '' if mode == 'security': @@ -435,28 +517,6 @@ def artifacts_page_pipeline(): return [pipeline(name='publish-artifacts-page', trigger=trigger, steps = [download_grabpl_step(), artifacts_page_step()], edition="all", environment = {'EDITION': 'all'} )] -def release_pipelines(ver_mode='release', trigger=None): - # 'enterprise' edition services contain both OSS and enterprise services - if not trigger: - trigger = { - 'event': { - 'exclude': [ - 'promote' - ] - }, - 'ref': ['refs/tags/v*',], - } - - # The release pipelines include also enterprise ones, so both editions are built for a release. - # We could also solve this by triggering a downstream build for the enterprise repo, but by including enterprise - # in OSS release builds, we simplify the UX for the release engineer. - oss_pipelines = get_oss_pipelines(ver_mode=ver_mode, trigger=trigger) - enterprise_pipelines = get_enterprise_pipelines(ver_mode=ver_mode, trigger=trigger) - - pipelines = oss_pipelines + enterprise_pipelines - - return pipelines - def get_e2e_suffix(): if not disable_tests: return '-e2e' diff --git a/scripts/drone/pipelines/build.star b/scripts/drone/pipelines/build.star index a09122ae65b..730efbd099b 100644 --- a/scripts/drone/pipelines/build.star +++ b/scripts/drone/pipelines/build.star @@ -93,8 +93,8 @@ def build_e2e(trigger, ver_mode, edition): build_steps.extend([ build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=False), build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=False), - publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana/grafana', trigger=trigger_oss), - publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana/grafana-oss', trigger=trigger_oss), + publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana', trigger=trigger_oss), + publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana-oss', trigger=trigger_oss), release_canary_npm_packages_step(edition, trigger=trigger_oss), upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss), upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss) diff --git a/scripts/drone/pipelines/publish_images.star b/scripts/drone/pipelines/publish_images.star index 7e760ef8ff0..3481dadcd73 100644 --- a/scripts/drone/pipelines/publish_images.star +++ b/scripts/drone/pipelines/publish_images.star @@ -34,9 +34,9 @@ def publish_image_pipelines_public(): 'target': [mode], } return [pipeline( - name='publish-docker-oss-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='oss', mode=mode, docker_repo='grafana/grafana'), edition="", environment = {'EDITION': 'oss'} + name='publish-docker-oss-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='oss', mode=mode, docker_repo='grafana'), edition="", environment = {'EDITION': 'oss'} ), pipeline( - name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='enterprise', mode=mode, docker_repo='grafana/grafana-enterprise'), edition="", environment = {'EDITION': 'enterprise'} + name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='enterprise', mode=mode, docker_repo='grafana-enterprise'), edition="", environment = {'EDITION': 'enterprise'} ),] def publish_image_pipelines_security(): @@ -46,5 +46,5 @@ def publish_image_pipelines_security(): 'target': [mode], } return [pipeline( - name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='enterprise', mode=mode, docker_repo='grafana/grafana-enterprise'), edition="", environment = {'EDITION': 'enterprise'} + name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='enterprise', mode=mode, docker_repo='grafana-enterprise'), edition="", environment = {'EDITION': 'enterprise'} ),] diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 3ee32a19cb3..b2513e5633e 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -622,15 +622,10 @@ def codespell_step(): def package_step(edition, ver_mode, include_enterprise2=False, variants=None): deps = [ 'build-plugins', - 'build-backend', + 'build-backend' + enterprise2_suffix(edition), 'build-frontend', 'build-frontend-packages', ] - if include_enterprise2: - sfx = '-enterprise2' - deps.extend([ - 'build-backend' + sfx, - ]) variants_str = '' if variants: @@ -773,12 +768,12 @@ def build_docs_website_step(): } -def copy_packages_for_docker_step(): +def copy_packages_for_docker_step(edition=None): return { 'name': 'copy-packages-for-docker', 'image': build_image, 'depends_on': [ - 'package', + 'package' + enterprise2_suffix(edition), ], 'commands': [ 'ls dist/*.tar.gz*', @@ -800,6 +795,13 @@ def build_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publis if archs: cmd += ' -archs {}'.format(','.join(archs)) + environment = { + 'GCP_KEY': from_secret('gcp_key'), + } + + if edition == 'enterprise2': + environment.update({'DOCKER_ENTERPRISE2_REPO': from_secret('docker_enterprise2_repo')}) + return { 'name': 'build-docker-images' + ubuntu_sfx, 'image': 'google/cloud-sdk', @@ -814,9 +816,7 @@ def build_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publis 'name': 'docker', 'path': '/var/run/docker.sock' }], - 'environment': { - 'GCP_KEY': from_secret('gcp_key'), - }, + 'environment': environment } def fetch_images_step(edition): @@ -827,6 +827,7 @@ def fetch_images_step(edition): 'GCP_KEY': from_secret('gcp_key'), 'DOCKER_USER': from_secret('docker_username'), 'DOCKER_PASSWORD': from_secret('docker_password'), + 'DOCKER_ENTERPRISE2_REPO': from_secret('docker_enterprise2_repo'), }, 'commands': ['./bin/build artifacts docker fetch --edition {}'.format(edition)], 'depends_on': ['compile-build-cmd'], @@ -838,13 +839,20 @@ def fetch_images_step(edition): def publish_images_step(edition, ver_mode, mode, docker_repo, trigger=None): + name = docker_repo + docker_repo = 'grafana/{}'.format(docker_repo) if mode == 'security': mode = '--{} '.format(mode) else: mode = '' - cmd = './bin/grabpl artifacts docker publish {}--dockerhub-repo {}'.format( - mode, docker_repo) + environment = { + 'GCP_KEY': from_secret('gcp_key'), + 'DOCKER_USER': from_secret('docker_username'), + 'DOCKER_PASSWORD': from_secret('docker_password'), + } + + cmd = './bin/grabpl artifacts docker publish {}--dockerhub-repo {}'.format(mode, docker_repo) if ver_mode == 'release': deps = ['fetch-images-{}'.format(edition)] @@ -852,14 +860,16 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, trigger=None): else: deps = ['build-docker-images', 'build-docker-images-ubuntu'] + if edition == 'enterprise2': + name = edition + docker_repo = '$${DOCKER_ENTERPRISE2_REPO}' + environment.update({'GCP_KEY': from_secret('gcp_key_hg'), 'DOCKER_ENTERPRISE2_REPO': from_secret('docker_enterprise2_repo')}) + cmd = './bin/build artifacts docker publish-enterprise2 --dockerhub-repo {}'.format(docker_repo) + step = { - 'name': 'publish-images-{}'.format(docker_repo), + 'name': 'publish-images-{}'.format(name), 'image': 'google/cloud-sdk', - 'environment': { - 'GCP_KEY': from_secret('gcp_key'), - 'DOCKER_USER': from_secret('docker_username'), - 'DOCKER_PASSWORD': from_secret('docker_password'), - }, + 'environment': environment, 'commands': [cmd], 'depends_on': deps, 'volumes': [{ diff --git a/scripts/drone/version.star b/scripts/drone/version.star index 80f0c6ac48f..651a557ec75 100644 --- a/scripts/drone/version.star +++ b/scripts/drone/version.star @@ -1,11 +1,12 @@ load( 'scripts/drone/events/release.star', - 'release_pipelines', + 'oss_pipelines', + 'enterprise_pipelines', + 'enterprise2_pipelines', ) ver_mode = 'release-branch' +trigger={'ref': ['refs/heads/v[0-9]*']} def version_branch_pipelines(): - return release_pipelines(ver_mode=ver_mode, trigger={ - 'ref': ['refs/heads/v[0-9]*'], - }) + return oss_pipelines(ver_mode=ver_mode, trigger=trigger) + enterprise_pipelines(ver_mode=ver_mode, trigger=trigger) + enterprise2_pipelines(ver_mode=ver_mode, trigger=trigger)